diff options
author | RaNaN <Mast3rRaNaN@hotmail.de> | 2011-12-22 23:45:38 +0100 |
---|---|---|
committer | RaNaN <Mast3rRaNaN@hotmail.de> | 2011-12-22 23:45:38 +0100 |
commit | 958bf611f5d9d117f19f824990ec6fd6b537e967 (patch) | |
tree | 558cb45fa61b1738629dff9727c028badccd9990 | |
parent | some bugfixes (diff) | |
download | pyload-958bf611f5d9d117f19f824990ec6fd6b537e967.tar.xz |
accountmanager v2, delete your accounts.conf and re-enter them in pyload,
new nice debug functions, try core.shell() and core.breakpoint()
30 files changed, 628 insertions, 743 deletions
diff --git a/module/Api.py b/module/Api.py index c787819e2..f6375f266 100644 --- a/module/Api.py +++ b/module/Api.py @@ -849,12 +849,11 @@ class Api(Iface): :param refresh: reload account info :return: list of `AccountInfo` """ - accs = self.core.accountManager.getAccountInfos(False, refresh) + accs = self.core.accountManager.getAllAccounts(refresh) accounts = [] - for group in accs.values(): - accounts.extend([AccountInfo(acc["validuntil"], acc["login"], acc["options"], acc["valid"], - acc["trafficleft"], acc["maxtraffic"], acc["premium"], acc["type"]) - for acc in group]) + for plugin in accs.itervalues(): + accounts.extend(plugin.values()) + return accounts @permission(PERMS.ALL) @@ -863,7 +862,7 @@ class Api(Iface): :return: list """ - return self.core.accountManager.accounts.keys() + return self.core.pluginManager.getAccountPlugins() @permission(PERMS.ACCOUNTS) def updateAccount(self, plugin, account, password=None, options={}): diff --git a/module/PluginThread.py b/module/PluginThread.py index e91d6d819..984b88e16 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -40,7 +40,6 @@ from Api import OnlineStatus class PluginThread(Thread): """abstract base class for thread types""" - #---------------------------------------------------------------------- def __init__(self, manager): """Constructor""" Thread.__init__(self) @@ -137,7 +136,7 @@ class PluginThread(Thread): if pyfile.pluginname in self.m.core.config.plugin: dump += "\n\nCONFIG: \n\n" - dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" + dump += pformat(self.m.core.config.values) + "\n" return dump @@ -150,7 +149,6 @@ class PluginThread(Thread): class DownloadThread(PluginThread): """thread for downloading files from 'real' hoster plugins""" - #---------------------------------------------------------------------- def __init__(self, manager): """Constructor""" PluginThread.__init__(self, manager) @@ -160,7 +158,6 @@ class DownloadThread(PluginThread): self.start() - #---------------------------------------------------------------------- def run(self): """run method""" pyfile = None @@ -461,7 +458,7 @@ class HookThread(PluginThread): #dirty method to filter out exceptions if "unexpected keyword argument 'thread'" not in e.args[0]: raise - + del self.kwargs["thread"] self.f(*self.args, **self.kwargs) finally: diff --git a/module/ThreadManager.py b/module/ThreadManager.py index 8937f4a29..033d80fdc 100644 --- a/module/ThreadManager.py +++ b/module/ThreadManager.py @@ -310,7 +310,8 @@ class ThreadManager: thread = PluginThread.DecrypterThread(self, job) def getLimit(self, thread): - limit = thread.active.plugin.account.getAccountData(thread.active.plugin.user)["options"].get("limitDL",["0"])[0] + limit = thread.active.plugin.account.options.get("limitDL","0") + if limit == "": limit = "0" return int(limit) def cleanup(self): diff --git a/module/Utils.py b/module/Utils.py index b26a9960c..b80cfe1c2 100644 --- a/module/Utils.py +++ b/module/Utils.py @@ -104,6 +104,8 @@ def formatSpeed(speed): def freeSpace(folder): + folder = fs_encode(folder) + if os.name == "nt": import ctypes diff --git a/module/config/ConfigParser.py b/module/config/ConfigParser.py index 33b3d26b6..1d3ae87d6 100644 --- a/module/config/ConfigParser.py +++ b/module/config/ConfigParser.py @@ -206,10 +206,6 @@ class ConfigParser: if base: if section not in self.baseSections: self.baseSections.append(section) - else: - if section in self.config: - print "Section already exists", section - return data = SectionTuple(gettext(name), gettext(desc), gettext(long_desc), d) self.config[section] = data diff --git a/module/database/AccountDatabase.py b/module/database/AccountDatabase.py new file mode 100644 index 000000000..1602451fa --- /dev/null +++ b/module/database/AccountDatabase.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +from module.database import queue, async +from module.database import DatabaseBackend + +class AccountMethods: + + @queue + def loadAccounts(db): + db.c.execute('SELECT plugin, loginname, activated, password, options FROM accounts;') + return db.c.fetchall() + + @async + def saveAccounts(db, data): + db.c.executemany('INSERT INTO accounts(plugin, loginname, activated, password, options) VALUES(?,?,?,?,?)', data) + + @async + def removeAccount(db, plugin, loginname): + db.c.execute('DELETE FROM accounts WHERE plugin=? AND loginname=?', (plugin, loginname)) + +DatabaseBackend.registerSub(AccountMethods)
\ No newline at end of file diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index db8b1aa3c..e10bcbbaf 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -32,55 +32,58 @@ try: except: import sqlite3 +DB = None DB_VERSION = 4 -class style(): - db = None - - @classmethod - def setDB(cls, db): - cls.db = db - - @classmethod - def inner(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return f(cls.db, *args, **kwargs) - return x - - @classmethod - def queue(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.queue(f, *args, **kwargs) - return x - - @classmethod - def async(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.async(f, *args, **kwargs) - return x +def set_DB(db): + global DB + DB = db + + +def queue(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return DB.queue(f, *args, **kwargs) + + return x + + +def async(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return DB.async(f, *args, **kwargs) + + return x + + +def inner(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return f(DB, *args, **kwargs) + + return x + class DatabaseJob(): def __init__(self, f, *args, **kwargs): self.done = Event() - + self.f = f self.args = args self.kwargs = kwargs - + self.result = None self.exception = False -# import inspect -# self.frame = inspect.currentframe() + # import inspect + # self.frame = inspect.currentframe() def __repr__(self): from os.path import basename + frame = self.frame.f_back output = "" for i in range(5): @@ -104,45 +107,50 @@ class DatabaseJob(): self.exception = e finally: self.done.set() - + def wait(self): self.done.wait() + class DatabaseBackend(Thread): subs = [] + + DB_FILE = "pyload.db" + VERSION_FILE = "db.version" + def __init__(self, core): Thread.__init__(self) self.setDaemon(True) self.core = core self.jobs = Queue() - + self.setuplock = Event() - - style.setDB(self) - + + set_DB(self) + def setup(self): self.start() self.setuplock.wait() - + def run(self): """main loop, which executes commands""" convert = self._checkVersion() #returns None or current version - - self.conn = sqlite3.connect("files.db") - chmod("files.db", 0600) + + self.conn = sqlite3.connect(self.DB_FILE) + chmod(self.DB_FILE, 0600) self.c = self.conn.cursor() #compatibility - + if convert is not None: self._convertDB(convert) - + self._createTables() self.conn.commit() - + self.setuplock.set() - + while True: j = self.jobs.get() if j == "quit": @@ -151,20 +159,23 @@ class DatabaseBackend(Thread): break j.processJob() - @style.queue + @queue def shutdown(self): self.conn.commit() self.jobs.put("quit") def _checkVersion(self): """ check db version and delete it if needed""" - if not exists("files.version"): - f = open("files.version", "wb") + if not exists(self.VERSION_FILE): + f = open(self.VERSION_FILE, "wb") f.write(str(DB_VERSION)) f.close() return - - f = open("files.version", "rb") + + if exists("files.db") and not exists(self.DB_FILE): + move("files.db", self.DB_FILE) + + f = open(self.VERSION_FILE, "rb") v = int(f.read().strip()) f.close() if v < DB_VERSION: @@ -173,13 +184,13 @@ class DatabaseBackend(Thread): self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) except: print "Filedatabase was deleted due to incompatible version." - remove("files.version") - move("files.db", "files.backup.db") - f = open("files.version", "wb") + remove(self.VERSION_FILE) + move(self.DB_FILE, self.DB_FILE + ".backup") + f = open(self.VERSION_FILE, "wb") f.write(str(DB_VERSION)) f.close() return v - + def _convertDB(self, v): try: getattr(self, "_convertV%i" % v)() @@ -188,22 +199,28 @@ class DatabaseBackend(Thread): self.core.log.error(_("Filedatabase could NOT be converted.")) except: print "Filedatabase could NOT be converted." - + #--convert scripts start def _convertV4(self): pass #--convert scripts end - + def _createTables(self): """create tables for database""" - self.c.execute('CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') - self.c.execute('CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)') - self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') - self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "users" ("name" TEXT PRIMARY KEY NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "accounts" ("plugin" TEXT NOT NULL, "loginname" TEXT NOT NULL, "activated" INTEGER DEFAULT 1, "password" TEXT DEFAULT "", "options" TEXT DEFAULT "", PRIMARY KEY (plugin, loginname) ON CONFLICT REPLACE)') self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \ SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\ @@ -221,7 +238,6 @@ class DatabaseBackend(Thread): fid = 0 self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links")) - self.c.execute('SELECT max(id) FROM packages') pid = self.c.fetchone()[0] if pid: @@ -235,39 +251,39 @@ class DatabaseBackend(Thread): def createCursor(self): return self.conn.cursor() - - @style.async + + @async def commit(self): self.conn.commit() - @style.queue + @queue def syncSave(self): self.conn.commit() - - @style.async + + @async def rollback(self): self.conn.rollback() - + def async(self, f, *args, **kwargs): args = (self, ) + args job = DatabaseJob(f, *args, **kwargs) self.jobs.put(job) - + def queue(self, f, *args, **kwargs): args = (self, ) + args job = DatabaseJob(f, *args, **kwargs) self.jobs.put(job) job.wait() return job.result - + @classmethod def registerSub(cls, klass): cls.subs.append(klass) - + @classmethod def unregisterSub(cls, klass): cls.subs.remove(klass) - + def __getattr__(self, attr): for sub in DatabaseBackend.subs: if hasattr(sub, attr): @@ -276,45 +292,47 @@ class DatabaseBackend(Thread): if __name__ == "__main__": db = DatabaseBackend() db.setup() - + class Test(): - @style.queue + @queue def insert(db): c = db.createCursor() for i in range(1000): c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - @style.async + + @async def insert2(db): c = db.createCursor() - for i in range(1000*1000): + for i in range(1000 * 1000): c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - - @style.queue + + @queue def select(db): c = db.createCursor() for i in range(10): res = c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", ("foo", i)) print res.fetchone() - - @style.queue + + @queue def error(db): c = db.createCursor() print "a" c.execute("SELECT myerror FROM storage WHERE identifier=? AND key=?", ("foo", i)) print "e" - + db.registerSub(Test) from time import time + start = time() for i in range(100): db.insert() end = time() - print end-start - + print end - start + start = time() db.insert2() end = time() - print end-start - + print end - start + db.error() diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 4084c46f7..b5c386802 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -25,7 +25,7 @@ from module.utils import formatSize, lock from module.interaction.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent from module.PyPackage import PyPackage from module.PyFile import PyFile -from module.database import style, DatabaseBackend +from module.database import DatabaseBackend, queue, async, inner try: from pysqlite2 import dbapi2 as sqlite3 @@ -574,25 +574,25 @@ class FileHandler: self.db.restartFailed() class FileMethods(): - @style.queue + @queue def filecount(self, queue): """returns number of files in queue""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue, )) return self.c.fetchone()[0] - @style.queue + @queue def queuecount(self, queue): """ number of files in queue not finished yet""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0,4)", (queue, )) return self.c.fetchone()[0] - @style.queue + @queue def processcount(self, queue, fid): """ number of files which have to be proccessed """ self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2,3,5,7,12) AND l.id != ?", (queue, str(fid))) return self.c.fetchone()[0] - @style.inner + @inner def _nextPackageOrder(self, queue=0): self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) max = self.c.fetchone()[0] @@ -601,7 +601,7 @@ class FileMethods(): else: return 0 - @style.inner + @inner def _nextFileOrder(self, package): self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) max = self.c.fetchone()[0] @@ -610,13 +610,13 @@ class FileMethods(): else: return 0 - @style.queue + @queue def addLink(self, url, name, plugin, package): order = self._nextFileOrder(package) self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', (url, name, plugin, package, order)) return self.c.lastrowid - @style.queue + @queue def addLinks(self, links, package): """ links is a list of tupels (url,plugin)""" order = self._nextFileOrder(package) @@ -624,27 +624,27 @@ class FileMethods(): links = [(x[0], x[0], x[1], package, o) for x, o in zip(links, orders)] self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) - @style.queue + @queue def addPackage(self, name, folder, queue): order = self._nextPackageOrder(queue) self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', (name, folder, queue, order)) return self.c.lastrowid - @style.queue + @queue def deletePackage(self, p): self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue)) - @style.queue + @queue def deleteLink(self, f): self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid))) - @style.queue + @queue def getAllLinks(self, q): """return information about all links in queue q @@ -677,7 +677,7 @@ class FileMethods(): return data - @style.queue + @queue def getAllPackages(self, q): """return information about packages in queue q (only useful in get all data) @@ -714,7 +714,7 @@ class FileMethods(): return data - @style.queue + @queue def getLinkData(self, id): """get link information as dict""" self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE id=?', (str(id), )) @@ -738,7 +738,7 @@ class FileMethods(): return data - @style.queue + @queue def getPackageData(self, id): """get data about links for a package""" self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id), )) @@ -762,15 +762,15 @@ class FileMethods(): return data - @style.async + @async def updateLink(self, f): self.c.execute('UPDATE links SET url=?,name=?,size=?,status=?,error=?,package=? WHERE id=?', (f.url, f.name, f.size, f.status, f.error, str(f.packageid), str(f.id))) - @style.queue + @queue def updatePackage(self, p): self.c.execute('UPDATE packages SET name=?,folder=?,site=?,password=?,queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) - @style.queue + @queue def updateLinkInfo(self, data): """ data is list of tupels (name, size, status, url) """ self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1,2,3,14)', data) @@ -780,7 +780,7 @@ class FileMethods(): ids.append(int(r[0])) return ids - @style.queue + @queue def reorderPackage(self, p, position, noMove=False): if position == -1: position = self._nextPackageOrder(p.queue) @@ -792,7 +792,7 @@ class FileMethods(): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) - @style.queue + @queue def reorderLink(self, f, position): """ reorder link with f as dict for pyfile """ if f["order"] > position: @@ -803,20 +803,20 @@ class FileMethods(): self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"])) - @style.queue + @queue def clearPackageOrder(self, p): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) - @style.async + @async def restartFile(self, id): self.c.execute('UPDATE links SET status=3,error="" WHERE id=?', (str(id),)) - @style.async + @async def restartPackage(self, id): self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) - @style.queue + @queue def getPackage(self, id): """return package instance from id""" self.c.execute("SELECT name,folder,site,password,queue,packageorder FROM packages WHERE id=?", (str(id), )) @@ -825,7 +825,7 @@ class FileMethods(): return PyPackage(self.manager, id, * r) #---------------------------------------------------------------------- - @style.queue + @queue def getFile(self, id): """return link instance from id""" self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id), )) @@ -834,7 +834,7 @@ class FileMethods(): return PyFile(self.manager, id, * r) - @style.queue + @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" @@ -854,7 +854,7 @@ class FileMethods(): return [x[0] for x in self.c] - @style.queue + @queue def getPluginJob(self, plugins): """returns pyfile ids with suited plugins""" cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins @@ -863,29 +863,29 @@ class FileMethods(): return [x[0] for x in self.c] - @style.queue + @queue def getUnfinished(self, pid): """return list of max length 3 ids with pyfiles in package not finished or processed""" self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) return [r[0] for r in self.c] - @style.queue + @queue def deleteFinished(self): self.c.execute("DELETE FROM links WHERE status IN (0,4)") self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") - @style.queue + @queue def restartFailed(self): self.c.execute("UPDATE links SET status=3,error='' WHERE status IN (8, 9)") - @style.queue + @queue def findDuplicates(self, id, folder, filename): """ checks if filename exists with different id and same package """ self.c.execute("SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", (folder, id, filename)) return self.c.fetchone() - @style.queue + @queue def purgeLinks(self): self.c.execute("DELETE FROM links;") self.c.execute("DELETE FROM packages;") diff --git a/module/database/StorageDatabase.py b/module/database/StorageDatabase.py index 3ed29625f..ffaf51763 100644 --- a/module/database/StorageDatabase.py +++ b/module/database/StorageDatabase.py @@ -16,11 +16,10 @@ @author: mkaay """ -from module.database import style -from module.database import DatabaseBackend +from module.database import DatabaseBackend, queue class StorageMethods(): - @style.queue + @queue def setStorage(db, identifier, key, value): db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) if db.c.fetchone() is not None: @@ -28,7 +27,7 @@ class StorageMethods(): else: db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) - @style.queue + @queue def getStorage(db, identifier, key=None): if key is not None: db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) @@ -42,7 +41,7 @@ class StorageMethods(): d[row[0]] = row[1] return d - @style.queue + @queue def delStorage(db, identifier, key): db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) diff --git a/module/database/UserDatabase.py b/module/database/UserDatabase.py index 0c781057d..a5077711d 100644 --- a/module/database/UserDatabase.py +++ b/module/database/UserDatabase.py @@ -19,11 +19,10 @@ from hashlib import sha1 import random -from DatabaseBackend import DatabaseBackend -from DatabaseBackend import style +from DatabaseBackend import DatabaseBackend, queue, async class UserMethods(): - @style.queue + @queue def checkAuth(db, user, password): c = db.c c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user, )) @@ -40,7 +39,7 @@ class UserMethods(): else: return {} - @style.queue + @queue def addUser(db, user, password): salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) h = sha1(salt + password) @@ -54,7 +53,7 @@ class UserMethods(): c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) - @style.queue + @queue def changePassword(db, user, oldpw, newpw): db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user, )) r = db.c.fetchone() @@ -75,16 +74,16 @@ class UserMethods(): return False - @style.async + @async def setPermission(db, user, perms): db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) - @style.async + @async def setRole(db, user, role): db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) - @style.queue + @queue def listUsers(db): db.c.execute('SELECT name FROM users') users = [] @@ -92,7 +91,7 @@ class UserMethods(): users.append(row[0]) return users - @style.queue + @queue def getAllUserData(db): db.c.execute("SELECT name, permission, role, template, email FROM users") user = {} @@ -101,7 +100,7 @@ class UserMethods(): return user - @style.queue + @queue def removeUser(db, user): db.c.execute('DELETE FROM users WHERE name=?', (user, )) diff --git a/module/database/__init__.py b/module/database/__init__.py index 545789c0c..39848ac58 100644 --- a/module/database/__init__.py +++ b/module/database/__init__.py @@ -1,6 +1,6 @@ -from DatabaseBackend import DatabaseBackend -from DatabaseBackend import style +from DatabaseBackend import DatabaseBackend, queue, async, inner from FileDatabase import FileHandler from UserDatabase import UserMethods -from StorageDatabase import StorageMethods
\ No newline at end of file +from StorageDatabase import StorageMethods +from AccountDatabase import AccountMethods
\ No newline at end of file diff --git a/module/interaction/EventManager.py b/module/interaction/EventManager.py new file mode 100644 index 000000000..c45c388f3 --- /dev/null +++ b/module/interaction/EventManager.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +from time import time + +from PullEvents import ReloadAllEvent +from module.utils import uniqify + +class EventManager: + def __init__(self, core): + self.core = core + self.clients = [] + + def newClient(self, uuid): + self.clients.append(Client(uuid)) + + def clean(self): + for n, client in enumerate(self.clients): + if client.lastActive + 30 < time(): + del self.clients[n] + + def getEvents(self, uuid): + events = [] + validUuid = False + for client in self.clients: + if client.uuid == uuid: + client.lastActive = time() + validUuid = True + while client.newEvents(): + events.append(client.popEvent().toList()) + break + if not validUuid: + self.newClient(uuid) + events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] + return uniqify(events, repr) + + def addEvent(self, event): + for client in self.clients: + client.addEvent(event) + + +class Client: + def __init__(self, uuid): + self.uuid = uuid + self.lastActive = time() + self.events = [] + + def newEvents(self): + return len(self.events) > 0 + + def popEvent(self): + if not len(self.events): + return None + return self.events.pop(0) + + def addEvent(self, event): + self.events.append(event)
\ No newline at end of file diff --git a/module/interaction/PullEvents.py b/module/interaction/PullEvents.py index 5ec76765e..f34b01d48 100644 --- a/module/interaction/PullEvents.py +++ b/module/interaction/PullEvents.py @@ -17,58 +17,6 @@ @author: mkaay """ -from time import time -from module.utils import uniqify - -class PullManager(): - def __init__(self, core): - self.core = core - self.clients = [] - - def newClient(self, uuid): - self.clients.append(Client(uuid)) - - def clean(self): - for n, client in enumerate(self.clients): - if client.lastActive + 30 < time(): - del self.clients[n] - - def getEvents(self, uuid): - events = [] - validUuid = False - for client in self.clients: - if client.uuid == uuid: - client.lastActive = time() - validUuid = True - while client.newEvents(): - events.append(client.popEvent().toList()) - break - if not validUuid: - self.newClient(uuid) - events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] - return uniqify(events, repr) - - def addEvent(self, event): - for client in self.clients: - client.addEvent(event) - -class Client(): - def __init__(self, uuid): - self.uuid = uuid - self.lastActive = time() - self.events = [] - - def newEvents(self): - return len(self.events) > 0 - - def popEvent(self): - if not len(self.events): - return None - return self.events.pop(0) - - def addEvent(self, event): - self.events.append(event) - class UpdateEvent(): def __init__(self, itype, iid, destination): assert itype == "pack" or itype == "file" diff --git a/module/network/Bucket.py b/module/network/Bucket.py index 69da277ae..ff80bda55 100644 --- a/module/network/Bucket.py +++ b/module/network/Bucket.py @@ -20,15 +20,18 @@ from time import time from threading import Lock +# 10kb minimum rate +MIN_RATE = 10240 + class Bucket: def __init__(self): - self.rate = 0 + self.rate = 0 # bytes per second, maximum targeted throughput self.tokens = 0 self.timestamp = time() self.lock = Lock() def __nonzero__(self): - return False if self.rate < 10240 else True + return False if self.rate < MIN_RATE else True def setRate(self, rate): self.lock.acquire() @@ -37,7 +40,7 @@ class Bucket: def consumed(self, amount): """ return time the process have to sleep, after consumed specified amount """ - if self.rate < 10240: return 0 #min. 10kb, may become unresponsive otherwise + if self.rate < MIN_RATE: return 0 #May become unresponsive otherwise self.lock.acquire() self.calc_tokens() @@ -47,7 +50,6 @@ class Bucket: time = -self.tokens/float(self.rate) else: time = 0 - self.lock.release() return time diff --git a/module/network/CookieJar.py b/module/network/CookieJar.py index c05812334..a020d6f9e 100644 --- a/module/network/CookieJar.py +++ b/module/network/CookieJar.py @@ -20,10 +20,12 @@ from time import time class CookieJar(): - def __init__(self, pluginname, account=None): + def __init__(self, pluginname): self.cookies = {} - self.plugin = pluginname - self.account = account + self.pluginname = pluginname + + def __repr__(self): + return ("<CookieJar plugin=%s>\n\t" % self.pluginname) + "\n\t".join(self.cookies.values()) def addCookies(self, clist): for c in clist: @@ -33,18 +35,18 @@ class CookieJar(): def getCookies(self): return self.cookies.values() - def parseCookie(self, name): + def getCookie(self, name): if name in self.cookies: return self.cookies[name].split("\t")[6] else: return None - def getCookie(self, name): - return self.parseCookie(name) + def setCookie(self, domain, name, value, path="/", exp=None): + if not exp: exp = time() + 3600 * 24 * 180 - def setCookie(self, domain, name, value, path="/", exp=time()+3600*24*180): - s = ".%s TRUE %s FALSE %s %s %s" % (domain, path, exp, name, value) + # dot makes it valid on all subdomains + s = ".%s TRUE %s FALSE %s %s %s" % (domain.strip("."), path, exp, name, value) self.cookies[name] = s def clear(self): - self.cookies = {} + self.cookies = {}
\ No newline at end of file diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 5b1528281..12fd66c95 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -24,34 +24,25 @@ from Bucket import Bucket from HTTPRequest import HTTPRequest from CookieJar import CookieJar -from XDCCRequest import XDCCRequest - class RequestFactory(): def __init__(self, core): self.lock = Lock() self.core = core self.bucket = Bucket() self.updateBucket() - self.cookiejars = {} + @property def iface(self): return self.core.config["download"]["interface"] - def getRequest(self, pluginName, account=None, type="HTTP"): - self.lock.acquire() - - if type == "XDCC": - return XDCCRequest(proxies=self.getProxies()) - + def getRequest(self, pluginName, cj=None): req = Browser(self.bucket, self.getOptions()) - if account: - cj = self.getCookieJar(pluginName, account) + if cj: req.setCookieJar(cj) else: req.setCookieJar(CookieJar(pluginName)) - self.lock.release() return req def getHTTPRequest(self, **kwargs): @@ -67,16 +58,12 @@ class RequestFactory(): rep = h.load(*args, **kwargs) finally: h.close() - - return rep - def getCookieJar(self, pluginName, account=None): - if (pluginName, account) in self.cookiejars: - return self.cookiejars[(pluginName, account)] + return rep - cj = CookieJar(pluginName, account) - self.cookiejars[(pluginName, account)] = cj - return cj + def openCookieJar(self, pluginname): + """Create new CookieJar""" + return CookieJar(pluginname) def getProxies(self): """ returns a proxy list for the request classes """ @@ -106,7 +93,7 @@ class RequestFactory(): def getOptions(self): """returns options needed for pycurl""" - return {"interface": self.iface(), + return {"interface": self.iface, "proxies": self.getProxies(), "ipv6": self.core.config["download"]["ipv6"]} diff --git a/module/plugins/Account.py b/module/plugins/Account.py index c147404e0..9da8d0357 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -1,63 +1,72 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from random import choice from time import time from traceback import print_exc from threading import RLock from Plugin import Base from module.utils import compare_time, parseFileSize, lock +from module.config.converter import from_string +from module.Api import AccountInfo +from module.network.CookieJar import CookieJar class WrongPassword(Exception): pass - -class Account(Base): +#noinspection PyUnresolvedReferences +class Account(Base, AccountInfo): """ Base class for every Account plugin. Just overwrite `login` and cookies will be stored and account becomes accessible in\ associated hoster plugin. Plugin should also provide `loadAccountInfo` """ - __name__ = "Account" - __version__ = "0.2" - __type__ = "account" - __description__ = """Account Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + + # Default values + valid = True + validuntil = None + trafficleft = None + maxtraffic = None + premium = True + activated = True #: after that time [in minutes] pyload will relogin the account login_timeout = 600 #: account data will be reloaded after this time info_threshold = 600 + # known options + known_opt = ["time", "limitDL"] - def __init__(self, manager, accounts): + + def __init__(self, manager, loginname, password, options): Base.__init__(self, manager.core) + if "activated" in options: + activated = from_string(options["activated"], "bool") + else: + activated = Account.activated + + for opt in self.known_opt: + if opt not in options: + options[opt] = "" + + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] + + # default account attributes + AccountInfo.__init__(self, self.__name__, loginname, Account.valid, Account.validuntil, Account.trafficleft, + Account.maxtraffic, Account.premium, activated, options) + self.manager = manager - self.accounts = {} - self.infos = {} # cache for account information + self.lock = RLock() + self.timestamp = 0 + self.login_ts = 0 # timestamp for login + self.cj = CookieJar(self.__name__) + self.password = password + self.error = None - self.timestamps = {} - self.setAccounts(accounts) self.init() def init(self): @@ -70,76 +79,68 @@ class Account(Base): :param data: data dictionary :param req: `Request` instance """ - pass + raise NotImplemented @lock - def _login(self, user, data): + def _login(self): # set timestamp for login - self.timestamps[user] = time() - - req = self.getAccountRequest(user) + self.login_ts = time() + + req = self.getAccountRequest() try: - self.login(user, data, req) + self.login(self.loginname, {"password": self.password}, req) + self.valid = True except WrongPassword: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": _("Wrong Password")}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": _("Wrong Password")}) + self.valid = False except Exception, e: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": e}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": e}) + self.valid = False if self.core.debug: print_exc() finally: - if req: req.close() - - def relogin(self, user): - req = self.getAccountRequest(user) - if req: - req.cj.clear() req.close() - if user in self.infos: - del self.infos[user] #delete old information - - self._login(user, self.accounts[user]) - def setAccounts(self, accounts): - self.accounts = accounts - for user, data in self.accounts.iteritems(): - self._login(user, data) - self.infos[user] = {} + def restoreDefaults(self): + self.valid = Account.valid + self.validuntil = Account.validuntil + self.trafficleft = Account.trafficleft + self.maxtraffic = Account.maxtraffic + self.premium = Account.premium + self.activated = Account.activated - def updateAccounts(self, user, password=None, options={}): + def update(self, password=None, options={}): """ updates account and return true if anything changed """ - if user in self.accounts: - self.accounts[user]["valid"] = True #do not remove or accounts will not login - if password: - self.accounts[user]["password"] = password - self.relogin(user) - return True - if options: - before = self.accounts[user]["options"] - self.accounts[user]["options"].update(options) - return self.accounts[user]["options"] != before - else: - self.accounts[user] = {"password": password, "options": options, "valid": True} - self._login(user, self.accounts[user]) + self.login_ts = 0 + + if "activated" in options: + self.activated = from_string(options["avtivated"], "bool") + + if password: + self.password = password + self._login() return True + if options: + # remove unknown options + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] - def removeAccount(self, user): - if user in self.accounts: - del self.accounts[user] - if user in self.infos: - del self.infos[user] - if user in self.timestamps: - del self.timestamps[user] + before = self.options + self.options.update(options) + return self.options != before + + def getAccountRequest(self): + return self.core.requestFactory.getRequest(self.__name__, self.cj) @lock - def getAccountInfo(self, name, force=False): + def getAccountInfo(self, force=False): """retrieve account infos for an user, do **not** overwrite this method!\\ just use it to retrieve infos in hoster plugins. see `loadAccountInfo` @@ -147,113 +148,55 @@ class Account(Base): :param force: reloads cached account information :return: dictionary with information """ - data = Account.loadAccountInfo(self, name) + if force or self.timestamp + self.info_threshold * 60 < time(): - if force or name not in self.infos: - self.logDebug("Get Account Info for %s" % name) - req = self.getAccountRequest(name) + # make sure to login + self.checkLogin() + self.logDebug("Get Account Info for %s" % self.loginname) + req = self.getAccountRequest() try: - infos = self.loadAccountInfo(name, req) - if not type(infos) == dict: - raise Exception("Wrong return format") + infos = self.loadAccountInfo(self.loginname, req) except Exception, e: infos = {"error": str(e)} - if req: req.close() + req.close() self.logDebug("Account Info: %s" % str(infos)) + self.timestamp = time() + + self.restoreDefaults() # reset to initial state + if type(infos) == dict: # copy result from dict to class + for k, v in infos.iteritems(): + if hasattr(self, k): + setattr(self, k, v) + else: + self.logDebug("Unknown attribute %s=%s" % (k, v)) + + def isPremium(self, user=None): + if user: self.logDebug("Deprecated Argument user for .isPremium()", user) + return self.premium + + def isUsable(self): + """Check several contraints to determine if account should be used""" + if not self.valid or not self.activated: return False + + if self.options["time"]: + time_data = "" + try: + time_data = self.options["time"] + start, end = time_data.split("-") + if not compare_time(start.split(":"), end.split(":")): + return False + except: + self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + + if 0 < self.validuntil < time(): + return False + if self.trafficleft is 0: # test explicity for 0 + return False - infos["timestamp"] = time() - self.infos[name] = infos - elif "timestamp" in self.infos[name] and self.infos[name][ - "timestamp"] + self.info_threshold * 60 < time(): - self.logDebug("Reached timeout for account data") - self.scheduleRefresh(name) - - data.update(self.infos[name]) - return data - - def isPremium(self, user): - info = self.getAccountInfo(user) - return info["premium"] - - def loadAccountInfo(self, name, req=None): - """this should be overwritten in account plugin,\ - and retrieving account information for user - - :param name: - :param req: `Request` instance - :return: - """ - return { - "validuntil": None, # -1 for unlimited - "login": name, - #"password": self.accounts[name]["password"], #@XXX: security - "options": self.accounts[name]["options"], - "valid": self.accounts[name]["valid"], - "trafficleft": None, # in kb, -1 for unlimited - "maxtraffic": None, - "premium": True, #useful for free accounts - "timestamp": 0, #time this info was retrieved - "type": self.__name__, - } - - def getAllAccounts(self, force=False): - return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] - - def getAccountRequest(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - req = self.core.requestFactory.getRequest(self.__name__, user) - return req - - def getAccountCookies(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - cj = self.core.requestFactory.getCookieJar(self.__name__, user) - return cj - - def getAccountData(self, user): - return self.accounts[user] - - def selectAccount(self): - """ returns an valid account name and data""" - usable = [] - for user, data in self.accounts.iteritems(): - if not data["valid"]: continue - - if "time" in data["options"] and data["options"]["time"]: - time_data = "" - try: - time_data = data["options"]["time"][0] - start, end = time_data.split("-") - if not compare_time(start.split(":"), end.split(":")): - continue - except: - self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) - - if user in self.infos: - if "validuntil" in self.infos[user]: - if self.infos[user]["validuntil"] > 0 and time() > self.infos[user]["validuntil"]: - continue - if "trafficleft" in self.infos[user]: - if self.infos[user]["trafficleft"] == 0: - continue - - usable.append((user, data)) - - if not usable: return None, None - return choice(usable) - - def canUse(self): - return False if self.selectAccount() == (None, None) else True + return True def parseTraffic(self, string): #returns kbyte return parseFileSize(string) / 1024 @@ -261,32 +204,36 @@ class Account(Base): def wrongPassword(self): raise WrongPassword - def empty(self, user): - if user in self.infos: - self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) + def empty(self, user=None): + if user: self.logDebug("Deprecated argument user for .empty()", user) + + self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % self.login) - self.infos[user].update({"trafficleft": 0}) - self.scheduleRefresh(user, 30 * 60) + self.trafficleft = 0 + self.scheduleRefresh(30 * 60) def expired(self, user): if user in self.infos: self.logWarning(_("Account %s is expired, checking again in 1h") % user) - self.infos[user].update({"validuntil": time() - 1}) - self.scheduleRefresh(user, 60 * 60) + self.validuntil = time() - 1 + self.scheduleRefresh(60 * 60) - def scheduleRefresh(self, user, time=0, force=True): + def scheduleRefresh(self, time=0, force=True): """ add task to refresh account info to sheduler """ - self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) - self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) + self.logDebug("Scheduled Account refresh for %s in %s seconds." % (self.loginname, time)) + self.core.scheduler.addJob(time, self.getAccountInfo, [force]) @lock - def checkLogin(self, user): + def checkLogin(self): """ checks if user is still logged in """ - if user in self.timestamps: - if self.timestamps[user] + self.login_timeout * 60 < time(): - self.logDebug("Reached login timeout for %s" % user) - self.relogin(user) - return False + if self.login_ts + self.login_timeout * 60 < time(): + if self.login_ts: # seperate from fresh login to have better debug logs + self.logDebug("Reached login timeout for %s" % self.loginname) + else: + self.logDebug("Login with %s" % self.loginname) + + self._login() + return False return True diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index 4f4d9f68d..c718510ed 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -17,169 +17,119 @@ @author: RaNaN """ -from os.path import exists -from shutil import copy - from threading import Lock +from random import choice +from module.common.json_layer import json from module.interaction.PullEvents import AccountUpdateEvent -from module.utils import chmod, lock - -ACC_VERSION = 1 +from module.utils import lock class AccountManager(): """manages all accounts""" - #---------------------------------------------------------------------- def __init__(self, core): """Constructor""" self.core = core self.lock = Lock() - self.initPlugins() - self.saveAccounts() # save to add categories to conf + self.loadAccounts() - def initPlugins(self): - self.accounts = {} # key = ( plugin ) - self.plugins = {} + def loadAccounts(self): + """loads all accounts available""" - self.initAccountPlugins() - self.loadAccounts() + self.accounts = {} + for plugin, loginname, activated, password, options in self.core.db.loadAccounts(): + # put into options as used in other context + options = json.loads(options) if options else {} + options["activated"] = activated - def getAccountPlugin(self, plugin): - """get account instance for plugin or None if anonymous""" - if plugin in self.accounts: - if plugin not in self.plugins: - self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin]) + self.createAccount(plugin, loginname, password, options) + + return - return self.plugins[plugin] - else: - return None - - def getAccountPlugins(self): - """ get all account instances""" - - plugins = [] - for plugin in self.accounts.keys(): - plugins.append(self.getAccountPlugin(plugin)) - - return plugins - #---------------------------------------------------------------------- - def loadAccounts(self): - """loads all accounts available""" - - if not exists("accounts.conf"): - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - - f = open("accounts.conf", "rb") - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - f.close() - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - self.core.log.warning(_("Account settings deleted, due to new config format.")) - return - - - - plugin = "" - name = "" - - for line in content[1:]: - line = line.strip() - - if not line: continue - if line.startswith("#"): continue - if line.startswith("version"): continue - - if line.endswith(":") and line.count(":") == 1: - plugin = line[:-1] - self.accounts[plugin] = {} - - elif line.startswith("@"): - try: - option = line[1:].split() - self.accounts[plugin][name]["options"][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) - except: - pass - - elif ":" in line: - name, sep, pw = line.partition(":") - self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - #---------------------------------------------------------------------- def saveAccounts(self): """save all account information""" - - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION) + "\n") - - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin+":\n") - - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data["password"]) ) - if data["options"]: - for option, values in data["options"].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) - - f.close() - chmod(f.name, 0600) - - - #---------------------------------------------------------------------- - def initAccountPlugins(self): - """init names""" - for name in self.core.pluginManager.getAccountPlugins(): - self.accounts[name] = {} - + + data = [] + for name, plugin in self.accounts.iteritems(): + data.extend([(name, acc.loginname, acc.activated, acc.password, json.dumps(acc.options)) for acc in + plugin.itervalues()]) + self.core.db.saveAccounts(data) + + def createAccount(self, plugin, loginname, password, options): + klass = self.core.pluginManager.loadClass("accounts", plugin) + if not klass: + self.core.log.warning(_("Unknown account plugin %s") % plugin) + return + + if plugin not in self.accounts: + self.accounts[plugin] = {} + + self.core.log.debug("Create account %s:%s" % (plugin, loginname)) + + self.accounts[plugin][loginname] = klass(self, loginname, password, options) + + @lock - def updateAccount(self, plugin , user, password=None, options={}): + def updateAccount(self, plugin, user, password=None, options={}): """add or update account""" - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - updated = p.updateAccounts(user, password, options) - #since accounts is a ref in plugin self.accounts doesnt need to be updated here - + if plugin in self.accounts and user in self.accounts[plugin]: + acc = self.accounts[plugin][user] + updated = acc.update(password, options) + self.saveAccounts() - if updated: p.scheduleRefresh(user, force=False) - + if updated: acc.scheduleRefresh(force=True) + else: + self.createAccount(plugin, user, password, options) + self.saveAccounts() + @lock def removeAccount(self, plugin, user): """remove account""" - + if plugin in self.accounts and user in self.accounts[plugin]: + del self.accounts[plugin][user] + self.core.db.removeAccount(plugin, user) + else: + self.core.log.debug("Remove non existing account %s %s" % (plugin, user)) + + + @lock + def getAccountForPlugin(self, plugin): if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - p.removeAccount(user) + accs = [x for x in self.accounts[plugin].values() if x.isUsable()] + if accs: return choice(accs) - self.saveAccounts() + return None @lock - def getAccountInfos(self, force=True, refresh=False): - data = {} + def getAllAccounts(self, refresh=False): + """ Return account info, refresh afterwards if needed + :param refresh: + :return: + """ if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - force = False - - for p in self.accounts.keys(): - if self.accounts[p]: - p = self.getAccountPlugin(p) - data[p.__name__] = p.getAllAccounts(force) - else: - data[p] = [] + self.core.scheduler.addJob(0, self.core.accountManager.getAllAccounts) + + # load unavailable account info + for p_dict in self.accounts.itervalues(): + for acc in p_dict.itervalues(): + acc.getAccountInfo() + e = AccountUpdateEvent() self.core.pullManager.addEvent(e) - return data - + + return self.accounts + + def refreshAllAccounts(self): + """ Force a refresh of every account """ + for p in self.accounts.itervalues(): + for acc in p.itervalues(): + acc.getAccountInfo(True) + + def sendChange(self): e = AccountUpdateEvent() self.core.pullManager.addEvent(e) diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 5a78fddc3..36df7e423 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -17,6 +17,8 @@ @author: RaNaN """ +import sys + # TODO: config format definition # more attributes if needed # get rid of catpcha & container plugins ?! (move to crypter & internals) @@ -119,3 +121,11 @@ class Base(object): def delStorage(self, key): """ Delete entry in db """ self.core.db.delStorage(self.__name__, key) + + def shell(self): + """ open ipython shell """ + if self.core.debug: + from IPython import embed + #noinspection PyUnresolvedReferences + sys.stdout = sys._stdout + embed() diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 327e717a0..59d0f46f2 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -75,26 +75,26 @@ class Plugin(Base): self.ocr = None #captcha reader instance #: account handler instance, see :py:class:`Account` - self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + self.account = self.core.accountManager.getAccountForPlugin(self.__name__) #: premium status self.premium = False #: username/login self.user = None - if self.account and not self.account.canUse(): self.account = None + if self.account and not self.account.isUsable(): self.account = None if self.account: - self.user, data = self.account.selectAccount() + self.user, data = self.account.loginname, {} #TODO change plugins to not use data anymore #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest(self.user) + self.req = self.account.getAccountRequest() self.chunkLimit = -1 # chunk limit, -1 for unlimited #: enables resume (will be ignored if server dont accept chunks) self.resumeDownload = True self.multiDL = True #every hoster with account should provide multiple downloads #: premium status - self.premium = self.account.isPremium(self.user) + self.premium = self.account.isPremium() else: - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) + self.req = self.core.requestFactory.getRequest(self.__name__) #: associated pyfile instance, see `PyFile` self.pyfile = pyfile @@ -134,7 +134,7 @@ class Plugin(Base): self.thread = thread if self.account: - self.account.checkLogin(self.user) + self.account.checkLogin() else: self.req.clearCookies() diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index 9845590cf..5c31930a5 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -259,28 +259,28 @@ class PluginManager: def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): for ptype in pluginlist: if name in self.plugins[ptype]: - return self.plugins[ptype][name], ptype + return ptype, self.plugins[ptype][name] return None, None def getPlugin(self, name, original=False): """return plugin module from hoster|decrypter|container""" - plugin, type = self.findPlugin(name) + type, plugin = self.findPlugin(name) if not plugin: self.log.warning("Plugin %s not found." % name) - plugin = self.plugins["hoster"]["BasePlugin"] + name = "BasePlugin" - if "new_module" in plugin and not original: - return plugin["new_module"] + if (type, name) in self.modules and not original: + return self.modules[(type, name)] return self.loadModule(type, name) def getPluginName(self, name): """ used to obtain new name if other plugin was injected""" - plugin, type = self.findPlugin(name) + type, plugin = self.findPlugin(name) - if "new_name" in plugin: - return plugin["new_name"] + if (type, name) in self.names: + return self.names[(type, name)] return name @@ -309,10 +309,23 @@ class PluginManager: module = self.loadModule(type, name) if module: return getattr(module, name) + def hasAccountPlugin(self, plugin): + return plugin in self.plugins["accounts"] + def getAccountPlugins(self): """return list of account plugin names""" return self.plugins["accounts"].keys() + def injectPlugin(self, type, name, module, new_name): + self.modules[(type, name)] = module + self.names[(type, name)] = new_name + + def restoreState(self, type, name): + if (type, name) in self.modules: + del self.modules[(type, name)] + if (type, name) in self.names: + del self.names[(type, name)] + def find_module(self, fullname, path=None): #redirecting imports if necesarry if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index d50df3943..872e0b770 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -5,6 +5,7 @@ import re from module.utils import remove_chars from module.plugins.Hook import Hook +from module.plugins.PluginManager import PluginTuple class MultiHoster(Hook): """ @@ -43,7 +44,7 @@ class MultiHoster(Hook): def coreReady(self): pluginMap = {} - for name in self.core.pluginManager.hosterPlugins.keys(): + for name in self.core.pluginManager.getPlugins("hoster").keys(): pluginMap[name.lower()] = name new_supported = [] @@ -66,25 +67,19 @@ class MultiHoster(Hook): # inject plugin plugin self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - dict["new_module"] = module - dict["new_name"] = self.__name__ + self.core.pluginManager.injectPlugin("hoster", hoster, module, self.__name__) self.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) # create new regexp regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict["pattern"] = regexp - dict["re"] = re.compile(regexp) + hoster = self.core.pluginManager.getPlugins("hoster") + p = hoster[self.__name__] + new = PluginTuple(p.version, re.compile(regexp), p.deps, p.user, p.path) + hoster[self.__name__] = new def unload(self): for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - if "module" in dict: - del dict["module"] - - del dict["new_module"] - del dict["new_name"]
\ No newline at end of file + self.core.pluginManager.restoreState("hoster", hoster)
\ No newline at end of file diff --git a/module/remote/socketbackend/ttypes.py b/module/remote/socketbackend/ttypes.py index ff5ae2467..6589e5923 100644 --- a/module/remote/socketbackend/ttypes.py +++ b/module/remote/socketbackend/ttypes.py @@ -27,10 +27,6 @@ class DownloadStatus: Unknown = 14 Waiting = 5 -class ElementType: - File = 1 - Package = 0 - class Input: BOOL = 4 CHOICE = 6 @@ -49,17 +45,18 @@ class Output: QUESTION = 2 class AccountInfo(BaseObject): - __slots__ = ['validuntil', 'login', 'options', 'valid', 'trafficleft', 'maxtraffic', 'premium', 'type'] + __slots__ = ['plugin', 'loginname', 'valid', 'validuntil', 'trafficleft', 'maxtraffic', 'premium', 'activated', 'options'] - def __init__(self, validuntil=None, login=None, options=None, valid=None, trafficleft=None, maxtraffic=None, premium=None, type=None): - self.validuntil = validuntil - self.login = login - self.options = options + def __init__(self, plugin=None, loginname=None, valid=None, validuntil=None, trafficleft=None, maxtraffic=None, premium=None, activated=None, options=None): + self.plugin = plugin + self.loginname = loginname self.valid = valid + self.validuntil = validuntil self.trafficleft = trafficleft self.maxtraffic = maxtraffic self.premium = premium - self.type = type + self.activated = activated + self.options = options class CaptchaTask(BaseObject): __slots__ = ['tid', 'data', 'type', 'resultType'] @@ -114,13 +111,11 @@ class DownloadInfo(BaseObject): self.plugin = plugin class EventInfo(BaseObject): - __slots__ = ['eventname', 'id', 'type', 'destination'] + __slots__ = ['eventname', 'event_args'] - def __init__(self, eventname=None, id=None, type=None, destination=None): + def __init__(self, eventname=None, event_args=None): self.eventname = eventname - self.id = id - self.type = type - self.destination = destination + self.event_args = event_args class FileData(BaseObject): __slots__ = ['fid', 'url', 'name', 'plugin', 'size', 'format_size', 'status', 'statusmsg', 'packageID', 'error', 'order'] diff --git a/module/remote/thriftbackend/pyload.thrift b/module/remote/thriftbackend/pyload.thrift index c498ef8bf..414a1ebf2 100644 --- a/module/remote/thriftbackend/pyload.thrift +++ b/module/remote/thriftbackend/pyload.thrift @@ -157,7 +157,7 @@ struct CaptchaTask { struct EventInfo { 1: string eventname, - 2: list<string> args, + 2: list<string> event_args, } struct UserData { @@ -170,14 +170,14 @@ struct UserData { struct AccountInfo { 1: PluginName plugin, - 2: string login, + 2: string loginname, 3: bool valid, 4: i64 validuntil, 5: i64 trafficleft, 6: i64 maxtraffic, 7: bool premium, 8: bool activated, - 9: map<string list<string>> options, + 9: map<string, list<string>> options, } struct ServiceCall { diff --git a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py index 3ccca992b..1925029ed 100644 --- a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py +++ b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py @@ -78,20 +78,6 @@ class Destination(TBase): "Queue": 1, } -class ElementType(TBase): - Package = 0 - File = 1 - - _VALUES_TO_NAMES = { - 0: "Package", - 1: "File", - } - - _NAMES_TO_VALUES = { - "Package": 0, - "File": 1, - } - class Input(TBase): NONE = 0 TEXT = 1 @@ -560,31 +546,23 @@ class EventInfo(TBase): """ Attributes: - eventname - - id - - type - - destination + - event_args """ __slots__ = [ 'eventname', - 'id', - 'type', - 'destination', + 'event_args', ] thrift_spec = ( None, # 0 (1, TType.STRING, 'eventname', None, None, ), # 1 - (2, TType.I32, 'id', None, None, ), # 2 - (3, TType.I32, 'type', None, None, ), # 3 - (4, TType.I32, 'destination', None, None, ), # 4 + (2, TType.LIST, 'event_args', (TType.STRING,None), None, ), # 2 ) - def __init__(self, eventname=None, id=None, type=None, destination=None,): + def __init__(self, eventname=None, event_args=None,): self.eventname = eventname - self.id = id - self.type = type - self.destination = destination + self.event_args = event_args class UserData(TBase): @@ -625,48 +603,52 @@ class UserData(TBase): class AccountInfo(TBase): """ Attributes: - - validuntil - - login - - options + - plugin + - loginname - valid + - validuntil - trafficleft - maxtraffic - premium - - type + - activated + - options """ __slots__ = [ - 'validuntil', - 'login', - 'options', + 'plugin', + 'loginname', 'valid', + 'validuntil', 'trafficleft', 'maxtraffic', 'premium', - 'type', + 'activated', + 'options', ] thrift_spec = ( None, # 0 - (1, TType.I64, 'validuntil', None, None, ), # 1 - (2, TType.STRING, 'login', None, None, ), # 2 - (3, TType.MAP, 'options', (TType.STRING,None,TType.LIST,(TType.STRING,None)), None, ), # 3 - (4, TType.BOOL, 'valid', None, None, ), # 4 + (1, TType.STRING, 'plugin', None, None, ), # 1 + (2, TType.STRING, 'loginname', None, None, ), # 2 + (3, TType.BOOL, 'valid', None, None, ), # 3 + (4, TType.I64, 'validuntil', None, None, ), # 4 (5, TType.I64, 'trafficleft', None, None, ), # 5 (6, TType.I64, 'maxtraffic', None, None, ), # 6 (7, TType.BOOL, 'premium', None, None, ), # 7 - (8, TType.STRING, 'type', None, None, ), # 8 + (8, TType.BOOL, 'activated', None, None, ), # 8 + (9, TType.MAP, 'options', (TType.STRING,None,TType.LIST,(TType.STRING,None)), None, ), # 9 ) - def __init__(self, validuntil=None, login=None, options=None, valid=None, trafficleft=None, maxtraffic=None, premium=None, type=None,): - self.validuntil = validuntil - self.login = login - self.options = options + def __init__(self, plugin=None, loginname=None, valid=None, validuntil=None, trafficleft=None, maxtraffic=None, premium=None, activated=None, options=None,): + self.plugin = plugin + self.loginname = loginname self.valid = valid + self.validuntil = validuntil self.trafficleft = trafficleft self.maxtraffic = maxtraffic self.premium = premium - self.type = type + self.activated = activated + self.options = options class ServiceCall(TBase): diff --git a/module/web/json_app.py b/module/web/json_app.py index 196c9e36d..e02aa0707 100644 --- a/module/web/json_app.py +++ b/module/web/json_app.py @@ -278,9 +278,9 @@ def update_accounts(): if action == "password": PYLOAD.updateAccount(plugin, user, value) elif action == "time" and "-" in value: - PYLOAD.updateAccount(plugin, user, options={"time": [value]}) + PYLOAD.updateAccount(plugin, user, options={"time": value}) elif action == "limitdl" and value.isdigit(): - PYLOAD.updateAccount(plugin, user, options={"limitDL": [value]}) + PYLOAD.updateAccount(plugin, user, options={"limitDL": value}) elif action == "delete": deleted.append((plugin,user)) PYLOAD.removeAccount(plugin, user) diff --git a/module/web/pyload_app.py b/module/web/pyload_app.py index 5e6d18584..a025f6bcb 100644 --- a/module/web/pyload_app.py +++ b/module/web/pyload_app.py @@ -189,7 +189,7 @@ def collector(): def downloads(): root = PYLOAD.getConfigValue("general", "download_folder") - if not isdir(root): + if not isdir(fs_encode(root)): return base([_('Download directory not found.')]) data = { 'folder': [], @@ -254,31 +254,27 @@ def config(): accs = PYLOAD.getAccounts(False) + # prefix attributes with _, because we would change them directly on the object otherweise for data in accs: if data.trafficleft == -1: - data.trafficleft = _("unlimited") + data._trafficleft = _("unlimited") elif not data.trafficleft: - data.trafficleft = _("not available") + data._trafficleft = _("not available") else: - data.trafficleft = formatSize(data.trafficleft * 1024) + data._trafficleft = formatSize(data.trafficleft * 1024) if data.validuntil == -1: - data.validuntil = _("unlimited") - elif not data.validuntil : - data.validuntil = _("not available") + data._validuntil = _("unlimited") + elif not data.validuntil: + data._validuntil = _("not available") else: t = time.localtime(data.validuntil) - data.validuntil = time.strftime("%d.%m.%Y", t) + data._validuntil = time.strftime("%d.%m.%Y", t) - if "time" in data.options: - try: - data.options["time"] = data.options["time"][0] - except: - data.options["time"] = "0:00-0:00" + if not data.options["time"]: + data.options["time"] = "0:00-0:00" - if "limitDL" in data.options: - data.options["limitdl"] = data.options["limitDL"][0] - else: + if not data.options["limitDL"]: data.options["limitdl"] = "0" return render_to_response('settings.html', diff --git a/module/web/templates/default/settings.html b/module/web/templates/default/settings.html index a4443025a..be320970b 100644 --- a/module/web/templates/default/settings.html +++ b/module/web/templates/default/settings.html @@ -102,18 +102,18 @@ {% for account in conf.accs %} - {% set plugin = account.type %} + {% set plugin = account.__name__ %} <tr> <td> <span style="padding:5px">{{ plugin }}</span> </td> - <td><label for="{{plugin}}|password;{{account.login}}" - style="color:#424242;">{{ account.login }}</label></td> + <td><label for="{{plugin}}|password;{{account.loginname}}" + style="color:#424242;">{{ account.loginname }}</label></td> <td> - <input id="{{plugin}}|password;{{account.login}}" - name="{{plugin}}|password;{{account.login}}" - type="password" value="{{account.password}}" size="12"/> + <input id="{{plugin}}|password;{{account.loginname}}" + name="{{plugin}}|password;{{account.loginname}}" + type="password" value="" size="12"/> </td> <td> {% if account.valid %} @@ -137,27 +137,27 @@ </td> <td> <span style="font-weight: bold;"> - {{ account.validuntil }} + {{ account._validuntil }} </span> </td> <td> <span style="font-weight: bold;"> - {{ account.trafficleft }} + {{ account._trafficleft }} </span> </td> <td> - <input id="{{plugin}}|time;{{account.login}}" - name="{{plugin}}|time;{{account.login}}" type="text" - size="7" value="{{account.time}}"/> + <input id="{{plugin}}|time;{{account.loginname}}" + name="{{plugin}}|time;{{account.loginname}}" type="text" + size="7" value="{{account.options.time}}"/> </td> <td> - <input id="{{plugin}}|limitdl;{{account.login}}" - name="{{plugin}}|limitdl;{{account.login}}" type="text" - size="2" value="{{account.limitdl}}"/> + <input id="{{plugin}}|limitdl;{{account.loginname}}" + name="{{plugin}}|limitdl;{{account.loginname}}" type="text" + size="2" value="{{account.options.limitdl}}"/> </td> <td> - <input id="{{plugin}}|delete;{{account.login}}" - name="{{plugin}}|delete;{{account.login}}" type="checkbox" + <input id="{{plugin}}|delete;{{account.loginname}}" + name="{{plugin}}|delete;{{account.loginname}}" type="checkbox" value="True"/> </td> </tr> diff --git a/module/web/templates/default/settings_item.html b/module/web/templates/default/settings_item.html index b81ba1b95..b3d7fe334 100644 --- a/module/web/templates/default/settings_item.html +++ b/module/web/templates/default/settings_item.html @@ -1,5 +1,5 @@ <table class="settable"> - {% if section.outline %} + {% if section.description %} <tr><th colspan="2">{{ section.description }}</th></tr> {% endif %} {% for option in section.items %} diff --git a/pyLoadCore.py b/pyLoadCore.py index 3bf742310..0651f1127 100755 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -48,7 +48,7 @@ from module.plugins.AccountManager import AccountManager from module.interaction.CaptchaManager import CaptchaManager from module.config.ConfigParser import ConfigParser from module.plugins.PluginManager import PluginManager -from module.interaction.PullEvents import PullManager +from module.interaction.EventManager import EventManager from module.network.RequestFactory import RequestFactory from module.web.ServerThread import WebServer from module.Scheduler import Scheduler @@ -58,7 +58,7 @@ from module.remote.RemoteManager import RemoteManager from module.database import DatabaseBackend, FileHandler import module.common.pylgettext as gettext -from module.utils import freeSpace, formatSize, get_console_encoding +from module.utils import freeSpace, formatSize, get_console_encoding, fs_encode from codecs import getwriter @@ -79,6 +79,7 @@ class Core(object): self.running = False self.daemon = False self.remote = True + self.pdb = None self.arg_links = [] self.pidfile = "pyload.pid" self.deleteLinks = False # will delete links on startup @@ -334,8 +335,6 @@ class Core(object): except Exception, e: print _("Failed changing user: %s") % e - self.check_file(self.config['log']['log_folder'], _("folder for logs"), True) - if self.debug: self.init_logger(logging.DEBUG) # logging level else: @@ -356,15 +355,9 @@ class Core(object): self.log.debug("Remote activated: %s" % self.remote) self.check_install("Crypto", _("pycrypto to decode container files")) - #img = self.check_install("Image", _("Python Image Libary (PIL) for captcha reading")) - #self.check_install("pycurl", _("pycurl to download any files"), True, True) - self.check_file("tmp", _("folder for temporary files"), True) - #tesser = self.check_install("tesseract", _("tesseract for captcha reading"), False) if os.name != "nt" else True self.captcha = True # checks seems to fail, althoug tesseract is available - self.check_file(self.config['general']['download_folder'], _("folder for downloads"), True) - if self.config['ssl']['activated']: self.check_install("OpenSSL", _("OpenSSL for secure connection")) @@ -393,7 +386,7 @@ class Core(object): #hell yeah, so many important managers :D self.pluginManager = PluginManager(self) - self.pullManager = PullManager(self) + self.pullManager = EventManager(self) self.accountManager = AccountManager(self) self.threadManager = ThreadManager(self) self.captchaManager = CaptchaManager(self) @@ -410,7 +403,12 @@ class Core(object): if web: self.init_webserver() - spaceLeft = freeSpace(self.config["general"]["download_folder"]) + dl_folder = fs_encode(self.config["general"]["download_folder"]) + + if not exists(dl_folder): + makedirs(dl_folder) + + spaceLeft = freeSpace(dl_folder) self.log.info(_("Free space: %s") % formatSize(spaceLeft)) @@ -433,8 +431,7 @@ class Core(object): #self.scheduler.addJob(0, self.accountManager.getAccountInfos) self.log.info(_("Activating Accounts...")) - self.accountManager.getAccountInfos() - + self.accountManager.refreshAllAccounts() self.threadManager.pause = False self.running = True @@ -458,11 +455,6 @@ class Core(object): # from meliae import scanner # scanner.dump_all_objects('objs.json') - #debugger -# from IPython import embed -# sys.stdout = sys._stdout -# embed() - locals().clear() while True: @@ -497,6 +489,9 @@ class Core(object): console.setFormatter(frm) self.log = logging.getLogger("log") # settable in config + if not exists(self.config['log']['log_folder']): + makedirs(self.config['log']['log_folder'], 0600) + if self.config['log']['file_log']: if self.config['log']['log_rotate']: file_handler = logging.handlers.RotatingFileHandler(join(self.config['log']['log_folder'], 'log.txt'), @@ -534,43 +529,6 @@ class Core(object): return False - def check_file(self, check_names, description="", folder=False, empty=True, essential=False, quiet=False): - """check wether needed files exists""" - tmp_names = [] - if not type(check_names) == list: - tmp_names.append(check_names) - else: - tmp_names.extend(check_names) - file_created = True - file_exists = True - for tmp_name in tmp_names: - if not exists(tmp_name): - file_exists = False - if empty: - try: - if folder: - tmp_name = tmp_name.replace("/", sep) - makedirs(tmp_name) - else: - open(tmp_name, "w") - except: - file_created = False - else: - file_created = False - - if not file_exists and not quiet: - if file_created: - #self.log.info( _("%s created") % description ) - pass - else: - if not empty: - self.log.warning( - _("could not find %(desc)s: %(name)s") % {"desc": description, "name": tmp_name}) - else: - print _("could not create %(desc)s: %(name)s") % {"desc": description, "name": tmp_name} - if essential: - exit() - def isClientConnected(self): return (self.lastClientConnected + 30) > time() @@ -613,6 +571,19 @@ class Core(object): self.deletePidFile() + def shell(self): + """ stop and open a ipython shell inplace""" + if self.debug: + from IPython import embed + sys.stdout = sys._stdout + embed() + + def breakpoint(self): + if self.debug: + from IPython.core.debugger import Pdb + sys.stdout = sys._stdout + if not self.pdb: self.pdb = Pdb() + self.pdb.set_trace() def path(self, *args): return join(pypath, *args) |