From 5499be89203a18ca61a21cfc7266cf0f4ebe6547 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 15 Dec 2011 23:18:21 +0100 Subject: refractoring --- module/CaptchaManager.py | 158 ------------------------------- module/ConfigParser.py | 4 + module/PullEvents.py | 120 ----------------------- module/PyFile.py | 2 +- module/PyPackage.py | 2 +- module/Utils.py | 8 ++ module/database/FileDatabase.py | 2 +- module/interaction/CaptchaManager.py | 158 +++++++++++++++++++++++++++++++ module/interaction/InteractionManager.py | 89 +++++++++++++++++ module/interaction/InteractionTask.py | 129 +++++++++++++++++++++++++ module/interaction/PullEvents.py | 120 +++++++++++++++++++++++ module/interaction/__init__.py | 2 + module/lib/namedtuple.py | 114 ++++++++++++++++++++++ module/plugins/AccountManager.py | 2 +- module/plugins/Base.py | 116 +++++++++++++++++++++++ module/plugins/Hoster.py | 22 ++--- module/plugins/Plugin.py | 96 +------------------ module/plugins/PluginManager.py | 6 ++ 18 files changed, 763 insertions(+), 387 deletions(-) delete mode 100644 module/CaptchaManager.py delete mode 100644 module/PullEvents.py create mode 100644 module/interaction/CaptchaManager.py create mode 100644 module/interaction/InteractionManager.py create mode 100644 module/interaction/InteractionTask.py create mode 100644 module/interaction/PullEvents.py create mode 100644 module/interaction/__init__.py create mode 100644 module/lib/namedtuple.py create mode 100644 module/plugins/Base.py (limited to 'module') diff --git a/module/CaptchaManager.py b/module/CaptchaManager.py deleted file mode 100644 index 02cd10a11..000000000 --- a/module/CaptchaManager.py +++ /dev/null @@ -1,158 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay, RaNaN -""" - -from time import time -from traceback import print_exc -from threading import Lock - -class CaptchaManager(): - def __init__(self, core): - self.lock = Lock() - self.core = core - self.tasks = [] #task store, for outgoing tasks only - - self.ids = 0 #only for internal purpose - - def newTask(self, img, format, file, result_type): - task = CaptchaTask(self.ids, img, format, file, result_type) - self.ids += 1 - return task - - def removeTask(self, task): - self.lock.acquire() - if task in self.tasks: - self.tasks.remove(task) - self.lock.release() - - def getTask(self): - self.lock.acquire() - for task in self.tasks: - if task.status in ("waiting", "shared-user"): - self.lock.release() - return task - self.lock.release() - return None - - def getTaskByID(self, tid): - self.lock.acquire() - for task in self.tasks: - if task.id == str(tid): #task ids are strings - self.lock.release() - return task - self.lock.release() - return None - - def handleCaptcha(self, task): - cli = self.core.isClientConnected() - - if cli: #client connected -> should solve the captcha - task.setWaiting(50) #wait 50 sec for response - - for plugin in self.core.hookManager.activePlugins(): - try: - plugin.newCaptchaTask(task) - except: - if self.core.debug: - print_exc() - - if task.handler or cli: #the captcha was handled - self.tasks.append(task) - return True - - task.error = _("No Client connected for captcha decrypting") - - return False - - -class CaptchaTask(): - def __init__(self, id, img, format, file, result_type='textual'): - self.id = str(id) - self.captchaImg = img - self.captchaFormat = format - self.captchaFile = file - self.captchaResultType = result_type - self.handler = [] #the hook plugins that will take care of the solution - self.result = None - self.waitUntil = None - self.error = None #error message - - self.status = "init" - self.data = {} #handler can store data here - - def getCaptcha(self): - return self.captchaImg, self.captchaFormat, self.captchaResultType - - def setResult(self, text): - if self.isTextual(): - self.result = text - if self.isPositional(): - try: - parts = text.split(',') - self.result = (int(parts[0]), int(parts[1])) - except: - self.result = None - - def getResult(self): - try: - res = self.result.encode("utf8", "replace") - except: - res = self.result - - return res - - def getStatus(self): - return self.status - - def setWaiting(self, sec): - """ let the captcha wait secs for the solution """ - self.waitUntil = max(time() + sec, self.waitUntil) - self.status = "waiting" - - def isWaiting(self): - if self.result or self.error or time() > self.waitUntil: - return False - - return True - - def isTextual(self): - """ returns if text is written on the captcha """ - return self.captchaResultType == 'textual' - - def isPositional(self): - """ returns if user have to click a specific region on the captcha """ - return self.captchaResultType == 'positional' - - def setWatingForUser(self, exclusive): - if exclusive: - self.status = "user" - else: - self.status = "shared-user" - - def timedOut(self): - return time() > self.waitUntil - - def invalid(self): - """ indicates the captcha was not correct """ - [x.captchaInvalid(self) for x in self.handler] - - def correct(self): - [x.captchaCorrect(self) for x in self.handler] - - def __str__(self): - return "" % self.id diff --git a/module/ConfigParser.py b/module/ConfigParser.py index 78b612f13..85c58a0a3 100644 --- a/module/ConfigParser.py +++ b/module/ConfigParser.py @@ -16,6 +16,10 @@ IGNORE = ( CONF_VERSION = 1 +from namedtuple import namedtuple + +ConfigTuple = namedtuple("ConfigTuple", "TODO") #TODO + class ConfigParser: """ holds and manage the configuration diff --git a/module/PullEvents.py b/module/PullEvents.py deleted file mode 100644 index 5ec76765e..000000000 --- a/module/PullEvents.py +++ /dev/null @@ -1,120 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from time import time -from module.utils import uniqify - -class PullManager(): - def __init__(self, core): - self.core = core - self.clients = [] - - def newClient(self, uuid): - self.clients.append(Client(uuid)) - - def clean(self): - for n, client in enumerate(self.clients): - if client.lastActive + 30 < time(): - del self.clients[n] - - def getEvents(self, uuid): - events = [] - validUuid = False - for client in self.clients: - if client.uuid == uuid: - client.lastActive = time() - validUuid = True - while client.newEvents(): - events.append(client.popEvent().toList()) - break - if not validUuid: - self.newClient(uuid) - events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] - return uniqify(events, repr) - - def addEvent(self, event): - for client in self.clients: - client.addEvent(event) - -class Client(): - def __init__(self, uuid): - self.uuid = uuid - self.lastActive = time() - self.events = [] - - def newEvents(self): - return len(self.events) > 0 - - def popEvent(self): - if not len(self.events): - return None - return self.events.pop(0) - - def addEvent(self, event): - self.events.append(event) - -class UpdateEvent(): - def __init__(self, itype, iid, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.destination = destination - - def toList(self): - return ["update", self.destination, self.type, self.id] - -class RemoveEvent(): - def __init__(self, itype, iid, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.destination = destination - - def toList(self): - return ["remove", self.destination, self.type, self.id] - -class InsertEvent(): - def __init__(self, itype, iid, after, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.after = after - self.destination = destination - - def toList(self): - return ["insert", self.destination, self.type, self.id, self.after] - -class ReloadAllEvent(): - def __init__(self, destination): - assert destination == "queue" or destination == "collector" - self.destination = destination - - def toList(self): - return ["reload", self.destination] - -class AccountUpdateEvent(): - def toList(self): - return ["account"] - -class ConfigUpdateEvent(): - def toList(self): - return ["config"] diff --git a/module/PyFile.py b/module/PyFile.py index 3dede9360..e2d906705 100644 --- a/module/PyFile.py +++ b/module/PyFile.py @@ -17,7 +17,7 @@ @author: mkaay """ -from module.PullEvents import UpdateEvent +from interaction.PullEvents import UpdateEvent from module.utils import formatSize, lock from time import sleep, time diff --git a/module/PyPackage.py b/module/PyPackage.py index f3be6c886..162a448a0 100644 --- a/module/PyPackage.py +++ b/module/PyPackage.py @@ -17,7 +17,7 @@ @author: mkaay """ -from module.PullEvents import UpdateEvent +from interaction.PullEvents import UpdateEvent from module.utils import save_path class PyPackage(): diff --git a/module/Utils.py b/module/Utils.py index c965e33c4..9ad7c2737 100644 --- a/module/Utils.py +++ b/module/Utils.py @@ -8,6 +8,7 @@ import time import re from os.path import join from string import maketrans +from itertools import islice from htmlentitydefs import name2codepoint def chmod(*args): @@ -168,6 +169,13 @@ def lock(func): return new +def chunks(iterable, size): + it = iter(iterable) + item = list(islice(it, size)) + while item: + yield item + item = list(islice(it, size)) + def fixup(m): text = m.group(0) diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 357cd766d..895e0de65 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -22,7 +22,7 @@ from threading import RLock from time import time from module.utils import formatSize, lock -from module.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent +from module.interaction.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent from module.PyPackage import PyPackage from module.PyFile import PyFile from module.database import style, DatabaseBackend diff --git a/module/interaction/CaptchaManager.py b/module/interaction/CaptchaManager.py new file mode 100644 index 000000000..02cd10a11 --- /dev/null +++ b/module/interaction/CaptchaManager.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay, RaNaN +""" + +from time import time +from traceback import print_exc +from threading import Lock + +class CaptchaManager(): + def __init__(self, core): + self.lock = Lock() + self.core = core + self.tasks = [] #task store, for outgoing tasks only + + self.ids = 0 #only for internal purpose + + def newTask(self, img, format, file, result_type): + task = CaptchaTask(self.ids, img, format, file, result_type) + self.ids += 1 + return task + + def removeTask(self, task): + self.lock.acquire() + if task in self.tasks: + self.tasks.remove(task) + self.lock.release() + + def getTask(self): + self.lock.acquire() + for task in self.tasks: + if task.status in ("waiting", "shared-user"): + self.lock.release() + return task + self.lock.release() + return None + + def getTaskByID(self, tid): + self.lock.acquire() + for task in self.tasks: + if task.id == str(tid): #task ids are strings + self.lock.release() + return task + self.lock.release() + return None + + def handleCaptcha(self, task): + cli = self.core.isClientConnected() + + if cli: #client connected -> should solve the captcha + task.setWaiting(50) #wait 50 sec for response + + for plugin in self.core.hookManager.activePlugins(): + try: + plugin.newCaptchaTask(task) + except: + if self.core.debug: + print_exc() + + if task.handler or cli: #the captcha was handled + self.tasks.append(task) + return True + + task.error = _("No Client connected for captcha decrypting") + + return False + + +class CaptchaTask(): + def __init__(self, id, img, format, file, result_type='textual'): + self.id = str(id) + self.captchaImg = img + self.captchaFormat = format + self.captchaFile = file + self.captchaResultType = result_type + self.handler = [] #the hook plugins that will take care of the solution + self.result = None + self.waitUntil = None + self.error = None #error message + + self.status = "init" + self.data = {} #handler can store data here + + def getCaptcha(self): + return self.captchaImg, self.captchaFormat, self.captchaResultType + + def setResult(self, text): + if self.isTextual(): + self.result = text + if self.isPositional(): + try: + parts = text.split(',') + self.result = (int(parts[0]), int(parts[1])) + except: + self.result = None + + def getResult(self): + try: + res = self.result.encode("utf8", "replace") + except: + res = self.result + + return res + + def getStatus(self): + return self.status + + def setWaiting(self, sec): + """ let the captcha wait secs for the solution """ + self.waitUntil = max(time() + sec, self.waitUntil) + self.status = "waiting" + + def isWaiting(self): + if self.result or self.error or time() > self.waitUntil: + return False + + return True + + def isTextual(self): + """ returns if text is written on the captcha """ + return self.captchaResultType == 'textual' + + def isPositional(self): + """ returns if user have to click a specific region on the captcha """ + return self.captchaResultType == 'positional' + + def setWatingForUser(self, exclusive): + if exclusive: + self.status = "user" + else: + self.status = "shared-user" + + def timedOut(self): + return time() > self.waitUntil + + def invalid(self): + """ indicates the captcha was not correct """ + [x.captchaInvalid(self) for x in self.handler] + + def correct(self): + [x.captchaCorrect(self) for x in self.handler] + + def __str__(self): + return "" % self.id diff --git a/module/interaction/InteractionManager.py b/module/interaction/InteractionManager.py new file mode 100644 index 000000000..8bb500f3b --- /dev/null +++ b/module/interaction/InteractionManager.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" +from time import time +from utils import lock +from traceback import print_exc +from threading import Lock + + + + +class InteractionManager: + """ + Class that gives ability to interact with the user. + Arbitary task with predefined output and input type can be set off. + Asyncronous callbacks and default values keeps the ability to fallback if no user is present. + """ + def __init__(self, core): + self.lock = Lock() + self.core = core + self.tasks = [] #task store, for outgoing tasks only + + self.ids = 0 #only for internal purpose + + def work(self): + """Mainloop that gets the work done""" + + def newTask(self, img, format, file, result_type): + task = CaptchaTask(self.ids, img, format, file, result_type) + self.ids += 1 + return task + + @lock + def removeTask(self, task): + if task in self.tasks: + self.tasks.remove(task) + + @lock + def getTask(self): + for task in self.tasks: + if task.status in ("waiting", "shared-user"): + return task + + @lock + def getTaskByID(self, tid): + for task in self.tasks: + if task.id == str(tid): #task ids are strings + self.lock.release() + return task + + def handleCaptcha(self, task): + cli = self.core.isClientConnected() + + if cli: #client connected -> should solve the captcha + task.setWaiting(50) #wait 50 sec for response + + for plugin in self.core.hookManager.activePlugins(): + try: + plugin.newCaptchaTask(task) + except: + if self.core.debug: + print_exc() + + if task.handler or cli: #the captcha was handled + self.tasks.append(task) + return True + + task.error = _("No Client connected for captcha decrypting") + + return False + + +if __name__ == "__main__": + + it = InteractionTask() \ No newline at end of file diff --git a/module/interaction/InteractionTask.py b/module/interaction/InteractionTask.py new file mode 100644 index 000000000..97cb16794 --- /dev/null +++ b/module/interaction/InteractionTask.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from module.Api import InteractionTask as BaseInteractionTask +from module.Api import Input, Output + +#noinspection PyUnresolvedReferences +class InteractionTask(BaseInteractionTask): + """ + General Interaction Task extends ITask defined by thrift with additional fields and methods. + """ + #: Plugins can put needed data here + storage = None + #: Timestamp when task expires + waitUntil = 0 + #: Default data to be used, or True if preset should be used + default = None + #: The received result as string representation + result = None + #: List of registered handles + handler = None + #: Callback functions + callbacks = None + #: Error Message + error = None + #: Status string + status = None + + def __init__(self, *args, **kwargs): + BaseInteractionTask.__init__(self, *args, **kwargs) + + # additional internal attributes + self.storage = {} + self.default = [] + self.handler = [] + self.callbacks = [] + + +class CaptchaTask: + def __init__(self, id, img, format, file, result_type='textual'): + self.id = str(id) + self.captchaImg = img + self.captchaFormat = format + self.captchaFile = file + self.captchaResultType = result_type + self.handler = [] #the hook plugins that will take care of the solution + self.result = None + self.waitUntil = None + self.error = None #error message + + self.status = "init" + self.data = {} #handler can store data here + + def getCaptcha(self): + return self.captchaImg, self.captchaFormat, self.captchaResultType + + def setResult(self, text): + if self.isTextual(): + self.result = text + if self.isPositional(): + try: + parts = text.split(',') + self.result = (int(parts[0]), int(parts[1])) + except: + self.result = None + + def getResult(self): + try: + res = self.result.encode("utf8", "replace") + except: + res = self.result + + return res + + def getStatus(self): + return self.status + + def setWaiting(self, sec): + """ let the captcha wait secs for the solution """ + self.waitUntil = max(time() + sec, self.waitUntil) + self.status = "waiting" + + def isWaiting(self): + if self.result or self.error or time() > self.waitUntil: + return False + + return True + + def isTextual(self): + """ returns if text is written on the captcha """ + return self.captchaResultType == 'textual' + + def isPositional(self): + """ returns if user have to click a specific region on the captcha """ + return self.captchaResultType == 'positional' + + def setWatingForUser(self, exclusive): + if exclusive: + self.status = "user" + else: + self.status = "shared-user" + + def timedOut(self): + return time() > self.waitUntil + + def invalid(self): + """ indicates the captcha was not correct """ + [x.captchaInvalid(self) for x in self.handler] + + def correct(self): + [x.captchaCorrect(self) for x in self.handler] + + def __str__(self): + return "" % self.id diff --git a/module/interaction/PullEvents.py b/module/interaction/PullEvents.py new file mode 100644 index 000000000..5ec76765e --- /dev/null +++ b/module/interaction/PullEvents.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from time import time +from module.utils import uniqify + +class PullManager(): + def __init__(self, core): + self.core = core + self.clients = [] + + def newClient(self, uuid): + self.clients.append(Client(uuid)) + + def clean(self): + for n, client in enumerate(self.clients): + if client.lastActive + 30 < time(): + del self.clients[n] + + def getEvents(self, uuid): + events = [] + validUuid = False + for client in self.clients: + if client.uuid == uuid: + client.lastActive = time() + validUuid = True + while client.newEvents(): + events.append(client.popEvent().toList()) + break + if not validUuid: + self.newClient(uuid) + events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] + return uniqify(events, repr) + + def addEvent(self, event): + for client in self.clients: + client.addEvent(event) + +class Client(): + def __init__(self, uuid): + self.uuid = uuid + self.lastActive = time() + self.events = [] + + def newEvents(self): + return len(self.events) > 0 + + def popEvent(self): + if not len(self.events): + return None + return self.events.pop(0) + + def addEvent(self, event): + self.events.append(event) + +class UpdateEvent(): + def __init__(self, itype, iid, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.destination = destination + + def toList(self): + return ["update", self.destination, self.type, self.id] + +class RemoveEvent(): + def __init__(self, itype, iid, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.destination = destination + + def toList(self): + return ["remove", self.destination, self.type, self.id] + +class InsertEvent(): + def __init__(self, itype, iid, after, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.after = after + self.destination = destination + + def toList(self): + return ["insert", self.destination, self.type, self.id, self.after] + +class ReloadAllEvent(): + def __init__(self, destination): + assert destination == "queue" or destination == "collector" + self.destination = destination + + def toList(self): + return ["reload", self.destination] + +class AccountUpdateEvent(): + def toList(self): + return ["account"] + +class ConfigUpdateEvent(): + def toList(self): + return ["config"] diff --git a/module/interaction/__init__.py b/module/interaction/__init__.py new file mode 100644 index 000000000..de6d13128 --- /dev/null +++ b/module/interaction/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'christian' + \ No newline at end of file diff --git a/module/lib/namedtuple.py b/module/lib/namedtuple.py new file mode 100644 index 000000000..6ff07e839 --- /dev/null +++ b/module/lib/namedtuple.py @@ -0,0 +1,114 @@ +## {{{ http://code.activestate.com/recipes/500261/ (r15) +from operator import itemgetter as _itemgetter +from keyword import iskeyword as _iskeyword +import sys as _sys + +def namedtuple(typename, field_names, verbose=False, rename=False): + """Returns a new subclass of tuple with named fields. + + >>> Point = namedtuple('Point', 'x y') + >>> Point.__doc__ # docstring for the new class + 'Point(x, y)' + >>> p = Point(11, y=22) # instantiate with positional args or keywords + >>> p[0] + p[1] # indexable like a plain tuple + 33 + >>> x, y = p # unpack like a regular tuple + >>> x, y + (11, 22) + >>> p.x + p.y # fields also accessable by name + 33 + >>> d = p._asdict() # convert to a dictionary + >>> d['x'] + 11 + >>> Point(**d) # convert from a dictionary + Point(x=11, y=22) + >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields + Point(x=100, y=22) + + """ + + # Parse and validate the field names. Validation serves two purposes, + # generating informative error messages and preventing template injection attacks. + if isinstance(field_names, basestring): + field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas + field_names = tuple(map(str, field_names)) + if rename: + names = list(field_names) + seen = set() + for i, name in enumerate(names): + if (not min(c.isalnum() or c=='_' for c in name) or _iskeyword(name) + or not name or name[0].isdigit() or name.startswith('_') + or name in seen): + names[i] = '_%d' % i + seen.add(name) + field_names = tuple(names) + for name in (typename,) + field_names: + if not min(c.isalnum() or c=='_' for c in name): + raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) + if _iskeyword(name): + raise ValueError('Type names and field names cannot be a keyword: %r' % name) + if name[0].isdigit(): + raise ValueError('Type names and field names cannot start with a number: %r' % name) + seen_names = set() + for name in field_names: + if name.startswith('_') and not rename: + raise ValueError('Field names cannot start with an underscore: %r' % name) + if name in seen_names: + raise ValueError('Encountered duplicate field name: %r' % name) + seen_names.add(name) + + # Create and fill-in the class template + numfields = len(field_names) + argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes + reprtxt = ', '.join('%s=%%r' % name for name in field_names) + template = '''class %(typename)s(tuple): + '%(typename)s(%(argtxt)s)' \n + __slots__ = () \n + _fields = %(field_names)r \n + def __new__(_cls, %(argtxt)s): + return _tuple.__new__(_cls, (%(argtxt)s)) \n + @classmethod + def _make(cls, iterable, new=tuple.__new__, len=len): + 'Make a new %(typename)s object from a sequence or iterable' + result = new(cls, iterable) + if len(result) != %(numfields)d: + raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result)) + return result \n + def __repr__(self): + return '%(typename)s(%(reprtxt)s)' %% self \n + def _asdict(self): + 'Return a new dict which maps field names to their values' + return dict(zip(self._fields, self)) \n + def _replace(_self, **kwds): + 'Return a new %(typename)s object replacing specified fields with new values' + result = _self._make(map(kwds.pop, %(field_names)r, _self)) + if kwds: + raise ValueError('Got unexpected field names: %%r' %% kwds.keys()) + return result \n + def __getnewargs__(self): + return tuple(self) \n\n''' % locals() + for i, name in enumerate(field_names): + template += ' %s = _property(_itemgetter(%d))\n' % (name, i) + if verbose: + print template + + # Execute the template string in a temporary namespace + namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, + _property=property, _tuple=tuple) + try: + exec template in namespace + except SyntaxError, e: + raise SyntaxError(e.message + ':\n' + template) + result = namespace[typename] + + # For pickling to work, the __module__ variable needs to be set to the frame + # where the named tuple is created. Bypass this step in enviroments where + # sys._getframe is not defined (Jython for example) or sys._getframe is not + # defined for arguments greater than 0 (IronPython). + try: + result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + + return result +## end of http://code.activestate.com/recipes/500261/ }}} diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index fc521d36c..4f4d9f68d 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -22,7 +22,7 @@ from shutil import copy from threading import Lock -from module.PullEvents import AccountUpdateEvent +from module.interaction.PullEvents import AccountUpdateEvent from module.utils import chmod, lock ACC_VERSION = 1 diff --git a/module/plugins/Base.py b/module/plugins/Base.py new file mode 100644 index 000000000..98573ea63 --- /dev/null +++ b/module/plugins/Base.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +class Base(object): + """ + The Base plugin class with all shared methods and every possible attribute for plugin definition. + """ + __version__ = "0.4" + #: Regexp pattern which will be matched for download plugins + __pattern__ = r"" + #: Flat config definition + __config__ = tuple() + #: Short description, one liner + __description__ = "" + #: More detailed text + __long_description__ = """""" + #: List of needed modules + __dependencies__ = tuple() + #: Tags to categorize the plugin + __tags__ = tuple() + #: Base64 encoded .png icon + __icon__ = "" + #: Alternative, link to png icon + __icon_url__ = "" + #: Url with general information/support/discussion + __url__ = "" + __author_name__ = tuple() + __author_mail__ = tuple() + + + def __init__(self, core): + self.__name__ = self.__class__.__name__ + + #: Core instance + self.core = core + #: logging instance + self.log = core.log + #: core config + self.config = core.config + + #log functions + def logInfo(self, *args): + self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logWarning(self, *args): + self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logError(self, *args): + self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logDebug(self, *args): + self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + + def setConf(self, option, value): + """ see `setConfig` """ + self.core.config.setPlugin(self.__name__, option, value) + + def setConfig(self, option, value): + """ Set config value for current plugin + + :param option: + :param value: + :return: + """ + self.setConf(option, value) + + def getConf(self, option): + """ see `getConfig` """ + return self.core.config.getPlugin(self.__name__, option) + + def getConfig(self, option): + """ Returns config value for current plugin + + :param option: + :return: + """ + return self.getConf(option) + + def setStorage(self, key, value): + """ Saves a value persistently to the database """ + self.core.db.setStorage(self.__name__, key, value) + + def store(self, key, value): + """ same as `setStorage` """ + self.core.db.setStorage(self.__name__, key, value) + + def getStorage(self, key=None, default=None): + """ Retrieves saved value or dict of all saved entries if key is None """ + if key is not None: + return self.core.db.getStorage(self.__name__, key) or default + return self.core.db.getStorage(self.__name__, key) + + def retrieve(self, *args, **kwargs): + """ same as `getStorage` """ + return self.getStorage(*args, **kwargs) + + def delStorage(self, key): + """ Delete entry in db """ + self.core.db.delStorage(self.__name__, key) diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 814a70949..aa50099fb 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -19,15 +19,15 @@ from module.plugins.Plugin import Plugin -def getInfo(self): - #result = [ .. (name, size, status, url) .. ] - return - class Hoster(Plugin): - __name__ = "Hoster" - __version__ = "0.1" - __pattern__ = None - __type__ = "hoster" - __description__ = """Base hoster plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + + @staticmethod + def getInfo(urls): + """This method is used to retrieve the online status of files for hoster plugins. + It has to *yield* list of tuples with the result in this format (name, size, status, url), + where status is one of API pyfile statusses. + + :param urls: List of urls + :return: + """ + pass \ No newline at end of file diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index f7587d3f2..b3c22f983 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -29,17 +29,9 @@ if os.name != "nt": from pwd import getpwnam from grp import getgrnam -from itertools import islice - -from module.utils import save_join, save_path, fs_encode - -def chunks(iterable, size): - it = iter(iterable) - item = list(islice(it, size)) - while item: - yield item - item = list(islice(it, size)) +from Base import Base +from module.utils import save_join, save_path, fs_encode, chunks class Abort(Exception): """ raised when aborted """ @@ -61,95 +53,11 @@ class SkipDownload(Exception): """ raised when download should be skipped """ -class Base(object): - """ - A Base class with log/config/db methods *all* plugin types can use - """ - - def __init__(self, core): - #: Core instance - self.core = core - #: logging instance - self.log = core.log - #: core config - self.config = core.config - - #log functions - def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - - def setConf(self, option, value): - """ see `setConfig` """ - self.core.config.setPlugin(self.__name__, option, value) - - def setConfig(self, option, value): - """ Set config value for current plugin - - :param option: - :param value: - :return: - """ - self.setConf(option, value) - - def getConf(self, option): - """ see `getConfig` """ - return self.core.config.getPlugin(self.__name__, option) - - def getConfig(self, option): - """ Returns config value for current plugin - - :param option: - :return: - """ - return self.getConf(option) - - def setStorage(self, key, value): - """ Saves a value persistently to the database """ - self.core.db.setStorage(self.__name__, key, value) - - def store(self, key, value): - """ same as `setStorage` """ - self.core.db.setStorage(self.__name__, key, value) - - def getStorage(self, key=None, default=None): - """ Retrieves saved value or dict of all saved entries if key is None """ - if key is not None: - return self.core.db.getStorage(self.__name__, key) or default - return self.core.db.getStorage(self.__name__, key) - - def retrieve(self, *args, **kwargs): - """ same as `getStorage` """ - return self.getStorage(*args, **kwargs) - - def delStorage(self, key): - """ Delete entry in db """ - self.core.db.delStorage(self.__name__, key) - - class Plugin(Base): """ Base plugin for hoster/crypter. Overwrite `process` / `decrypt` in your subclassed plugin. """ - __name__ = "Plugin" - __version__ = "0.4" - __pattern__ = None - __type__ = "hoster" - __config__ = [("name", "type", "desc", "default")] - __description__ = """Base Plugin""" - __author_name__ = ("RaNaN", "spoob", "mkaay") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") - def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index f3f5f47bc..4bf41484a 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -29,6 +29,10 @@ from traceback import print_exc from module.lib.SafeEval import const_eval as literal_eval from module.ConfigParser import IGNORE +from namedtuple import namedtuple + +PluginTuple = namedtuple("PluginTuple", "version pattern desc long_desc deps user name module") + class PluginManager: ROOT = "module.plugins." USERROOT = "userplugins." @@ -40,6 +44,8 @@ class PluginManager: DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') + ATTR = re.compile(r'__([a-z0-9_])__\s*=\s*({|\[|\(|"|\')([^__]+)', re.M | re.I) + def __init__(self, core): self.core = core -- cgit v1.2.3