diff options
author | RaNaN <Mast3rRaNaN@hotmail.de> | 2011-12-15 23:18:21 +0100 |
---|---|---|
committer | RaNaN <Mast3rRaNaN@hotmail.de> | 2011-12-15 23:18:21 +0100 |
commit | 5499be89203a18ca61a21cfc7266cf0f4ebe6547 (patch) | |
tree | 55f3d207573920be3a6970dbc099d058f417c05f /module | |
parent | Added tag v0.4.9 for changeset 1babca12c25f (diff) | |
download | pyload-5499be89203a18ca61a21cfc7266cf0f4ebe6547.tar.xz |
refractoring
Diffstat (limited to 'module')
-rw-r--r-- | module/ConfigParser.py | 4 | ||||
-rw-r--r-- | module/PyFile.py | 2 | ||||
-rw-r--r-- | module/PyPackage.py | 2 | ||||
-rw-r--r-- | module/Utils.py | 8 | ||||
-rw-r--r-- | module/database/FileDatabase.py | 2 | ||||
-rw-r--r-- | module/interaction/CaptchaManager.py (renamed from module/CaptchaManager.py) | 0 | ||||
-rw-r--r-- | module/interaction/InteractionManager.py | 89 | ||||
-rw-r--r-- | module/interaction/InteractionTask.py | 129 | ||||
-rw-r--r-- | module/interaction/PullEvents.py (renamed from module/PullEvents.py) | 0 | ||||
-rw-r--r-- | module/interaction/__init__.py | 2 | ||||
-rw-r--r-- | module/lib/namedtuple.py | 114 | ||||
-rw-r--r-- | module/plugins/AccountManager.py | 2 | ||||
-rw-r--r-- | module/plugins/Base.py | 116 | ||||
-rw-r--r-- | module/plugins/Hoster.py | 22 | ||||
-rw-r--r-- | module/plugins/Plugin.py | 96 | ||||
-rw-r--r-- | module/plugins/PluginManager.py | 6 |
16 files changed, 485 insertions, 109 deletions
diff --git a/module/ConfigParser.py b/module/ConfigParser.py index 78b612f13..85c58a0a3 100644 --- a/module/ConfigParser.py +++ b/module/ConfigParser.py @@ -16,6 +16,10 @@ IGNORE = ( CONF_VERSION = 1 +from namedtuple import namedtuple + +ConfigTuple = namedtuple("ConfigTuple", "TODO") #TODO + class ConfigParser: """ holds and manage the configuration diff --git a/module/PyFile.py b/module/PyFile.py index 3dede9360..e2d906705 100644 --- a/module/PyFile.py +++ b/module/PyFile.py @@ -17,7 +17,7 @@ @author: mkaay """ -from module.PullEvents import UpdateEvent +from interaction.PullEvents import UpdateEvent from module.utils import formatSize, lock from time import sleep, time diff --git a/module/PyPackage.py b/module/PyPackage.py index f3be6c886..162a448a0 100644 --- a/module/PyPackage.py +++ b/module/PyPackage.py @@ -17,7 +17,7 @@ @author: mkaay """ -from module.PullEvents import UpdateEvent +from interaction.PullEvents import UpdateEvent from module.utils import save_path class PyPackage(): diff --git a/module/Utils.py b/module/Utils.py index c965e33c4..9ad7c2737 100644 --- a/module/Utils.py +++ b/module/Utils.py @@ -8,6 +8,7 @@ import time import re from os.path import join from string import maketrans +from itertools import islice from htmlentitydefs import name2codepoint def chmod(*args): @@ -168,6 +169,13 @@ def lock(func): return new +def chunks(iterable, size): + it = iter(iterable) + item = list(islice(it, size)) + while item: + yield item + item = list(islice(it, size)) + def fixup(m): text = m.group(0) diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 357cd766d..895e0de65 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -22,7 +22,7 @@ from threading import RLock from time import time from module.utils import formatSize, lock -from module.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent +from module.interaction.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent from module.PyPackage import PyPackage from module.PyFile import PyFile from module.database import style, DatabaseBackend diff --git a/module/CaptchaManager.py b/module/interaction/CaptchaManager.py index 02cd10a11..02cd10a11 100644 --- a/module/CaptchaManager.py +++ b/module/interaction/CaptchaManager.py diff --git a/module/interaction/InteractionManager.py b/module/interaction/InteractionManager.py new file mode 100644 index 000000000..8bb500f3b --- /dev/null +++ b/module/interaction/InteractionManager.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" +from time import time +from utils import lock +from traceback import print_exc +from threading import Lock + + + + +class InteractionManager: + """ + Class that gives ability to interact with the user. + Arbitary task with predefined output and input type can be set off. + Asyncronous callbacks and default values keeps the ability to fallback if no user is present. + """ + def __init__(self, core): + self.lock = Lock() + self.core = core + self.tasks = [] #task store, for outgoing tasks only + + self.ids = 0 #only for internal purpose + + def work(self): + """Mainloop that gets the work done""" + + def newTask(self, img, format, file, result_type): + task = CaptchaTask(self.ids, img, format, file, result_type) + self.ids += 1 + return task + + @lock + def removeTask(self, task): + if task in self.tasks: + self.tasks.remove(task) + + @lock + def getTask(self): + for task in self.tasks: + if task.status in ("waiting", "shared-user"): + return task + + @lock + def getTaskByID(self, tid): + for task in self.tasks: + if task.id == str(tid): #task ids are strings + self.lock.release() + return task + + def handleCaptcha(self, task): + cli = self.core.isClientConnected() + + if cli: #client connected -> should solve the captcha + task.setWaiting(50) #wait 50 sec for response + + for plugin in self.core.hookManager.activePlugins(): + try: + plugin.newCaptchaTask(task) + except: + if self.core.debug: + print_exc() + + if task.handler or cli: #the captcha was handled + self.tasks.append(task) + return True + + task.error = _("No Client connected for captcha decrypting") + + return False + + +if __name__ == "__main__": + + it = InteractionTask()
\ No newline at end of file diff --git a/module/interaction/InteractionTask.py b/module/interaction/InteractionTask.py new file mode 100644 index 000000000..97cb16794 --- /dev/null +++ b/module/interaction/InteractionTask.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" + +from module.Api import InteractionTask as BaseInteractionTask +from module.Api import Input, Output + +#noinspection PyUnresolvedReferences +class InteractionTask(BaseInteractionTask): + """ + General Interaction Task extends ITask defined by thrift with additional fields and methods. + """ + #: Plugins can put needed data here + storage = None + #: Timestamp when task expires + waitUntil = 0 + #: Default data to be used, or True if preset should be used + default = None + #: The received result as string representation + result = None + #: List of registered handles + handler = None + #: Callback functions + callbacks = None + #: Error Message + error = None + #: Status string + status = None + + def __init__(self, *args, **kwargs): + BaseInteractionTask.__init__(self, *args, **kwargs) + + # additional internal attributes + self.storage = {} + self.default = [] + self.handler = [] + self.callbacks = [] + + +class CaptchaTask: + def __init__(self, id, img, format, file, result_type='textual'): + self.id = str(id) + self.captchaImg = img + self.captchaFormat = format + self.captchaFile = file + self.captchaResultType = result_type + self.handler = [] #the hook plugins that will take care of the solution + self.result = None + self.waitUntil = None + self.error = None #error message + + self.status = "init" + self.data = {} #handler can store data here + + def getCaptcha(self): + return self.captchaImg, self.captchaFormat, self.captchaResultType + + def setResult(self, text): + if self.isTextual(): + self.result = text + if self.isPositional(): + try: + parts = text.split(',') + self.result = (int(parts[0]), int(parts[1])) + except: + self.result = None + + def getResult(self): + try: + res = self.result.encode("utf8", "replace") + except: + res = self.result + + return res + + def getStatus(self): + return self.status + + def setWaiting(self, sec): + """ let the captcha wait secs for the solution """ + self.waitUntil = max(time() + sec, self.waitUntil) + self.status = "waiting" + + def isWaiting(self): + if self.result or self.error or time() > self.waitUntil: + return False + + return True + + def isTextual(self): + """ returns if text is written on the captcha """ + return self.captchaResultType == 'textual' + + def isPositional(self): + """ returns if user have to click a specific region on the captcha """ + return self.captchaResultType == 'positional' + + def setWatingForUser(self, exclusive): + if exclusive: + self.status = "user" + else: + self.status = "shared-user" + + def timedOut(self): + return time() > self.waitUntil + + def invalid(self): + """ indicates the captcha was not correct """ + [x.captchaInvalid(self) for x in self.handler] + + def correct(self): + [x.captchaCorrect(self) for x in self.handler] + + def __str__(self): + return "<CaptchaTask '%s'>" % self.id diff --git a/module/PullEvents.py b/module/interaction/PullEvents.py index 5ec76765e..5ec76765e 100644 --- a/module/PullEvents.py +++ b/module/interaction/PullEvents.py diff --git a/module/interaction/__init__.py b/module/interaction/__init__.py new file mode 100644 index 000000000..de6d13128 --- /dev/null +++ b/module/interaction/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'christian' +
\ No newline at end of file diff --git a/module/lib/namedtuple.py b/module/lib/namedtuple.py new file mode 100644 index 000000000..6ff07e839 --- /dev/null +++ b/module/lib/namedtuple.py @@ -0,0 +1,114 @@ +## {{{ http://code.activestate.com/recipes/500261/ (r15) +from operator import itemgetter as _itemgetter +from keyword import iskeyword as _iskeyword +import sys as _sys + +def namedtuple(typename, field_names, verbose=False, rename=False): + """Returns a new subclass of tuple with named fields. + + >>> Point = namedtuple('Point', 'x y') + >>> Point.__doc__ # docstring for the new class + 'Point(x, y)' + >>> p = Point(11, y=22) # instantiate with positional args or keywords + >>> p[0] + p[1] # indexable like a plain tuple + 33 + >>> x, y = p # unpack like a regular tuple + >>> x, y + (11, 22) + >>> p.x + p.y # fields also accessable by name + 33 + >>> d = p._asdict() # convert to a dictionary + >>> d['x'] + 11 + >>> Point(**d) # convert from a dictionary + Point(x=11, y=22) + >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields + Point(x=100, y=22) + + """ + + # Parse and validate the field names. Validation serves two purposes, + # generating informative error messages and preventing template injection attacks. + if isinstance(field_names, basestring): + field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas + field_names = tuple(map(str, field_names)) + if rename: + names = list(field_names) + seen = set() + for i, name in enumerate(names): + if (not min(c.isalnum() or c=='_' for c in name) or _iskeyword(name) + or not name or name[0].isdigit() or name.startswith('_') + or name in seen): + names[i] = '_%d' % i + seen.add(name) + field_names = tuple(names) + for name in (typename,) + field_names: + if not min(c.isalnum() or c=='_' for c in name): + raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) + if _iskeyword(name): + raise ValueError('Type names and field names cannot be a keyword: %r' % name) + if name[0].isdigit(): + raise ValueError('Type names and field names cannot start with a number: %r' % name) + seen_names = set() + for name in field_names: + if name.startswith('_') and not rename: + raise ValueError('Field names cannot start with an underscore: %r' % name) + if name in seen_names: + raise ValueError('Encountered duplicate field name: %r' % name) + seen_names.add(name) + + # Create and fill-in the class template + numfields = len(field_names) + argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes + reprtxt = ', '.join('%s=%%r' % name for name in field_names) + template = '''class %(typename)s(tuple): + '%(typename)s(%(argtxt)s)' \n + __slots__ = () \n + _fields = %(field_names)r \n + def __new__(_cls, %(argtxt)s): + return _tuple.__new__(_cls, (%(argtxt)s)) \n + @classmethod + def _make(cls, iterable, new=tuple.__new__, len=len): + 'Make a new %(typename)s object from a sequence or iterable' + result = new(cls, iterable) + if len(result) != %(numfields)d: + raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result)) + return result \n + def __repr__(self): + return '%(typename)s(%(reprtxt)s)' %% self \n + def _asdict(self): + 'Return a new dict which maps field names to their values' + return dict(zip(self._fields, self)) \n + def _replace(_self, **kwds): + 'Return a new %(typename)s object replacing specified fields with new values' + result = _self._make(map(kwds.pop, %(field_names)r, _self)) + if kwds: + raise ValueError('Got unexpected field names: %%r' %% kwds.keys()) + return result \n + def __getnewargs__(self): + return tuple(self) \n\n''' % locals() + for i, name in enumerate(field_names): + template += ' %s = _property(_itemgetter(%d))\n' % (name, i) + if verbose: + print template + + # Execute the template string in a temporary namespace + namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, + _property=property, _tuple=tuple) + try: + exec template in namespace + except SyntaxError, e: + raise SyntaxError(e.message + ':\n' + template) + result = namespace[typename] + + # For pickling to work, the __module__ variable needs to be set to the frame + # where the named tuple is created. Bypass this step in enviroments where + # sys._getframe is not defined (Jython for example) or sys._getframe is not + # defined for arguments greater than 0 (IronPython). + try: + result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + + return result +## end of http://code.activestate.com/recipes/500261/ }}} diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index fc521d36c..4f4d9f68d 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -22,7 +22,7 @@ from shutil import copy from threading import Lock -from module.PullEvents import AccountUpdateEvent +from module.interaction.PullEvents import AccountUpdateEvent from module.utils import chmod, lock ACC_VERSION = 1 diff --git a/module/plugins/Base.py b/module/plugins/Base.py new file mode 100644 index 000000000..98573ea63 --- /dev/null +++ b/module/plugins/Base.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" + +class Base(object): + """ + The Base plugin class with all shared methods and every possible attribute for plugin definition. + """ + __version__ = "0.4" + #: Regexp pattern which will be matched for download plugins + __pattern__ = r"" + #: Flat config definition + __config__ = tuple() + #: Short description, one liner + __description__ = "" + #: More detailed text + __long_description__ = """""" + #: List of needed modules + __dependencies__ = tuple() + #: Tags to categorize the plugin + __tags__ = tuple() + #: Base64 encoded .png icon + __icon__ = "" + #: Alternative, link to png icon + __icon_url__ = "" + #: Url with general information/support/discussion + __url__ = "" + __author_name__ = tuple() + __author_mail__ = tuple() + + + def __init__(self, core): + self.__name__ = self.__class__.__name__ + + #: Core instance + self.core = core + #: logging instance + self.log = core.log + #: core config + self.config = core.config + + #log functions + def logInfo(self, *args): + self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logWarning(self, *args): + self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logError(self, *args): + self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logDebug(self, *args): + self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + + def setConf(self, option, value): + """ see `setConfig` """ + self.core.config.setPlugin(self.__name__, option, value) + + def setConfig(self, option, value): + """ Set config value for current plugin + + :param option: + :param value: + :return: + """ + self.setConf(option, value) + + def getConf(self, option): + """ see `getConfig` """ + return self.core.config.getPlugin(self.__name__, option) + + def getConfig(self, option): + """ Returns config value for current plugin + + :param option: + :return: + """ + return self.getConf(option) + + def setStorage(self, key, value): + """ Saves a value persistently to the database """ + self.core.db.setStorage(self.__name__, key, value) + + def store(self, key, value): + """ same as `setStorage` """ + self.core.db.setStorage(self.__name__, key, value) + + def getStorage(self, key=None, default=None): + """ Retrieves saved value or dict of all saved entries if key is None """ + if key is not None: + return self.core.db.getStorage(self.__name__, key) or default + return self.core.db.getStorage(self.__name__, key) + + def retrieve(self, *args, **kwargs): + """ same as `getStorage` """ + return self.getStorage(*args, **kwargs) + + def delStorage(self, key): + """ Delete entry in db """ + self.core.db.delStorage(self.__name__, key) diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 814a70949..aa50099fb 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -19,15 +19,15 @@ from module.plugins.Plugin import Plugin -def getInfo(self): - #result = [ .. (name, size, status, url) .. ] - return - class Hoster(Plugin): - __name__ = "Hoster" - __version__ = "0.1" - __pattern__ = None - __type__ = "hoster" - __description__ = """Base hoster plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + + @staticmethod + def getInfo(urls): + """This method is used to retrieve the online status of files for hoster plugins. + It has to *yield* list of tuples with the result in this format (name, size, status, url), + where status is one of API pyfile statusses. + + :param urls: List of urls + :return: + """ + pass
\ No newline at end of file diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index f7587d3f2..b3c22f983 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -29,17 +29,9 @@ if os.name != "nt": from pwd import getpwnam from grp import getgrnam -from itertools import islice - -from module.utils import save_join, save_path, fs_encode - -def chunks(iterable, size): - it = iter(iterable) - item = list(islice(it, size)) - while item: - yield item - item = list(islice(it, size)) +from Base import Base +from module.utils import save_join, save_path, fs_encode, chunks class Abort(Exception): """ raised when aborted """ @@ -61,95 +53,11 @@ class SkipDownload(Exception): """ raised when download should be skipped """ -class Base(object): - """ - A Base class with log/config/db methods *all* plugin types can use - """ - - def __init__(self, core): - #: Core instance - self.core = core - #: logging instance - self.log = core.log - #: core config - self.config = core.config - - #log functions - def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - - def setConf(self, option, value): - """ see `setConfig` """ - self.core.config.setPlugin(self.__name__, option, value) - - def setConfig(self, option, value): - """ Set config value for current plugin - - :param option: - :param value: - :return: - """ - self.setConf(option, value) - - def getConf(self, option): - """ see `getConfig` """ - return self.core.config.getPlugin(self.__name__, option) - - def getConfig(self, option): - """ Returns config value for current plugin - - :param option: - :return: - """ - return self.getConf(option) - - def setStorage(self, key, value): - """ Saves a value persistently to the database """ - self.core.db.setStorage(self.__name__, key, value) - - def store(self, key, value): - """ same as `setStorage` """ - self.core.db.setStorage(self.__name__, key, value) - - def getStorage(self, key=None, default=None): - """ Retrieves saved value or dict of all saved entries if key is None """ - if key is not None: - return self.core.db.getStorage(self.__name__, key) or default - return self.core.db.getStorage(self.__name__, key) - - def retrieve(self, *args, **kwargs): - """ same as `getStorage` """ - return self.getStorage(*args, **kwargs) - - def delStorage(self, key): - """ Delete entry in db """ - self.core.db.delStorage(self.__name__, key) - - class Plugin(Base): """ Base plugin for hoster/crypter. Overwrite `process` / `decrypt` in your subclassed plugin. """ - __name__ = "Plugin" - __version__ = "0.4" - __pattern__ = None - __type__ = "hoster" - __config__ = [("name", "type", "desc", "default")] - __description__ = """Base Plugin""" - __author_name__ = ("RaNaN", "spoob", "mkaay") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") - def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index f3f5f47bc..4bf41484a 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -29,6 +29,10 @@ from traceback import print_exc from module.lib.SafeEval import const_eval as literal_eval from module.ConfigParser import IGNORE +from namedtuple import namedtuple + +PluginTuple = namedtuple("PluginTuple", "version pattern desc long_desc deps user name module") + class PluginManager: ROOT = "module.plugins." USERROOT = "userplugins." @@ -40,6 +44,8 @@ class PluginManager: DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') + ATTR = re.compile(r'__([a-z0-9_])__\s*=\s*({|\[|\(|"|\')([^__]+)', re.M | re.I) + def __init__(self, core): self.core = core |