module -> pyload

1 files changed, 589 insertions, 0 deletions

diff --git a/pyload/lib/beaker/cache.py b/pyload/lib/beaker/cache.py
new file mode 100644
index 000000000..0ae96e020
--- /dev/null
+++ b/pyload/lib/beaker/cache.py
@@ -0,0 +1,589 @@
+"""This package contains the "front end" classes and functions
+for Beaker caching.
+
+Included are the :class:`.Cache` and :class:`.CacheManager` classes,
+as well as the function decorators :func:`.region_decorate`,
+:func:`.region_invalidate`.
+
+"""
+import warnings
+
+import beaker.container as container
+import beaker.util as util
+from beaker.crypto.util import sha1
+from beaker.exceptions import BeakerException, InvalidCacheBackendError
+from beaker.synchronization import _threading
+
+import beaker.ext.memcached as memcached
+import beaker.ext.database as database
+import beaker.ext.sqla as sqla
+import beaker.ext.google as google
+
+# Initialize the cache region dict
+cache_regions = {}
+"""Dictionary of 'region' arguments.
+
+A "region" is a string name that refers to a series of cache
+configuration arguments.    An application may have multiple
+"regions" - one which stores things in a memory cache, one
+which writes data to files, etc.
+
+The dictionary stores string key names mapped to dictionaries
+of configuration arguments.  Example::
+
+    from beaker.cache import cache_regions
+    cache_regions.update({
+        'short_term':{
+            'expire':'60',
+            'type':'memory'
+        },
+        'long_term':{
+            'expire':'1800',
+            'type':'dbm',
+            'data_dir':'/tmp',
+        }
+    })
+"""
+
+
+cache_managers = {}
+
+
+class _backends(object):
+    initialized = False
+
+    def __init__(self, clsmap):
+        self._clsmap = clsmap
+        self._mutex = _threading.Lock()
+
+    def __getitem__(self, key):
+        try:
+            return self._clsmap[key]
+        except KeyError, e:
+            if not self.initialized:
+                self._mutex.acquire()
+                try:
+                    if not self.initialized:
+                        self._init()
+                        self.initialized = True
+
+                    return self._clsmap[key]
+                finally:
+                    self._mutex.release()
+
+            raise e
+
+    def _init(self):
+        try:
+            import pkg_resources
+
+            # Load up the additional entry point defined backends
+            for entry_point in pkg_resources.iter_entry_points('beaker.backends'):
+                try:
+                    namespace_manager = entry_point.load()
+                    name = entry_point.name
+                    if name in self._clsmap:
+                        raise BeakerException("NamespaceManager name conflict,'%s' "
+                                              "already loaded" % name)
+                    self._clsmap[name] = namespace_manager
+                except (InvalidCacheBackendError, SyntaxError):
+                    # Ignore invalid backends
+                    pass
+                except:
+                    import sys
+                    from pkg_resources import DistributionNotFound
+                    # Warn when there's a problem loading a NamespaceManager
+                    if not isinstance(sys.exc_info()[1], DistributionNotFound):
+                        import traceback
+                        from StringIO import StringIO
+                        tb = StringIO()
+                        traceback.print_exc(file=tb)
+                        warnings.warn(
+                            "Unable to load NamespaceManager "
+                            "entry point: '%s': %s" % (
+                                        entry_point,
+                                        tb.getvalue()),
+                                        RuntimeWarning, 2)
+        except ImportError:
+            pass
+
+# Initialize the basic available backends
+clsmap = _backends({
+          'memory': container.MemoryNamespaceManager,
+          'dbm': container.DBMNamespaceManager,
+          'file': container.FileNamespaceManager,
+          'ext:memcached': memcached.MemcachedNamespaceManager,
+          'ext:database': database.DatabaseNamespaceManager,
+          'ext:sqla': sqla.SqlaNamespaceManager,
+          'ext:google': google.GoogleNamespaceManager,
+          })
+
+
+def cache_region(region, *args):
+    """Decorate a function such that its return result is cached,
+    using a "region" to indicate the cache arguments.
+
+    Example::
+
+        from beaker.cache import cache_regions, cache_region
+
+        # configure regions
+        cache_regions.update({
+            'short_term':{
+                'expire':'60',
+                'type':'memory'
+            }
+        })
+
+        @cache_region('short_term', 'load_things')
+        def load(search_term, limit, offset):
+            '''Load from a database given a search term, limit, offset.'''
+            return database.query(search_term)[offset:offset + limit]
+
+    The decorator can also be used with object methods.  The ``self``
+    argument is not part of the cache key.  This is based on the
+    actual string name ``self`` being in the first argument
+    position (new in 1.6)::
+
+        class MyThing(object):
+            @cache_region('short_term', 'load_things')
+            def load(self, search_term, limit, offset):
+                '''Load from a database given a search term, limit, offset.'''
+                return database.query(search_term)[offset:offset + limit]
+
+    Classmethods work as well - use ``cls`` as the name of the class argument,
+    and place the decorator around the function underneath ``@classmethod``
+    (new in 1.6)::
+
+        class MyThing(object):
+            @classmethod
+            @cache_region('short_term', 'load_things')
+            def load(cls, search_term, limit, offset):
+                '''Load from a database given a search term, limit, offset.'''
+                return database.query(search_term)[offset:offset + limit]
+
+    :param region: String name of the region corresponding to the desired
+      caching arguments, established in :attr:`.cache_regions`.
+
+    :param \*args: Optional ``str()``-compatible arguments which will uniquely
+      identify the key used by this decorated function, in addition
+      to the positional arguments passed to the function itself at call time.
+      This is recommended as it is needed to distinguish between any two functions
+      or methods that have the same name (regardless of parent class or not).
+
+    .. note::
+
+        The function being decorated must only be called with
+        positional arguments, and the arguments must support
+        being stringified with ``str()``.  The concatenation
+        of the ``str()`` version of each argument, combined
+        with that of the ``*args`` sent to the decorator,
+        forms the unique cache key.
+
+    .. note::
+
+        When a method on a class is decorated, the ``self`` or ``cls``
+        argument in the first position is
+        not included in the "key" used for caching.   New in 1.6.
+
+    """
+    return _cache_decorate(args, None, None, region)
+
+
+def region_invalidate(namespace, region, *args):
+    """Invalidate a cache region corresponding to a function
+    decorated with :func:`.cache_region`.
+
+    :param namespace: The namespace of the cache to invalidate.  This is typically
+      a reference to the original function (as returned by the :func:`.cache_region`
+      decorator), where the :func:`.cache_region` decorator applies a "memo" to
+      the function in order to locate the string name of the namespace.
+
+    :param region: String name of the region used with the decorator.  This can be
+     ``None`` in the usual case that the decorated function itself is passed,
+     not the string name of the namespace.
+
+    :param args: Stringifyable arguments that are used to locate the correct
+     key.  This consists of the ``*args`` sent to the :func:`.cache_region`
+     decorator itself, plus the ``*args`` sent to the function itself
+     at runtime.
+
+    Example::
+
+        from beaker.cache import cache_regions, cache_region, region_invalidate
+
+        # configure regions
+        cache_regions.update({
+            'short_term':{
+                'expire':'60',
+                'type':'memory'
+            }
+        })
+
+        @cache_region('short_term', 'load_data')
+        def load(search_term, limit, offset):
+            '''Load from a database given a search term, limit, offset.'''
+            return database.query(search_term)[offset:offset + limit]
+
+        def invalidate_search(search_term, limit, offset):
+            '''Invalidate the cached storage for a given search term, limit, offset.'''
+            region_invalidate(load, 'short_term', 'load_data', search_term, limit, offset)
+
+    Note that when a method on a class is decorated, the first argument ``cls``
+    or ``self`` is not included in the cache key.  This means you don't send
+    it to :func:`.region_invalidate`::
+
+        class MyThing(object):
+            @cache_region('short_term', 'some_data')
+            def load(self, search_term, limit, offset):
+                '''Load from a database given a search term, limit, offset.'''
+                return database.query(search_term)[offset:offset + limit]
+
+            def invalidate_search(self, search_term, limit, offset):
+                '''Invalidate the cached storage for a given search term, limit, offset.'''
+                region_invalidate(self.load, 'short_term', 'some_data', search_term, limit, offset)
+
+    """
+    if callable(namespace):
+        if not region:
+            region = namespace._arg_region
+        namespace = namespace._arg_namespace
+
+    if not region:
+        raise BeakerException("Region or callable function "
+                                    "namespace is required")
+    else:
+        region = cache_regions[region]
+
+    cache = Cache._get_cache(namespace, region)
+    _cache_decorator_invalidate(cache, region['key_length'], args)
+
+
+class Cache(object):
+    """Front-end to the containment API implementing a data cache.
+
+    :param namespace: the namespace of this Cache
+
+    :param type: type of cache to use
+
+    :param expire: seconds to keep cached data
+
+    :param expiretime: seconds to keep cached data (legacy support)
+
+    :param starttime: time when cache was cache was
+
+    """
+    def __init__(self, namespace, type='memory', expiretime=None,
+                 starttime=None, expire=None, **nsargs):
+        try:
+            cls = clsmap[type]
+            if isinstance(cls, InvalidCacheBackendError):
+                raise cls
+        except KeyError:
+            raise TypeError("Unknown cache implementation %r" % type)
+        self.namespace_name = namespace
+        self.namespace = cls(namespace, **nsargs)
+        self.expiretime = expiretime or expire
+        self.starttime = starttime
+        self.nsargs = nsargs
+
+    @classmethod
+    def _get_cache(cls, namespace, kw):
+        key = namespace + str(kw)
+        try:
+            return cache_managers[key]
+        except KeyError:
+            cache_managers[key] = cache = cls(namespace, **kw)
+            return cache
+
+    def put(self, key, value, **kw):
+        self._get_value(key, **kw).set_value(value)
+    set_value = put
+
+    def get(self, key, **kw):
+        """Retrieve a cached value from the container"""
+        return self._get_value(key, **kw).get_value()
+    get_value = get
+
+    def remove_value(self, key, **kw):
+        mycontainer = self._get_value(key, **kw)
+        mycontainer.clear_value()
+    remove = remove_value
+
+    def _get_value(self, key, **kw):
+        if isinstance(key, unicode):
+            key = key.encode('ascii', 'backslashreplace')
+
+        if 'type' in kw:
+            return self._legacy_get_value(key, **kw)
+
+        kw.setdefault('expiretime', self.expiretime)
+        kw.setdefault('starttime', self.starttime)
+
+        return container.Value(key, self.namespace, **kw)
+
+    @util.deprecated("Specifying a "
+            "'type' and other namespace configuration with cache.get()/put()/etc. "
+            "is deprecated. Specify 'type' and other namespace configuration to "
+            "cache_manager.get_cache() and/or the Cache constructor instead.")
+    def _legacy_get_value(self, key, type, **kw):
+        expiretime = kw.pop('expiretime', self.expiretime)
+        starttime = kw.pop('starttime', None)
+        createfunc = kw.pop('createfunc', None)
+        kwargs = self.nsargs.copy()
+        kwargs.update(kw)
+        c = Cache(self.namespace.namespace, type=type, **kwargs)
+        return c._get_value(key, expiretime=expiretime, createfunc=createfunc,
+                            starttime=starttime)
+
+    def clear(self):
+        """Clear all the values from the namespace"""
+        self.namespace.remove()
+
+    # dict interface
+    def __getitem__(self, key):
+        return self.get(key)
+
+    def __contains__(self, key):
+        return self._get_value(key).has_current_value()
+
+    def has_key(self, key):
+        return key in self
+
+    def __delitem__(self, key):
+        self.remove_value(key)
+
+    def __setitem__(self, key, value):
+        self.put(key, value)
+
+
+class CacheManager(object):
+    def __init__(self, **kwargs):
+        """Initialize a CacheManager object with a set of options
+
+        Options should be parsed with the
+        :func:`~beaker.util.parse_cache_config_options` function to
+        ensure only valid options are used.
+
+        """
+        self.kwargs = kwargs
+        self.regions = kwargs.pop('cache_regions', {})
+
+        # Add these regions to the module global
+        cache_regions.update(self.regions)
+
+    def get_cache(self, name, **kwargs):
+        kw = self.kwargs.copy()
+        kw.update(kwargs)
+        return Cache._get_cache(name, kw)
+
+    def get_cache_region(self, name, region):
+        if region not in self.regions:
+            raise BeakerException('Cache region not configured: %s' % region)
+        kw = self.regions[region]
+        return Cache._get_cache(name, kw)
+
+    def region(self, region, *args):
+        """Decorate a function to cache itself using a cache region
+
+        The region decorator requires arguments if there are more than
+        two of the same named function, in the same module. This is
+        because the namespace used for the functions cache is based on
+        the functions name and the module.
+
+
+        Example::
+
+            # Assuming a cache object is available like:
+            cache = CacheManager(dict_of_config_options)
+
+
+            def populate_things():
+
+                @cache.region('short_term', 'some_data')
+                def load(search_term, limit, offset):
+                    return load_the_data(search_term, limit, offset)
+
+                return load('rabbits', 20, 0)
+
+        .. note::
+
+            The function being decorated must only be called with
+            positional arguments.
+
+        """
+        return cache_region(region, *args)
+
+    def region_invalidate(self, namespace, region, *args):
+        """Invalidate a cache region namespace or decorated function
+
+        This function only invalidates cache spaces created with the
+        cache_region decorator.
+
+        :param namespace: Either the namespace of the result to invalidate, or the
+           cached function
+
+        :param region: The region the function was cached to. If the function was
+            cached to a single region then this argument can be None
+
+        :param args: Arguments that were used to differentiate the cached
+            function as well as the arguments passed to the decorated
+            function
+
+        Example::
+
+            # Assuming a cache object is available like:
+            cache = CacheManager(dict_of_config_options)
+
+            def populate_things(invalidate=False):
+
+                @cache.region('short_term', 'some_data')
+                def load(search_term, limit, offset):
+                    return load_the_data(search_term, limit, offset)
+
+                # If the results should be invalidated first
+                if invalidate:
+                    cache.region_invalidate(load, None, 'some_data',
+                                            'rabbits', 20, 0)
+                return load('rabbits', 20, 0)
+
+
+        """
+        return region_invalidate(namespace, region, *args)
+
+    def cache(self, *args, **kwargs):
+        """Decorate a function to cache itself with supplied parameters
+
+        :param args: Used to make the key unique for this function, as in region()
+            above.
+
+        :param kwargs: Parameters to be passed to get_cache(), will override defaults
+
+        Example::
+
+            # Assuming a cache object is available like:
+            cache = CacheManager(dict_of_config_options)
+
+
+            def populate_things():
+
+                @cache.cache('mycache', expire=15)
+                def load(search_term, limit, offset):
+                    return load_the_data(search_term, limit, offset)
+
+                return load('rabbits', 20, 0)
+
+        .. note::
+
+            The function being decorated must only be called with
+            positional arguments.
+
+        """
+        return _cache_decorate(args, self, kwargs, None)
+
+    def invalidate(self, func, *args, **kwargs):
+        """Invalidate a cache decorated function
+
+        This function only invalidates cache spaces created with the
+        cache decorator.
+
+        :param func: Decorated function to invalidate
+
+        :param args: Used to make the key unique for this function, as in region()
+            above.
+
+        :param kwargs: Parameters that were passed for use by get_cache(), note that
+            this is only required if a ``type`` was specified for the
+            function
+
+        Example::
+
+            # Assuming a cache object is available like:
+            cache = CacheManager(dict_of_config_options)
+
+
+            def populate_things(invalidate=False):
+
+                @cache.cache('mycache', type="file", expire=15)
+                def load(search_term, limit, offset):
+                    return load_the_data(search_term, limit, offset)
+
+                # If the results should be invalidated first
+                if invalidate:
+                    cache.invalidate(load, 'mycache', 'rabbits', 20, 0, type="file")
+                return load('rabbits', 20, 0)
+
+        """
+        namespace = func._arg_namespace
+
+        cache = self.get_cache(namespace, **kwargs)
+        if hasattr(func, '_arg_region'):
+            key_length = cache_regions[func._arg_region]['key_length']
+        else:
+            key_length = kwargs.pop('key_length', 250)
+        _cache_decorator_invalidate(cache, key_length, args)
+
+
+def _cache_decorate(deco_args, manager, kwargs, region):
+    """Return a caching function decorator."""
+
+    cache = [None]
+
+    def decorate(func):
+        namespace = util.func_namespace(func)
+        skip_self = util.has_self_arg(func)
+
+        def cached(*args):
+            if not cache[0]:
+                if region is not None:
+                    if region not in cache_regions:
+                        raise BeakerException(
+                            'Cache region not configured: %s' % region)
+                    reg = cache_regions[region]
+                    if not reg.get('enabled', True):
+                        return func(*args)
+                    cache[0] = Cache._get_cache(namespace, reg)
+                elif manager:
+                    cache[0] = manager.get_cache(namespace, **kwargs)
+                else:
+                    raise Exception("'manager + kwargs' or 'region' "
+                                    "argument is required")
+
+            if skip_self:
+                try:
+                    cache_key = " ".join(map(str, deco_args + args[1:]))
+                except UnicodeEncodeError:
+                    cache_key = " ".join(map(unicode, deco_args + args[1:]))
+            else:
+                try:
+                    cache_key = " ".join(map(str, deco_args + args))
+                except UnicodeEncodeError:
+                    cache_key = " ".join(map(unicode, deco_args + args))
+            if region:
+                key_length = cache_regions[region]['key_length']
+            else:
+                key_length = kwargs.pop('key_length', 250)
+            if len(cache_key) + len(namespace) > int(key_length):
+                cache_key = sha1(cache_key).hexdigest()
+
+            def go():
+                return func(*args)
+
+            return cache[0].get_value(cache_key, createfunc=go)
+        cached._arg_namespace = namespace
+        if region is not None:
+            cached._arg_region = region
+        return cached
+    return decorate
+
+
+def _cache_decorator_invalidate(cache, key_length, args):
+    """Invalidate a cache key based on function arguments."""
+
+    try:
+        cache_key = " ".join(map(str, args))
+    except UnicodeEncodeError:
+        cache_key = " ".join(map(unicode, args))
+    if len(cache_key) + len(cache.namespace_name) > key_length:
+        cache_key = sha1(cache_key).hexdigest()
+    cache.remove_value(cache_key)