diff options
Diffstat (limited to 'pyload/lib/beaker/cache.py')
| -rw-r--r-- | pyload/lib/beaker/cache.py | 589 |
1 files changed, 589 insertions, 0 deletions
diff --git a/pyload/lib/beaker/cache.py b/pyload/lib/beaker/cache.py new file mode 100644 index 000000000..0ae96e020 --- /dev/null +++ b/pyload/lib/beaker/cache.py @@ -0,0 +1,589 @@ +"""This package contains the "front end" classes and functions +for Beaker caching. + +Included are the :class:`.Cache` and :class:`.CacheManager` classes, +as well as the function decorators :func:`.region_decorate`, +:func:`.region_invalidate`. + +""" +import warnings + +import beaker.container as container +import beaker.util as util +from beaker.crypto.util import sha1 +from beaker.exceptions import BeakerException, InvalidCacheBackendError +from beaker.synchronization import _threading + +import beaker.ext.memcached as memcached +import beaker.ext.database as database +import beaker.ext.sqla as sqla +import beaker.ext.google as google + +# Initialize the cache region dict +cache_regions = {} +"""Dictionary of 'region' arguments. + +A "region" is a string name that refers to a series of cache +configuration arguments. An application may have multiple +"regions" - one which stores things in a memory cache, one +which writes data to files, etc. + +The dictionary stores string key names mapped to dictionaries +of configuration arguments. Example:: + + from beaker.cache import cache_regions + cache_regions.update({ + 'short_term':{ + 'expire':'60', + 'type':'memory' + }, + 'long_term':{ + 'expire':'1800', + 'type':'dbm', + 'data_dir':'/tmp', + } + }) +""" + + +cache_managers = {} + + +class _backends(object): + initialized = False + + def __init__(self, clsmap): + self._clsmap = clsmap + self._mutex = _threading.Lock() + + def __getitem__(self, key): + try: + return self._clsmap[key] + except KeyError, e: + if not self.initialized: + self._mutex.acquire() + try: + if not self.initialized: + self._init() + self.initialized = True + + return self._clsmap[key] + finally: + self._mutex.release() + + raise e + + def _init(self): + try: + import pkg_resources + + # Load up the additional entry point defined backends + for entry_point in pkg_resources.iter_entry_points('beaker.backends'): + try: + namespace_manager = entry_point.load() + name = entry_point.name + if name in self._clsmap: + raise BeakerException("NamespaceManager name conflict,'%s' " + "already loaded" % name) + self._clsmap[name] = namespace_manager + except (InvalidCacheBackendError, SyntaxError): + # Ignore invalid backends + pass + except: + import sys + from pkg_resources import DistributionNotFound + # Warn when there's a problem loading a NamespaceManager + if not isinstance(sys.exc_info()[1], DistributionNotFound): + import traceback + from StringIO import StringIO + tb = StringIO() + traceback.print_exc(file=tb) + warnings.warn( + "Unable to load NamespaceManager " + "entry point: '%s': %s" % ( + entry_point, + tb.getvalue()), + RuntimeWarning, 2) + except ImportError: + pass + +# Initialize the basic available backends +clsmap = _backends({ + 'memory': container.MemoryNamespaceManager, + 'dbm': container.DBMNamespaceManager, + 'file': container.FileNamespaceManager, + 'ext:memcached': memcached.MemcachedNamespaceManager, + 'ext:database': database.DatabaseNamespaceManager, + 'ext:sqla': sqla.SqlaNamespaceManager, + 'ext:google': google.GoogleNamespaceManager, + }) + + +def cache_region(region, *args): + """Decorate a function such that its return result is cached, + using a "region" to indicate the cache arguments. + + Example:: + + from beaker.cache import cache_regions, cache_region + + # configure regions + cache_regions.update({ + 'short_term':{ + 'expire':'60', + 'type':'memory' + } + }) + + @cache_region('short_term', 'load_things') + def load(search_term, limit, offset): + '''Load from a database given a search term, limit, offset.''' + return database.query(search_term)[offset:offset + limit] + + The decorator can also be used with object methods. The ``self`` + argument is not part of the cache key. This is based on the + actual string name ``self`` being in the first argument + position (new in 1.6):: + + class MyThing(object): + @cache_region('short_term', 'load_things') + def load(self, search_term, limit, offset): + '''Load from a database given a search term, limit, offset.''' + return database.query(search_term)[offset:offset + limit] + + Classmethods work as well - use ``cls`` as the name of the class argument, + and place the decorator around the function underneath ``@classmethod`` + (new in 1.6):: + + class MyThing(object): + @classmethod + @cache_region('short_term', 'load_things') + def load(cls, search_term, limit, offset): + '''Load from a database given a search term, limit, offset.''' + return database.query(search_term)[offset:offset + limit] + + :param region: String name of the region corresponding to the desired + caching arguments, established in :attr:`.cache_regions`. + + :param \*args: Optional ``str()``-compatible arguments which will uniquely + identify the key used by this decorated function, in addition + to the positional arguments passed to the function itself at call time. + This is recommended as it is needed to distinguish between any two functions + or methods that have the same name (regardless of parent class or not). + + .. note:: + + The function being decorated must only be called with + positional arguments, and the arguments must support + being stringified with ``str()``. The concatenation + of the ``str()`` version of each argument, combined + with that of the ``*args`` sent to the decorator, + forms the unique cache key. + + .. note:: + + When a method on a class is decorated, the ``self`` or ``cls`` + argument in the first position is + not included in the "key" used for caching. New in 1.6. + + """ + return _cache_decorate(args, None, None, region) + + +def region_invalidate(namespace, region, *args): + """Invalidate a cache region corresponding to a function + decorated with :func:`.cache_region`. + + :param namespace: The namespace of the cache to invalidate. This is typically + a reference to the original function (as returned by the :func:`.cache_region` + decorator), where the :func:`.cache_region` decorator applies a "memo" to + the function in order to locate the string name of the namespace. + + :param region: String name of the region used with the decorator. This can be + ``None`` in the usual case that the decorated function itself is passed, + not the string name of the namespace. + + :param args: Stringifyable arguments that are used to locate the correct + key. This consists of the ``*args`` sent to the :func:`.cache_region` + decorator itself, plus the ``*args`` sent to the function itself + at runtime. + + Example:: + + from beaker.cache import cache_regions, cache_region, region_invalidate + + # configure regions + cache_regions.update({ + 'short_term':{ + 'expire':'60', + 'type':'memory' + } + }) + + @cache_region('short_term', 'load_data') + def load(search_term, limit, offset): + '''Load from a database given a search term, limit, offset.''' + return database.query(search_term)[offset:offset + limit] + + def invalidate_search(search_term, limit, offset): + '''Invalidate the cached storage for a given search term, limit, offset.''' + region_invalidate(load, 'short_term', 'load_data', search_term, limit, offset) + + Note that when a method on a class is decorated, the first argument ``cls`` + or ``self`` is not included in the cache key. This means you don't send + it to :func:`.region_invalidate`:: + + class MyThing(object): + @cache_region('short_term', 'some_data') + def load(self, search_term, limit, offset): + '''Load from a database given a search term, limit, offset.''' + return database.query(search_term)[offset:offset + limit] + + def invalidate_search(self, search_term, limit, offset): + '''Invalidate the cached storage for a given search term, limit, offset.''' + region_invalidate(self.load, 'short_term', 'some_data', search_term, limit, offset) + + """ + if callable(namespace): + if not region: + region = namespace._arg_region + namespace = namespace._arg_namespace + + if not region: + raise BeakerException("Region or callable function " + "namespace is required") + else: + region = cache_regions[region] + + cache = Cache._get_cache(namespace, region) + _cache_decorator_invalidate(cache, region['key_length'], args) + + +class Cache(object): + """Front-end to the containment API implementing a data cache. + + :param namespace: the namespace of this Cache + + :param type: type of cache to use + + :param expire: seconds to keep cached data + + :param expiretime: seconds to keep cached data (legacy support) + + :param starttime: time when cache was cache was + + """ + def __init__(self, namespace, type='memory', expiretime=None, + starttime=None, expire=None, **nsargs): + try: + cls = clsmap[type] + if isinstance(cls, InvalidCacheBackendError): + raise cls + except KeyError: + raise TypeError("Unknown cache implementation %r" % type) + self.namespace_name = namespace + self.namespace = cls(namespace, **nsargs) + self.expiretime = expiretime or expire + self.starttime = starttime + self.nsargs = nsargs + + @classmethod + def _get_cache(cls, namespace, kw): + key = namespace + str(kw) + try: + return cache_managers[key] + except KeyError: + cache_managers[key] = cache = cls(namespace, **kw) + return cache + + def put(self, key, value, **kw): + self._get_value(key, **kw).set_value(value) + set_value = put + + def get(self, key, **kw): + """Retrieve a cached value from the container""" + return self._get_value(key, **kw).get_value() + get_value = get + + def remove_value(self, key, **kw): + mycontainer = self._get_value(key, **kw) + mycontainer.clear_value() + remove = remove_value + + def _get_value(self, key, **kw): + if isinstance(key, unicode): + key = key.encode('ascii', 'backslashreplace') + + if 'type' in kw: + return self._legacy_get_value(key, **kw) + + kw.setdefault('expiretime', self.expiretime) + kw.setdefault('starttime', self.starttime) + + return container.Value(key, self.namespace, **kw) + + @util.deprecated("Specifying a " + "'type' and other namespace configuration with cache.get()/put()/etc. " + "is deprecated. Specify 'type' and other namespace configuration to " + "cache_manager.get_cache() and/or the Cache constructor instead.") + def _legacy_get_value(self, key, type, **kw): + expiretime = kw.pop('expiretime', self.expiretime) + starttime = kw.pop('starttime', None) + createfunc = kw.pop('createfunc', None) + kwargs = self.nsargs.copy() + kwargs.update(kw) + c = Cache(self.namespace.namespace, type=type, **kwargs) + return c._get_value(key, expiretime=expiretime, createfunc=createfunc, + starttime=starttime) + + def clear(self): + """Clear all the values from the namespace""" + self.namespace.remove() + + # dict interface + def __getitem__(self, key): + return self.get(key) + + def __contains__(self, key): + return self._get_value(key).has_current_value() + + def has_key(self, key): + return key in self + + def __delitem__(self, key): + self.remove_value(key) + + def __setitem__(self, key, value): + self.put(key, value) + + +class CacheManager(object): + def __init__(self, **kwargs): + """Initialize a CacheManager object with a set of options + + Options should be parsed with the + :func:`~beaker.util.parse_cache_config_options` function to + ensure only valid options are used. + + """ + self.kwargs = kwargs + self.regions = kwargs.pop('cache_regions', {}) + + # Add these regions to the module global + cache_regions.update(self.regions) + + def get_cache(self, name, **kwargs): + kw = self.kwargs.copy() + kw.update(kwargs) + return Cache._get_cache(name, kw) + + def get_cache_region(self, name, region): + if region not in self.regions: + raise BeakerException('Cache region not configured: %s' % region) + kw = self.regions[region] + return Cache._get_cache(name, kw) + + def region(self, region, *args): + """Decorate a function to cache itself using a cache region + + The region decorator requires arguments if there are more than + two of the same named function, in the same module. This is + because the namespace used for the functions cache is based on + the functions name and the module. + + + Example:: + + # Assuming a cache object is available like: + cache = CacheManager(dict_of_config_options) + + + def populate_things(): + + @cache.region('short_term', 'some_data') + def load(search_term, limit, offset): + return load_the_data(search_term, limit, offset) + + return load('rabbits', 20, 0) + + .. note:: + + The function being decorated must only be called with + positional arguments. + + """ + return cache_region(region, *args) + + def region_invalidate(self, namespace, region, *args): + """Invalidate a cache region namespace or decorated function + + This function only invalidates cache spaces created with the + cache_region decorator. + + :param namespace: Either the namespace of the result to invalidate, or the + cached function + + :param region: The region the function was cached to. If the function was + cached to a single region then this argument can be None + + :param args: Arguments that were used to differentiate the cached + function as well as the arguments passed to the decorated + function + + Example:: + + # Assuming a cache object is available like: + cache = CacheManager(dict_of_config_options) + + def populate_things(invalidate=False): + + @cache.region('short_term', 'some_data') + def load(search_term, limit, offset): + return load_the_data(search_term, limit, offset) + + # If the results should be invalidated first + if invalidate: + cache.region_invalidate(load, None, 'some_data', + 'rabbits', 20, 0) + return load('rabbits', 20, 0) + + + """ + return region_invalidate(namespace, region, *args) + + def cache(self, *args, **kwargs): + """Decorate a function to cache itself with supplied parameters + + :param args: Used to make the key unique for this function, as in region() + above. + + :param kwargs: Parameters to be passed to get_cache(), will override defaults + + Example:: + + # Assuming a cache object is available like: + cache = CacheManager(dict_of_config_options) + + + def populate_things(): + + @cache.cache('mycache', expire=15) + def load(search_term, limit, offset): + return load_the_data(search_term, limit, offset) + + return load('rabbits', 20, 0) + + .. note:: + + The function being decorated must only be called with + positional arguments. + + """ + return _cache_decorate(args, self, kwargs, None) + + def invalidate(self, func, *args, **kwargs): + """Invalidate a cache decorated function + + This function only invalidates cache spaces created with the + cache decorator. + + :param func: Decorated function to invalidate + + :param args: Used to make the key unique for this function, as in region() + above. + + :param kwargs: Parameters that were passed for use by get_cache(), note that + this is only required if a ``type`` was specified for the + function + + Example:: + + # Assuming a cache object is available like: + cache = CacheManager(dict_of_config_options) + + + def populate_things(invalidate=False): + + @cache.cache('mycache', type="file", expire=15) + def load(search_term, limit, offset): + return load_the_data(search_term, limit, offset) + + # If the results should be invalidated first + if invalidate: + cache.invalidate(load, 'mycache', 'rabbits', 20, 0, type="file") + return load('rabbits', 20, 0) + + """ + namespace = func._arg_namespace + + cache = self.get_cache(namespace, **kwargs) + if hasattr(func, '_arg_region'): + key_length = cache_regions[func._arg_region]['key_length'] + else: + key_length = kwargs.pop('key_length', 250) + _cache_decorator_invalidate(cache, key_length, args) + + +def _cache_decorate(deco_args, manager, kwargs, region): + """Return a caching function decorator.""" + + cache = [None] + + def decorate(func): + namespace = util.func_namespace(func) + skip_self = util.has_self_arg(func) + + def cached(*args): + if not cache[0]: + if region is not None: + if region not in cache_regions: + raise BeakerException( + 'Cache region not configured: %s' % region) + reg = cache_regions[region] + if not reg.get('enabled', True): + return func(*args) + cache[0] = Cache._get_cache(namespace, reg) + elif manager: + cache[0] = manager.get_cache(namespace, **kwargs) + else: + raise Exception("'manager + kwargs' or 'region' " + "argument is required") + + if skip_self: + try: + cache_key = " ".join(map(str, deco_args + args[1:])) + except UnicodeEncodeError: + cache_key = " ".join(map(unicode, deco_args + args[1:])) + else: + try: + cache_key = " ".join(map(str, deco_args + args)) + except UnicodeEncodeError: + cache_key = " ".join(map(unicode, deco_args + args)) + if region: + key_length = cache_regions[region]['key_length'] + else: + key_length = kwargs.pop('key_length', 250) + if len(cache_key) + len(namespace) > int(key_length): + cache_key = sha1(cache_key).hexdigest() + + def go(): + return func(*args) + + return cache[0].get_value(cache_key, createfunc=go) + cached._arg_namespace = namespace + if region is not None: + cached._arg_region = region + return cached + return decorate + + +def _cache_decorator_invalidate(cache, key_length, args): + """Invalidate a cache key based on function arguments.""" + + try: + cache_key = " ".join(map(str, args)) + except UnicodeEncodeError: + cache_key = " ".join(map(unicode, args)) + if len(cache_key) + len(cache.namespace_name) > key_length: + cache_key = sha1(cache_key).hexdigest() + cache.remove_value(cache_key) |