diff options
Diffstat (limited to 'module/lib/beaker/cache.py')
| -rw-r--r-- | module/lib/beaker/cache.py | 566 |
1 files changed, 348 insertions, 218 deletions
diff --git a/module/lib/beaker/cache.py b/module/lib/beaker/cache.py index 4a96537ff..0ae96e020 100644 --- a/module/lib/beaker/cache.py +++ b/module/lib/beaker/cache.py @@ -1,156 +1,248 @@ -"""Cache object +"""This package contains the "front end" classes and functions +for Beaker caching. -The Cache object is used to manage a set of cache files and their -associated backend. The backends can be rotated on the fly by -specifying an alternate type when used. - -Advanced users can add new backends in beaker.backends +Included are the :class:`.Cache` and :class:`.CacheManager` classes, +as well as the function decorators :func:`.region_decorate`, +:func:`.region_invalidate`. """ - import warnings import beaker.container as container import beaker.util as util +from beaker.crypto.util import sha1 from beaker.exceptions import BeakerException, InvalidCacheBackendError +from beaker.synchronization import _threading import beaker.ext.memcached as memcached import beaker.ext.database as database import beaker.ext.sqla as sqla import beaker.ext.google as google -# Initialize the basic available backends -clsmap = { - 'memory':container.MemoryNamespaceManager, - 'dbm':container.DBMNamespaceManager, - 'file':container.FileNamespaceManager, - 'ext:memcached':memcached.MemcachedNamespaceManager, - 'ext:database':database.DatabaseNamespaceManager, - 'ext:sqla': sqla.SqlaNamespaceManager, - 'ext:google': google.GoogleNamespaceManager, - } - # Initialize the cache region dict cache_regions = {} +"""Dictionary of 'region' arguments. + +A "region" is a string name that refers to a series of cache +configuration arguments. An application may have multiple +"regions" - one which stores things in a memory cache, one +which writes data to files, etc. + +The dictionary stores string key names mapped to dictionaries +of configuration arguments. Example:: + + from beaker.cache import cache_regions + cache_regions.update({ + 'short_term':{ + 'expire':'60', + 'type':'memory' + }, + 'long_term':{ + 'expire':'1800', + 'type':'dbm', + 'data_dir':'/tmp', + } + }) +""" + + cache_managers = {} -try: - import pkg_resources - # Load up the additional entry point defined backends - for entry_point in pkg_resources.iter_entry_points('beaker.backends'): +class _backends(object): + initialized = False + + def __init__(self, clsmap): + self._clsmap = clsmap + self._mutex = _threading.Lock() + + def __getitem__(self, key): try: - NamespaceManager = entry_point.load() - name = entry_point.name - if name in clsmap: - raise BeakerException("NamespaceManager name conflict,'%s' " - "already loaded" % name) - clsmap[name] = NamespaceManager - except (InvalidCacheBackendError, SyntaxError): - # Ignore invalid backends + return self._clsmap[key] + except KeyError, e: + if not self.initialized: + self._mutex.acquire() + try: + if not self.initialized: + self._init() + self.initialized = True + + return self._clsmap[key] + finally: + self._mutex.release() + + raise e + + def _init(self): + try: + import pkg_resources + + # Load up the additional entry point defined backends + for entry_point in pkg_resources.iter_entry_points('beaker.backends'): + try: + namespace_manager = entry_point.load() + name = entry_point.name + if name in self._clsmap: + raise BeakerException("NamespaceManager name conflict,'%s' " + "already loaded" % name) + self._clsmap[name] = namespace_manager + except (InvalidCacheBackendError, SyntaxError): + # Ignore invalid backends + pass + except: + import sys + from pkg_resources import DistributionNotFound + # Warn when there's a problem loading a NamespaceManager + if not isinstance(sys.exc_info()[1], DistributionNotFound): + import traceback + from StringIO import StringIO + tb = StringIO() + traceback.print_exc(file=tb) + warnings.warn( + "Unable to load NamespaceManager " + "entry point: '%s': %s" % ( + entry_point, + tb.getvalue()), + RuntimeWarning, 2) + except ImportError: pass - except: - import sys - from pkg_resources import DistributionNotFound - # Warn when there's a problem loading a NamespaceManager - if not isinstance(sys.exc_info()[1], DistributionNotFound): - import traceback - from StringIO import StringIO - tb = StringIO() - traceback.print_exc(file=tb) - warnings.warn("Unable to load NamespaceManager entry point: '%s': " - "%s" % (entry_point, tb.getvalue()), RuntimeWarning, - 2) -except ImportError: - pass - - - - -def cache_region(region, *deco_args): - """Decorate a function to cache itself using a cache region - - The region decorator requires arguments if there are more than - 2 of the same named function, in the same module. This is - because the namespace used for the functions cache is based on - the functions name and the module. - - + +# Initialize the basic available backends +clsmap = _backends({ + 'memory': container.MemoryNamespaceManager, + 'dbm': container.DBMNamespaceManager, + 'file': container.FileNamespaceManager, + 'ext:memcached': memcached.MemcachedNamespaceManager, + 'ext:database': database.DatabaseNamespaceManager, + 'ext:sqla': sqla.SqlaNamespaceManager, + 'ext:google': google.GoogleNamespaceManager, + }) + + +def cache_region(region, *args): + """Decorate a function such that its return result is cached, + using a "region" to indicate the cache arguments. + Example:: - - # Add cache region settings to beaker: - beaker.cache.cache_regions.update(dict_of_config_region_options)) - - @cache_region('short_term', 'some_data') - def populate_things(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - return load('rabbits', 20, 0) - + + from beaker.cache import cache_regions, cache_region + + # configure regions + cache_regions.update({ + 'short_term':{ + 'expire':'60', + 'type':'memory' + } + }) + + @cache_region('short_term', 'load_things') + def load(search_term, limit, offset): + '''Load from a database given a search term, limit, offset.''' + return database.query(search_term)[offset:offset + limit] + + The decorator can also be used with object methods. The ``self`` + argument is not part of the cache key. This is based on the + actual string name ``self`` being in the first argument + position (new in 1.6):: + + class MyThing(object): + @cache_region('short_term', 'load_things') + def load(self, search_term, limit, offset): + '''Load from a database given a search term, limit, offset.''' + return database.query(search_term)[offset:offset + limit] + + Classmethods work as well - use ``cls`` as the name of the class argument, + and place the decorator around the function underneath ``@classmethod`` + (new in 1.6):: + + class MyThing(object): + @classmethod + @cache_region('short_term', 'load_things') + def load(cls, search_term, limit, offset): + '''Load from a database given a search term, limit, offset.''' + return database.query(search_term)[offset:offset + limit] + + :param region: String name of the region corresponding to the desired + caching arguments, established in :attr:`.cache_regions`. + + :param \*args: Optional ``str()``-compatible arguments which will uniquely + identify the key used by this decorated function, in addition + to the positional arguments passed to the function itself at call time. + This is recommended as it is needed to distinguish between any two functions + or methods that have the same name (regardless of parent class or not). + .. note:: - + The function being decorated must only be called with - positional arguments. - + positional arguments, and the arguments must support + being stringified with ``str()``. The concatenation + of the ``str()`` version of each argument, combined + with that of the ``*args`` sent to the decorator, + forms the unique cache key. + + .. note:: + + When a method on a class is decorated, the ``self`` or ``cls`` + argument in the first position is + not included in the "key" used for caching. New in 1.6. + """ - cache = [None] - - def decorate(func): - namespace = util.func_namespace(func) - def cached(*args): - reg = cache_regions[region] - if not reg.get('enabled', True): - return func(*args) - - if not cache[0]: - if region not in cache_regions: - raise BeakerException('Cache region not configured: %s' % region) - cache[0] = Cache._get_cache(namespace, reg) - - cache_key = " ".join(map(str, deco_args + args)) - def go(): - return func(*args) - - return cache[0].get_value(cache_key, createfunc=go) - cached._arg_namespace = namespace - cached._arg_region = region - return cached - return decorate + return _cache_decorate(args, None, None, region) def region_invalidate(namespace, region, *args): - """Invalidate a cache region namespace or decorated function - - This function only invalidates cache spaces created with the - cache_region decorator. - - :param namespace: Either the namespace of the result to invalidate, or the - cached function reference - - :param region: The region the function was cached to. If the function was - cached to a single region then this argument can be None - - :param args: Arguments that were used to differentiate the cached - function as well as the arguments passed to the decorated - function + """Invalidate a cache region corresponding to a function + decorated with :func:`.cache_region`. + + :param namespace: The namespace of the cache to invalidate. This is typically + a reference to the original function (as returned by the :func:`.cache_region` + decorator), where the :func:`.cache_region` decorator applies a "memo" to + the function in order to locate the string name of the namespace. + + :param region: String name of the region used with the decorator. This can be + ``None`` in the usual case that the decorated function itself is passed, + not the string name of the namespace. + + :param args: Stringifyable arguments that are used to locate the correct + key. This consists of the ``*args`` sent to the :func:`.cache_region` + decorator itself, plus the ``*args`` sent to the function itself + at runtime. Example:: - - # Add cache region settings to beaker: - beaker.cache.cache_regions.update(dict_of_config_region_options)) - - def populate_things(invalidate=False): - + + from beaker.cache import cache_regions, cache_region, region_invalidate + + # configure regions + cache_regions.update({ + 'short_term':{ + 'expire':'60', + 'type':'memory' + } + }) + + @cache_region('short_term', 'load_data') + def load(search_term, limit, offset): + '''Load from a database given a search term, limit, offset.''' + return database.query(search_term)[offset:offset + limit] + + def invalidate_search(search_term, limit, offset): + '''Invalidate the cached storage for a given search term, limit, offset.''' + region_invalidate(load, 'short_term', 'load_data', search_term, limit, offset) + + Note that when a method on a class is decorated, the first argument ``cls`` + or ``self`` is not included in the cache key. This means you don't send + it to :func:`.region_invalidate`:: + + class MyThing(object): @cache_region('short_term', 'some_data') - def load(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - # If the results should be invalidated first - if invalidate: - region_invalidate(load, None, 'some_data', - 'rabbits', 20, 0) - return load('rabbits', 20, 0) - + def load(self, search_term, limit, offset): + '''Load from a database given a search term, limit, offset.''' + return database.query(search_term)[offset:offset + limit] + + def invalidate_search(self, search_term, limit, offset): + '''Invalidate the cached storage for a given search term, limit, offset.''' + region_invalidate(self.load, 'short_term', 'some_data', search_term, limit, offset) + """ if callable(namespace): if not region: @@ -162,10 +254,9 @@ def region_invalidate(namespace, region, *args): "namespace is required") else: region = cache_regions[region] - + cache = Cache._get_cache(namespace, region) - cache_key = " ".join(str(x) for x in args) - cache.remove_value(cache_key) + _cache_decorator_invalidate(cache, region['key_length'], args) class Cache(object): @@ -180,7 +271,7 @@ class Cache(object): :param expiretime: seconds to keep cached data (legacy support) :param starttime: time when cache was cache was - + """ def __init__(self, namespace, type='memory', expiretime=None, starttime=None, expire=None, **nsargs): @@ -190,12 +281,12 @@ class Cache(object): raise cls except KeyError: raise TypeError("Unknown cache implementation %r" % type) - + self.namespace_name = namespace self.namespace = cls(namespace, **nsargs) self.expiretime = expiretime or expire self.starttime = starttime self.nsargs = nsargs - + @classmethod def _get_cache(cls, namespace, kw): key = namespace + str(kw) @@ -204,20 +295,19 @@ class Cache(object): except KeyError: cache_managers[key] = cache = cls(namespace, **kw) return cache - + def put(self, key, value, **kw): self._get_value(key, **kw).set_value(value) set_value = put - + def get(self, key, **kw): """Retrieve a cached value from the container""" return self._get_value(key, **kw).get_value() get_value = get - + def remove_value(self, key, **kw): mycontainer = self._get_value(key, **kw) - if mycontainer.has_current_value(): - mycontainer.clear_value() + mycontainer.clear_value() remove = remove_value def _get_value(self, key, **kw): @@ -229,9 +319,9 @@ class Cache(object): kw.setdefault('expiretime', self.expiretime) kw.setdefault('starttime', self.starttime) - + return container.Value(key, self.namespace, **kw) - + @util.deprecated("Specifying a " "'type' and other namespace configuration with cache.get()/put()/etc. " "is deprecated. Specify 'type' and other namespace configuration to " @@ -243,26 +333,26 @@ class Cache(object): kwargs = self.nsargs.copy() kwargs.update(kw) c = Cache(self.namespace.namespace, type=type, **kwargs) - return c._get_value(key, expiretime=expiretime, createfunc=createfunc, + return c._get_value(key, expiretime=expiretime, createfunc=createfunc, starttime=starttime) - + def clear(self): """Clear all the values from the namespace""" self.namespace.remove() - + # dict interface def __getitem__(self, key): return self.get(key) - + def __contains__(self, key): return self._get_value(key).has_current_value() - + def has_key(self, key): return key in self - + def __delitem__(self, key): self.remove_value(key) - + def __setitem__(self, key, value): self.put(key, value) @@ -270,110 +360,96 @@ class Cache(object): class CacheManager(object): def __init__(self, **kwargs): """Initialize a CacheManager object with a set of options - + Options should be parsed with the :func:`~beaker.util.parse_cache_config_options` function to ensure only valid options are used. - + """ self.kwargs = kwargs self.regions = kwargs.pop('cache_regions', {}) - + # Add these regions to the module global cache_regions.update(self.regions) - + def get_cache(self, name, **kwargs): kw = self.kwargs.copy() kw.update(kwargs) return Cache._get_cache(name, kw) - + def get_cache_region(self, name, region): if region not in self.regions: raise BeakerException('Cache region not configured: %s' % region) kw = self.regions[region] return Cache._get_cache(name, kw) - + def region(self, region, *args): """Decorate a function to cache itself using a cache region - + The region decorator requires arguments if there are more than - 2 of the same named function, in the same module. This is + two of the same named function, in the same module. This is because the namespace used for the functions cache is based on the functions name and the module. - - + + Example:: - + # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - - + + def populate_things(): - + @cache.region('short_term', 'some_data') def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + return load('rabbits', 20, 0) - + .. note:: - + The function being decorated must only be called with positional arguments. - + """ return cache_region(region, *args) def region_invalidate(self, namespace, region, *args): """Invalidate a cache region namespace or decorated function - + This function only invalidates cache spaces created with the cache_region decorator. - + :param namespace: Either the namespace of the result to invalidate, or the - name of the cached function - + cached function + :param region: The region the function was cached to. If the function was cached to a single region then this argument can be None - + :param args: Arguments that were used to differentiate the cached function as well as the arguments passed to the decorated function Example:: - + # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - + def populate_things(invalidate=False): - + @cache.region('short_term', 'some_data') def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + # If the results should be invalidated first if invalidate: cache.region_invalidate(load, None, 'some_data', 'rabbits', 20, 0) return load('rabbits', 20, 0) - - + + """ return region_invalidate(namespace, region, *args) - if callable(namespace): - if not region: - region = namespace._arg_region - namespace = namespace._arg_namespace - - if not region: - raise BeakerException("Region or callable function " - "namespace is required") - else: - region = self.regions[region] - - cache = self.get_cache(namespace, **region) - cache_key = " ".join(str(x) for x in args) - cache.remove_value(cache_key) def cache(self, *args, **kwargs): """Decorate a function to cache itself with supplied parameters @@ -387,46 +463,32 @@ class CacheManager(object): # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - - + + def populate_things(): - + @cache.cache('mycache', expire=15) def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + return load('rabbits', 20, 0) - + .. note:: - + The function being decorated must only be called with - positional arguments. + positional arguments. """ - cache = [None] - key = " ".join(str(x) for x in args) - - def decorate(func): - namespace = util.func_namespace(func) - def cached(*args): - if not cache[0]: - cache[0] = self.get_cache(namespace, **kwargs) - cache_key = key + " " + " ".join(str(x) for x in args) - def go(): - return func(*args) - return cache[0].get_value(cache_key, createfunc=go) - cached._arg_namespace = namespace - return cached - return decorate + return _cache_decorate(args, self, kwargs, None) def invalidate(self, func, *args, **kwargs): """Invalidate a cache decorated function - + This function only invalidates cache spaces created with the cache decorator. - + :param func: Decorated function to invalidate - + :param args: Used to make the key unique for this function, as in region() above. @@ -435,25 +497,93 @@ class CacheManager(object): function Example:: - + # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - - + + def populate_things(invalidate=False): - + @cache.cache('mycache', type="file", expire=15) def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + # If the results should be invalidated first if invalidate: cache.invalidate(load, 'mycache', 'rabbits', 20, 0, type="file") return load('rabbits', 20, 0) - + """ namespace = func._arg_namespace cache = self.get_cache(namespace, **kwargs) - cache_key = " ".join(str(x) for x in args) - cache.remove_value(cache_key) + if hasattr(func, '_arg_region'): + key_length = cache_regions[func._arg_region]['key_length'] + else: + key_length = kwargs.pop('key_length', 250) + _cache_decorator_invalidate(cache, key_length, args) + + +def _cache_decorate(deco_args, manager, kwargs, region): + """Return a caching function decorator.""" + + cache = [None] + + def decorate(func): + namespace = util.func_namespace(func) + skip_self = util.has_self_arg(func) + + def cached(*args): + if not cache[0]: + if region is not None: + if region not in cache_regions: + raise BeakerException( + 'Cache region not configured: %s' % region) + reg = cache_regions[region] + if not reg.get('enabled', True): + return func(*args) + cache[0] = Cache._get_cache(namespace, reg) + elif manager: + cache[0] = manager.get_cache(namespace, **kwargs) + else: + raise Exception("'manager + kwargs' or 'region' " + "argument is required") + + if skip_self: + try: + cache_key = " ".join(map(str, deco_args + args[1:])) + except UnicodeEncodeError: + cache_key = " ".join(map(unicode, deco_args + args[1:])) + else: + try: + cache_key = " ".join(map(str, deco_args + args)) + except UnicodeEncodeError: + cache_key = " ".join(map(unicode, deco_args + args)) + if region: + key_length = cache_regions[region]['key_length'] + else: + key_length = kwargs.pop('key_length', 250) + if len(cache_key) + len(namespace) > int(key_length): + cache_key = sha1(cache_key).hexdigest() + + def go(): + return func(*args) + + return cache[0].get_value(cache_key, createfunc=go) + cached._arg_namespace = namespace + if region is not None: + cached._arg_region = region + return cached + return decorate + + +def _cache_decorator_invalidate(cache, key_length, args): + """Invalidate a cache key based on function arguments.""" + + try: + cache_key = " ".join(map(str, args)) + except UnicodeEncodeError: + cache_key = " ".join(map(unicode, args)) + if len(cache_key) + len(cache.namespace_name) > key_length: + cache_key = sha1(cache_key).hexdigest() + cache.remove_value(cache_key) |