diff options
Diffstat (limited to 'module/lib')
| -rw-r--r-- | module/lib/Unzip.py | 2 | ||||
| -rw-r--r-- | module/lib/beaker/cache.py | 160 | ||||
| -rw-r--r-- | module/lib/beaker/container.py | 100 | ||||
| -rw-r--r-- | module/lib/beaker/crypto/pbkdf2.py | 40 | ||||
| -rw-r--r-- | module/lib/beaker/crypto/pycrypto.py | 6 | ||||
| -rw-r--r-- | module/lib/beaker/crypto/util.py | 4 | ||||
| -rw-r--r-- | module/lib/beaker/ext/database.py | 20 | ||||
| -rw-r--r-- | module/lib/beaker/ext/google.py | 22 | ||||
| -rw-r--r-- | module/lib/beaker/ext/memcached.py | 14 | ||||
| -rw-r--r-- | module/lib/beaker/middleware.py | 46 | ||||
| -rw-r--r-- | module/lib/beaker/session.py | 166 | ||||
| -rw-r--r-- | module/lib/beaker/synchronization.py | 54 | ||||
| -rw-r--r-- | module/lib/beaker/util.py | 39 | ||||
| -rw-r--r-- | module/lib/bottle.py | 8 | ||||
| -rw-r--r-- | module/lib/feedparser.py | 192 | ||||
| -rw-r--r-- | module/lib/simplejson/__init__.py | 4 | ||||
| -rw-r--r-- | module/lib/simplejson/encoder.py | 2 | ||||
| -rw-r--r-- | module/lib/thrift/protocol/TBase.py | 8 | ||||
| -rw-r--r-- | module/lib/thrift/protocol/TProtocol.py | 2 | ||||
| -rw-r--r-- | module/lib/thrift/server/TNonblockingServer.py | 22 | ||||
| -rw-r--r-- | module/lib/thrift/transport/TZlibTransport.py | 22 | ||||
| -rw-r--r-- | module/lib/wsgiserver/__init__.py | 347 |
22 files changed, 636 insertions, 644 deletions
diff --git a/module/lib/Unzip.py b/module/lib/Unzip.py index f56fbe751..6d2ada8f2 100644 --- a/module/lib/Unzip.py +++ b/module/lib/Unzip.py @@ -4,7 +4,7 @@ import os class Unzip: def __init__(self): pass - + def extract(self, file, dir): if not dir.endswith(':') and not os.path.exists(dir): os.mkdir(dir) diff --git a/module/lib/beaker/cache.py b/module/lib/beaker/cache.py index 4a96537ff..f59c1e44d 100644 --- a/module/lib/beaker/cache.py +++ b/module/lib/beaker/cache.py @@ -7,7 +7,7 @@ specifying an alternate type when used. Advanced users can add new backends in beaker.backends """ - + import warnings import beaker.container as container @@ -63,54 +63,53 @@ try: 2) except ImportError: pass - + def cache_region(region, *deco_args): """Decorate a function to cache itself using a cache region - + The region decorator requires arguments if there are more than 2 of the same named function, in the same module. This is because the namespace used for the functions cache is based on the functions name and the module. - - + Example:: - + # Add cache region settings to beaker: beaker.cache.cache_regions.update(dict_of_config_region_options)) - + @cache_region('short_term', 'some_data') def populate_things(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + return load('rabbits', 20, 0) - + .. note:: - + The function being decorated must only be called with positional arguments. - + """ cache = [None] - + def decorate(func): namespace = util.func_namespace(func) def cached(*args): reg = cache_regions[region] if not reg.get('enabled', True): return func(*args) - + if not cache[0]: if region not in cache_regions: raise BeakerException('Cache region not configured: %s' % region) cache[0] = Cache._get_cache(namespace, reg) - + cache_key = " ".join(map(str, deco_args + args)) def go(): return func(*args) - + return cache[0].get_value(cache_key, createfunc=go) cached._arg_namespace = namespace cached._arg_region = region @@ -120,37 +119,37 @@ def cache_region(region, *deco_args): def region_invalidate(namespace, region, *args): """Invalidate a cache region namespace or decorated function - + This function only invalidates cache spaces created with the cache_region decorator. - + :param namespace: Either the namespace of the result to invalidate, or the cached function reference - + :param region: The region the function was cached to. If the function was cached to a single region then this argument can be None - + :param args: Arguments that were used to differentiate the cached function as well as the arguments passed to the decorated function Example:: - + # Add cache region settings to beaker: beaker.cache.cache_regions.update(dict_of_config_region_options)) - + def populate_things(invalidate=False): - + @cache_region('short_term', 'some_data') def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + # If the results should be invalidated first if invalidate: region_invalidate(load, None, 'some_data', 'rabbits', 20, 0) return load('rabbits', 20, 0) - + """ if callable(namespace): if not region: @@ -162,7 +161,7 @@ def region_invalidate(namespace, region, *args): "namespace is required") else: region = cache_regions[region] - + cache = Cache._get_cache(namespace, region) cache_key = " ".join(str(x) for x in args) cache.remove_value(cache_key) @@ -180,7 +179,7 @@ class Cache(object): :param expiretime: seconds to keep cached data (legacy support) :param starttime: time when cache was cache was - + """ def __init__(self, namespace, type='memory', expiretime=None, starttime=None, expire=None, **nsargs): @@ -190,12 +189,12 @@ class Cache(object): raise cls except KeyError: raise TypeError("Unknown cache implementation %r" % type) - + self.namespace = cls(namespace, **nsargs) self.expiretime = expiretime or expire self.starttime = starttime self.nsargs = nsargs - + @classmethod def _get_cache(cls, namespace, kw): key = namespace + str(kw) @@ -204,16 +203,16 @@ class Cache(object): except KeyError: cache_managers[key] = cache = cls(namespace, **kw) return cache - + def put(self, key, value, **kw): self._get_value(key, **kw).set_value(value) set_value = put - + def get(self, key, **kw): """Retrieve a cached value from the container""" return self._get_value(key, **kw).get_value() get_value = get - + def remove_value(self, key, **kw): mycontainer = self._get_value(key, **kw) if mycontainer.has_current_value(): @@ -229,9 +228,9 @@ class Cache(object): kw.setdefault('expiretime', self.expiretime) kw.setdefault('starttime', self.starttime) - + return container.Value(key, self.namespace, **kw) - + @util.deprecated("Specifying a " "'type' and other namespace configuration with cache.get()/put()/etc. " "is deprecated. Specify 'type' and other namespace configuration to " @@ -245,24 +244,24 @@ class Cache(object): c = Cache(self.namespace.namespace, type=type, **kwargs) return c._get_value(key, expiretime=expiretime, createfunc=createfunc, starttime=starttime) - + def clear(self): """Clear all the values from the namespace""" self.namespace.remove() - + # dict interface def __getitem__(self, key): return self.get(key) - + def __contains__(self, key): return self._get_value(key).has_current_value() - + def has_key(self, key): return key in self - + def __delitem__(self, key): self.remove_value(key) - + def __setitem__(self, key, value): self.put(key, value) @@ -270,94 +269,91 @@ class Cache(object): class CacheManager(object): def __init__(self, **kwargs): """Initialize a CacheManager object with a set of options - + Options should be parsed with the :func:`~beaker.util.parse_cache_config_options` function to ensure only valid options are used. - + """ self.kwargs = kwargs self.regions = kwargs.pop('cache_regions', {}) - + # Add these regions to the module global cache_regions.update(self.regions) - + def get_cache(self, name, **kwargs): kw = self.kwargs.copy() kw.update(kwargs) return Cache._get_cache(name, kw) - + def get_cache_region(self, name, region): if region not in self.regions: raise BeakerException('Cache region not configured: %s' % region) kw = self.regions[region] return Cache._get_cache(name, kw) - + def region(self, region, *args): """Decorate a function to cache itself using a cache region - + The region decorator requires arguments if there are more than 2 of the same named function, in the same module. This is because the namespace used for the functions cache is based on the functions name and the module. - - + Example:: - + # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - - + def populate_things(): - + @cache.region('short_term', 'some_data') def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + return load('rabbits', 20, 0) - + .. note:: - + The function being decorated must only be called with positional arguments. - + """ return cache_region(region, *args) def region_invalidate(self, namespace, region, *args): """Invalidate a cache region namespace or decorated function - + This function only invalidates cache spaces created with the cache_region decorator. - + :param namespace: Either the namespace of the result to invalidate, or the name of the cached function - + :param region: The region the function was cached to. If the function was cached to a single region then this argument can be None - + :param args: Arguments that were used to differentiate the cached function as well as the arguments passed to the decorated function Example:: - + # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - + def populate_things(invalidate=False): - + @cache.region('short_term', 'some_data') def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + # If the results should be invalidated first if invalidate: cache.region_invalidate(load, None, 'some_data', 'rabbits', 20, 0) return load('rabbits', 20, 0) - - + """ return region_invalidate(namespace, region, *args) if callable(namespace): @@ -370,7 +366,7 @@ class CacheManager(object): "namespace is required") else: region = self.regions[region] - + cache = self.get_cache(namespace, **region) cache_key = " ".join(str(x) for x in args) cache.remove_value(cache_key) @@ -387,25 +383,24 @@ class CacheManager(object): # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - - + def populate_things(): - + @cache.cache('mycache', expire=15) def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + return load('rabbits', 20, 0) - + .. note:: - + The function being decorated must only be called with positional arguments. """ cache = [None] key = " ".join(str(x) for x in args) - + def decorate(func): namespace = util.func_namespace(func) def cached(*args): @@ -421,12 +416,12 @@ class CacheManager(object): def invalidate(self, func, *args, **kwargs): """Invalidate a cache decorated function - + This function only invalidates cache spaces created with the cache decorator. - + :param func: Decorated function to invalidate - + :param args: Used to make the key unique for this function, as in region() above. @@ -435,22 +430,21 @@ class CacheManager(object): function Example:: - + # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - - + def populate_things(invalidate=False): - + @cache.cache('mycache', type="file", expire=15) def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + # If the results should be invalidated first if invalidate: cache.invalidate(load, 'mycache', 'rabbits', 20, 0, type="file") return load('rabbits', 20, 0) - + """ namespace = func._arg_namespace diff --git a/module/lib/beaker/container.py b/module/lib/beaker/container.py index 515e97af6..e7a938c6e 100644 --- a/module/lib/beaker/container.py +++ b/module/lib/beaker/container.py @@ -28,34 +28,34 @@ else: class NamespaceManager(object): """Handles dictionary operations and locking for a namespace of values. - + The implementation for setting and retrieving the namespace data is handled by subclasses. - + NamespaceManager may be used alone, or may be privately accessed by one or more Container objects. Container objects provide per-key services like expiration times and automatic recreation of values. - + Multiple NamespaceManagers created with a particular name will all share access to the same underlying datasource and will attempt to synchronize against a common mutex object. The scope of this sharing may be within a single process or across multiple processes, depending on the type of NamespaceManager used. - + The NamespaceManager itself is generally threadsafe, except in the case of the DBMNamespaceManager in conjunction with the gdbm dbm implementation. """ - + @classmethod def _init_dependencies(cls): pass - + def __init__(self, namespace): self._init_dependencies() self.namespace = namespace - + def get_creation_lock(self, key): raise NotImplementedError() @@ -79,37 +79,37 @@ class NamespaceManager(object): def __getitem__(self, key): raise NotImplementedError() - + def __setitem__(self, key, value): raise NotImplementedError() - + def set_value(self, key, value, expiretime=None): """Optional set_value() method called by Value. - + Allows an expiretime to be passed, for namespace implementations which can prune their collections using expiretime. - + """ self[key] = value - + def __contains__(self, key): raise NotImplementedError() def __delitem__(self, key): raise NotImplementedError() - + def keys(self): raise NotImplementedError() - + def remove(self): self.do_remove() - + class OpenResourceNamespaceManager(NamespaceManager): """A NamespaceManager where read/write operations require opening/ closing of a resource which is possibly mutexed. - + """ def __init__(self, namespace): NamespaceManager.__init__(self, namespace) @@ -133,13 +133,13 @@ class OpenResourceNamespaceManager(NamespaceManager): except: self.access_lock.release_read_lock() raise - + def release_read_lock(self): try: self.close(checkcount = True) finally: self.access_lock.release_read_lock() - + def acquire_write_lock(self, wait=True): r = self.access_lock.acquire_write_lock(wait) try: @@ -149,7 +149,7 @@ class OpenResourceNamespaceManager(NamespaceManager): except: self.access_lock.release_write_lock() raise - + def release_write_lock(self): try: self.close(checkcount=True) @@ -258,7 +258,7 @@ class Value(object): except KeyError: # guard against un-mutexed backends raising KeyError has_value = False - + if not self.createfunc: raise KeyError(self.key) finally: @@ -349,29 +349,29 @@ class Value(object): class AbstractDictionaryNSManager(NamespaceManager): """A subclassable NamespaceManager that places data in a dictionary. - + Subclasses should provide a "dictionary" attribute or descriptor which returns a dict-like object. The dictionary will store keys that are local to the "namespace" attribute of this manager, so ensure that the dictionary will not be used by any other namespace. e.g.:: - + import collections cached_data = collections.defaultdict(dict) - + class MyDictionaryManager(AbstractDictionaryNSManager): def __init__(self, namespace): AbstractDictionaryNSManager.__init__(self, namespace) self.dictionary = cached_data[self.namespace] - + The above stores data in a global dictionary called "cached_data", which is structured as a dictionary of dictionaries, keyed first on namespace name to a sub-dictionary, then on actual cache key to value. - + """ - + def get_creation_lock(self, key): return NameLock( identifier="memorynamespace/funclock/%s/%s" % (self.namespace, key), @@ -386,19 +386,19 @@ class AbstractDictionaryNSManager(NamespaceManager): def has_key(self, key): return self.dictionary.__contains__(key) - + def __setitem__(self, key, value): self.dictionary[key] = value - + def __delitem__(self, key): del self.dictionary[key] def do_remove(self): self.dictionary.clear() - + def keys(self): return self.dictionary.keys() - + class MemoryNamespaceManager(AbstractDictionaryNSManager): namespaces = util.SyncDict() @@ -411,7 +411,7 @@ class DBMNamespaceManager(OpenResourceNamespaceManager): def __init__(self, namespace, dbmmodule=None, data_dir=None, dbm_dir=None, lock_dir=None, digest_filenames=True, **kwargs): self.digest_filenames = digest_filenames - + if not dbm_dir and not data_dir: raise MissingCacheParameter("data_dir or dbm_dir is required") elif dbm_dir: @@ -419,7 +419,7 @@ class DBMNamespaceManager(OpenResourceNamespaceManager): else: self.dbm_dir = data_dir + "/container_dbm" util.verify_directory(self.dbm_dir) - + if not lock_dir and not data_dir: raise MissingCacheParameter("data_dir or lock_dir is required") elif lock_dir: @@ -437,14 +437,14 @@ class DBMNamespaceManager(OpenResourceNamespaceManager): identifiers=[self.namespace], extension='.dbm', digest_filenames=self.digest_filenames) - + debug("data file %s", self.file) self._checkfile() def get_access_lock(self): return file_synchronizer(identifier=self.namespace, lock_dir=self.lock_dir) - + def get_creation_lock(self, key): return file_synchronizer( identifier = "dbmcontainer/funclock/%s" % self.namespace, @@ -458,19 +458,19 @@ class DBMNamespaceManager(OpenResourceNamespaceManager): for ext in ('db', 'dat', 'pag', 'dir'): if os.access(file + os.extsep + ext, os.F_OK): return True - + return False - + def _checkfile(self): if not self.file_exists(self.file): g = self.dbmmodule.open(self.file, 'c') g.close() - + def get_filenames(self): list = [] if os.access(self.file, os.F_OK): list.append(self.file) - + for ext in ('pag', 'dir', 'db', 'dat'): if os.access(self.file + os.extsep + ext, os.F_OK): list.append(self.file + os.extsep + ext) @@ -488,17 +488,17 @@ class DBMNamespaceManager(OpenResourceNamespaceManager): if self.dbm is not None: debug("closing dbm file %s", self.file) self.dbm.close() - + def do_remove(self): for f in self.get_filenames(): os.remove(f) - + def __getitem__(self, key): return cPickle.loads(self.dbm[key]) def __contains__(self, key): return self.dbm.has_key(key) - + def __setitem__(self, key, value): self.dbm[key] = cPickle.dumps(value) @@ -513,7 +513,7 @@ class FileNamespaceManager(OpenResourceNamespaceManager): def __init__(self, namespace, data_dir=None, file_dir=None, lock_dir=None, digest_filenames=True, **kwargs): self.digest_filenames = digest_filenames - + if not file_dir and not data_dir: raise MissingCacheParameter("data_dir or file_dir is required") elif file_dir: @@ -536,19 +536,19 @@ class FileNamespaceManager(OpenResourceNamespaceManager): extension='.cache', digest_filenames=self.digest_filenames) self.hash = {} - + debug("data file %s", self.file) def get_access_lock(self): return file_synchronizer(identifier=self.namespace, lock_dir=self.lock_dir) - + def get_creation_lock(self, key): return file_synchronizer( identifier = "filecontainer/funclock/%s" % self.namespace, lock_dir = self.lock_dir ) - + def file_exists(self, file): return os.access(file, os.F_OK) @@ -562,7 +562,7 @@ class FileNamespaceManager(OpenResourceNamespaceManager): fh.close() self.flags = flags - + def do_close(self): if self.flags == 'c' or self.flags == 'w': fh = open(self.file, 'wb') @@ -571,7 +571,7 @@ class FileNamespaceManager(OpenResourceNamespaceManager): self.hash = {} self.flags = None - + def do_remove(self): try: os.remove(self.file) @@ -580,13 +580,13 @@ class FileNamespaceManager(OpenResourceNamespaceManager): # but client code has asked for a clear() operation... pass self.hash = {} - + def __getitem__(self, key): return self.hash[key] def __contains__(self, key): return self.hash.has_key(key) - + def __setitem__(self, key, value): self.hash[key] = value @@ -602,7 +602,7 @@ class FileNamespaceManager(OpenResourceNamespaceManager): namespace_classes = {} ContainerContext = dict - + class ContainerMeta(type): def __init__(cls, classname, bases, dict_): namespace_classes[cls] = cls.namespace_class diff --git a/module/lib/beaker/crypto/pbkdf2.py b/module/lib/beaker/crypto/pbkdf2.py index 96dc5fbb2..6c683ef30 100644 --- a/module/lib/beaker/crypto/pbkdf2.py +++ b/module/lib/beaker/crypto/pbkdf2.py @@ -79,7 +79,7 @@ def strxor(a, b): class PBKDF2(object): """PBKDF2.py : PKCS#5 v2.0 Password-Based Key Derivation - + This implementation takes a passphrase and a salt (and optionally an iteration count, a digest module, and a MAC module) and provides a file-like object from which an arbitrarily-sized key can be read. @@ -89,10 +89,10 @@ class PBKDF2(object): The idea behind PBKDF2 is to derive a cryptographic key from a passphrase and a salt. - + PBKDF2 may also be used as a strong salted password hash. The 'crypt' function is provided for that purpose. - + Remember: Keys generated using PBKDF2 are only as strong as the passphrases they are derived from. """ @@ -109,7 +109,7 @@ class PBKDF2(object): """Pseudorandom function. e.g. HMAC-SHA1""" return self.__macmodule(key=key, msg=msg, digestmod=self.__digestmodule).digest() - + def read(self, bytes): """Read the specified number of key bytes.""" if self.closed: @@ -131,7 +131,7 @@ class PBKDF2(object): self.__buf = buf[bytes:] self.__blockNum = i return retval - + def __f(self, i): # i must fit within 32 bits assert (1 <= i <= 0xffffffff) @@ -141,7 +141,7 @@ class PBKDF2(object): U = self.__prf(self.__passphrase, U) result = strxor(result, U) return result - + def hexread(self, octets): """Read the specified number of octets. Return them as hexadecimal. @@ -151,7 +151,7 @@ class PBKDF2(object): def _setup(self, passphrase, salt, iterations, prf): # Sanity checks: - + # passphrase and salt must be str or unicode (in the latter # case, we convert to UTF-8) if isinstance(passphrase, unicode): @@ -168,7 +168,7 @@ class PBKDF2(object): raise TypeError("iterations must be an integer") if iterations < 1: raise ValueError("iterations must be at least 1") - + # prf must be callable if not callable(prf): raise TypeError("prf must be callable") @@ -180,7 +180,7 @@ class PBKDF2(object): self.__blockNum = 0 self.__buf = "" self.closed = False - + def close(self): """Close the stream.""" if not self.closed: @@ -194,13 +194,13 @@ class PBKDF2(object): def crypt(word, salt=None, iterations=None): """PBKDF2-based unix crypt(3) replacement. - + The number of iterations specified in the salt overrides the 'iterations' parameter. The effective hash length is 192 bits. """ - + # Generate a (pseudo-)random salt if the user hasn't provided one. if salt is None: salt = _makesalt() @@ -229,7 +229,7 @@ def crypt(word, salt=None, iterations=None): iterations = converted if not (iterations >= 1): raise ValueError("Invalid salt") - + # Make sure the salt matches the allowed character set allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789./" for ch in salt: @@ -251,7 +251,7 @@ PBKDF2.crypt = staticmethod(crypt) def _makesalt(): """Return a 48-bit pseudorandom salt for crypt(). - + This function is not suitable for generating cryptographic secrets. """ binarysalt = "".join([pack("@H", randint(0, 0xffff)) for i in range(3)]) @@ -260,7 +260,7 @@ def _makesalt(): def test_pbkdf2(): """Module self-test""" from binascii import a2b_hex - + # # Test vectors from RFC 3962 # @@ -284,18 +284,18 @@ def test_pbkdf2(): "c5ec59f1a452f5cc9ad940fea0598ed1") if result != expected: raise RuntimeError("self-test failed") - + # Test 4 result = PBKDF2("X"*65, "pass phrase exceeds block size", 1200).hexread(32) expected = ("9ccad6d468770cd51b10e6a68721be61" "1a8b4d282601db3b36be9246915ec82a") if result != expected: raise RuntimeError("self-test failed") - + # # Other test vectors # - + # Chunked read f = PBKDF2("kickstart", "workbench", 256) result = f.read(17) @@ -306,7 +306,7 @@ def test_pbkdf2(): expected = PBKDF2("kickstart", "workbench", 256).read(40) if result != expected: raise RuntimeError("self-test failed") - + # # crypt() test vectors # @@ -316,7 +316,7 @@ def test_pbkdf2(): expected = '$p5k2$$exec$r1EWMCMk7Rlv3L/RNcFXviDefYa0hlql' if result != expected: raise RuntimeError("self-test failed") - + # crypt 2 result = crypt("gnu", '$p5k2$c$u9HvcT4d$.....') expected = '$p5k2$c$u9HvcT4d$Sd1gwSVCLZYAuqZ25piRnbBEoAesaa/g' @@ -328,7 +328,7 @@ def test_pbkdf2(): expected = "$p5k2$d$tUsch7fU$nqDkaxMDOFBeJsTSfABsyn.PYUXilHwL" if result != expected: raise RuntimeError("self-test failed") - + # crypt 4 (unicode) result = crypt(u'\u0399\u03c9\u03b1\u03bd\u03bd\u03b7\u03c2', '$p5k2$$KosHgqNo$9mjN8gqjt02hDoP0c2J0ABtLIwtot8cQ') diff --git a/module/lib/beaker/crypto/pycrypto.py b/module/lib/beaker/crypto/pycrypto.py index a3eb4d9db..1939555ce 100644 --- a/module/lib/beaker/crypto/pycrypto.py +++ b/module/lib/beaker/crypto/pycrypto.py @@ -9,16 +9,16 @@ try: def aesEncrypt(data, key): cipher = aes.AES(key) return cipher.process(data) - + # magic. aesDecrypt = aesEncrypt - + except ImportError: from Crypto.Cipher import AES def aesEncrypt(data, key): cipher = AES.new(key) - + data = data + (" " * (16 - (len(data) % 16))) return cipher.encrypt(data) diff --git a/module/lib/beaker/crypto/util.py b/module/lib/beaker/crypto/util.py index d97e8ce6f..7f96ac856 100644 --- a/module/lib/beaker/crypto/util.py +++ b/module/lib/beaker/crypto/util.py @@ -6,9 +6,9 @@ try: # Use PyCrypto (if available) from Crypto.Hash import HMAC as hmac, SHA as hmac_sha1 sha1 = hmac_sha1.new - + except ImportError: - + # PyCrypto not available. Use the Python standard library. import hmac diff --git a/module/lib/beaker/ext/database.py b/module/lib/beaker/ext/database.py index 701e6f7d2..7e3db6899 100644 --- a/module/lib/beaker/ext/database.py +++ b/module/lib/beaker/ext/database.py @@ -30,12 +30,12 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): except ImportError: raise InvalidCacheBackendError("Database cache backend requires " "the 'sqlalchemy' library") - + def __init__(self, namespace, url=None, sa_opts=None, optimistic=False, table_name='beaker_cache', data_dir=None, lock_dir=None, **params): """Creates a database namespace manager - + ``url`` SQLAlchemy compliant db url ``sa_opts`` @@ -49,7 +49,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): The table name to use in the database for the cache. """ OpenResourceNamespaceManager.__init__(self, namespace) - + if sa_opts is None: sa_opts = params @@ -59,7 +59,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): self.lock_dir = data_dir + "/container_db_lock" if self.lock_dir: verify_directory(self.lock_dir) - + # Check to see if the table's been created before url = url or sa_opts['sa.url'] table_key = url + table_name @@ -90,7 +90,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): self._is_new = False self.loaded = False self.cache = DatabaseNamespaceManager.tables.get(table_key, make_cache) - + def get_access_lock(self): return null_synchronizer() @@ -104,7 +104,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): if self.loaded: self.flags = flags return - + cache = self.cache result = sa.select([cache.c.data], cache.c.namespace==self.namespace @@ -123,7 +123,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): self._is_new = True self.flags = flags self.loaded = True - + def do_close(self): if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): cache = self.cache @@ -136,12 +136,12 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): cache.update(cache.c.namespace==self.namespace).execute( data=self.hash, accessed=datetime.now()) self.flags = None - + def do_remove(self): cache = self.cache cache.delete(cache.c.namespace==self.namespace).execute() self.hash = {} - + # We can retain the fact that we did a load attempt, but since the # file is gone this will be a new namespace should it be saved. self._is_new = True @@ -151,7 +151,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): def __contains__(self, key): return self.hash.has_key(key) - + def __setitem__(self, key, value): self.hash[key] = value diff --git a/module/lib/beaker/ext/google.py b/module/lib/beaker/ext/google.py index dd8380d7f..b6d617d6b 100644 --- a/module/lib/beaker/ext/google.py +++ b/module/lib/beaker/ext/google.py @@ -23,11 +23,11 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): except ImportError: raise InvalidCacheBackendError("Datastore cache backend requires the " "'google.appengine.ext' library") - + def __init__(self, namespace, table_name='beaker_cache', **params): """Creates a datastore namespace manager""" OpenResourceNamespaceManager.__init__(self, namespace) - + def make_cache(): table_dict = dict(created=db.DateTimeProperty(), accessed=db.DateTimeProperty(), @@ -40,11 +40,11 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): self._is_new = False self.loaded = False self.log_debug = logging.DEBUG >= log.getEffectiveLevel() - + # Google wants namespaces to start with letters, change the namespace # to start with a letter self.namespace = 'p%s' % self.namespace - + def get_access_lock(self): return null_synchronizer() @@ -57,9 +57,9 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): if self.loaded: self.flags = flags return - + item = self.cache.get_by_key_name(self.namespace) - + if not item: self._is_new = True self.hash = {} @@ -74,7 +74,7 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): self._is_new = True self.flags = flags self.loaded = True - + def do_close(self): if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): if self._is_new: @@ -90,12 +90,12 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): item.accessed = datetime.now() item.put() self.flags = None - + def do_remove(self): item = self.cache.get_by_key_name(self.namespace) item.delete() self.hash = {} - + # We can retain the fact that we did a load attempt, but since the # file is gone this will be a new namespace should it be saved. self._is_new = True @@ -105,7 +105,7 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): def __contains__(self, key): return self.hash.has_key(key) - + def __setitem__(self, key, value): self.hash[key] = value @@ -114,7 +114,7 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): def keys(self): return self.hash.keys() - + class GoogleContainer(Container): namespace_class = GoogleNamespaceManager diff --git a/module/lib/beaker/ext/memcached.py b/module/lib/beaker/ext/memcached.py index 96516953f..2f367c36d 100644 --- a/module/lib/beaker/ext/memcached.py +++ b/module/lib/beaker/ext/memcached.py @@ -8,7 +8,7 @@ memcache = None class MemcachedNamespaceManager(NamespaceManager): clients = SyncDict() - + @classmethod def _init_dependencies(cls): global memcache @@ -27,20 +27,20 @@ class MemcachedNamespaceManager(NamespaceManager): except ImportError: raise InvalidCacheBackendError("Memcached cache backend requires either " "the 'memcache' or 'cmemcache' library") - + def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params): NamespaceManager.__init__(self, namespace) - + if not url: raise MissingCacheParameter("url is required") - + if lock_dir: self.lock_dir = lock_dir elif data_dir: self.lock_dir = data_dir + "/container_mcd_lock" if self.lock_dir: verify_directory(self.lock_dir) - + self.mc = MemcachedNamespaceManager.clients.get(url, memcache.Client, url.split(';')) def get_creation_lock(self, key): @@ -68,13 +68,13 @@ class MemcachedNamespaceManager(NamespaceManager): def __setitem__(self, key, value): self.set_value(key, value) - + def __delitem__(self, key): self.mc.delete(self._format_key(key)) def do_remove(self): self.mc.flush_all() - + def keys(self): raise NotImplementedError("Memcache caching does not support iteration of all cache keys") diff --git a/module/lib/beaker/middleware.py b/module/lib/beaker/middleware.py index 7ba88b37d..785d25377 100644 --- a/module/lib/beaker/middleware.py +++ b/module/lib/beaker/middleware.py @@ -16,15 +16,15 @@ from beaker.util import coerce_cache_params, coerce_session_params, \ class CacheMiddleware(object): cache = beaker_cache - + def __init__(self, app, config=None, environ_key='beaker.cache', **kwargs): """Initialize the Cache Middleware - + The Cache middleware will make a Cache instance available every request under the ``environ['beaker.cache']`` key by default. The location in environ can be changed by setting ``environ_key``. - + ``config`` dict All settings should be prefixed by 'cache.'. This method of passing variables is intended for Paste and other @@ -32,11 +32,11 @@ class CacheMiddleware(object): single dictionary. If config contains *no cache. prefixed args*, then *all* of the config options will be used to intialize the Cache objects. - + ``environ_key`` Location where the Cache instance will keyed in the WSGI environ - + ``**kwargs`` All keyword arguments are assumed to be cache settings and will override any settings found in ``config`` @@ -44,26 +44,26 @@ class CacheMiddleware(object): """ self.app = app config = config or {} - + self.options = {} - + # Update the options with the parsed config self.options.update(parse_cache_config_options(config)) - + # Add any options from kwargs, but leave out the defaults this # time self.options.update( parse_cache_config_options(kwargs, include_defaults=False)) - + # Assume all keys are intended for cache if none are prefixed with # 'cache.' if not self.options and config: self.options = config - + self.options.update(kwargs) self.cache_manager = CacheManager(**self.options) self.environ_key = environ_key - + def __call__(self, environ, start_response): if environ.get('paste.registry'): if environ['paste.registry'].reglist: @@ -75,16 +75,16 @@ class CacheMiddleware(object): class SessionMiddleware(object): session = beaker_session - + def __init__(self, wrap_app, config=None, environ_key='beaker.session', **kwargs): """Initialize the Session Middleware - + The Session middleware will make a lazy session instance available every request under the ``environ['beaker.session']`` key by default. The location in environ can be changed by setting ``environ_key``. - + ``config`` dict All settings should be prefixed by 'session.'. This method of passing variables is intended for Paste and other @@ -92,18 +92,18 @@ class SessionMiddleware(object): single dictionary. If config contains *no cache. prefixed args*, then *all* of the config options will be used to intialize the Cache objects. - + ``environ_key`` Location where the Session instance will keyed in the WSGI environ - + ``**kwargs`` All keyword arguments are assumed to be session settings and will override any settings found in ``config`` """ config = config or {} - + # Load up the default params self.options = dict(invalidate_corrupt=True, type=None, data_dir=None, key='beaker.session.id', @@ -120,19 +120,19 @@ class SessionMiddleware(object): warnings.warn('Session options should start with session. ' 'instead of session_.', DeprecationWarning, 2) self.options[key[8:]] = val - + # Coerce and validate session params coerce_session_params(self.options) - + # Assume all keys are intended for cache if none are prefixed with # 'cache.' if not self.options and config: self.options = config - + self.options.update(kwargs) self.wrap_app = wrap_app self.environ_key = environ_key - + def __call__(self, environ, start_response): session = SessionObject(environ, **self.options) if environ.get('paste.registry'): @@ -140,7 +140,7 @@ class SessionMiddleware(object): environ['paste.registry'].register(self.session, session) environ[self.environ_key] = session environ['beaker.get_session'] = self._get_session - + def session_start_response(status, headers, exc_info = None): if session.accessed(): session.persist() @@ -150,7 +150,7 @@ class SessionMiddleware(object): headers.append(('Set-cookie', cookie)) return start_response(status, headers, exc_info) return self.wrap_app(environ, session_start_response) - + def _get_session(self): return Session({}, use_cookies=False, **self.options) diff --git a/module/lib/beaker/session.py b/module/lib/beaker/session.py index 7d465530b..7957be66b 100644 --- a/module/lib/beaker/session.py +++ b/module/lib/beaker/session.py @@ -3,7 +3,7 @@ import os import random import time from datetime import datetime, timedelta - + from beaker.crypto import hmac as HMAC, hmac_sha1 as SHA1, md5 from beaker.util import pickle @@ -22,25 +22,25 @@ class SignedCookie(Cookie.BaseCookie): def __init__(self, secret, input=None): self.secret = secret Cookie.BaseCookie.__init__(self, input) - + def value_decode(self, val): val = val.strip('"') sig = HMAC.new(self.secret, val[40:], SHA1).hexdigest() - + # Avoid timing attacks invalid_bits = 0 input_sig = val[:40] if len(sig) != len(input_sig): return None, val - + for a, b in zip(sig, input_sig): invalid_bits += a != b - + if invalid_bits: return None, val else: return val[40:], val - + def value_encode(self, val): sig = HMAC.new(self.secret, val, SHA1).hexdigest() return str(val), ("%s%s" % (sig, val)) @@ -48,7 +48,7 @@ class SignedCookie(Cookie.BaseCookie): class Session(dict): """Session object that uses container package for storage. - + ``key`` The name the cookie should be set to. ``timeout`` @@ -76,15 +76,15 @@ class Session(dict): self.namespace_class = namespace_class or clsmap[self.type] self.namespace_args = namespace_args - + self.request = request self.data_dir = data_dir self.key = key - + self.timeout = timeout self.use_cookies = use_cookies self.cookie_expires = cookie_expires - + # Default cookie domain/path self._domain = cookie_domain self._path = '/' @@ -93,7 +93,7 @@ class Session(dict): self.secure = secure self.id = id self.accessed_dict = {} - + if self.use_cookies: cookieheader = request.get('cookie', '') if secret: @@ -103,10 +103,10 @@ class Session(dict): self.cookie = SignedCookie(secret, input=None) else: self.cookie = Cookie.SimpleCookie(input=cookieheader) - + if not self.id and self.key in self.cookie: self.id = self.cookie[self.key].value - + self.is_new = self.id is None if self.is_new: self._create_id() @@ -119,7 +119,7 @@ class Session(dict): self.invalidate() else: raise - + def _create_id(self): self.id = md5( md5("%f%s%f%s" % (time.time(), id({}), random.random(), @@ -148,31 +148,31 @@ class Session(dict): expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) self.request['cookie_out'] = self.cookie[self.key].output(header='') self.request['set_cookie'] = False - + def created(self): return self['_creation_time'] created = property(created) - + def _set_domain(self, domain): self['_domain'] = domain self.cookie[self.key]['domain'] = domain self.request['cookie_out'] = self.cookie[self.key].output(header='') self.request['set_cookie'] = True - + def _get_domain(self): return self._domain - + domain = property(_get_domain, _set_domain) - + def _set_path(self, path): self['_path'] = path self.cookie[self.key]['path'] = path self.request['cookie_out'] = self.cookie[self.key].output(header='') self.request['set_cookie'] = True - + def _get_path(self): return self._path - + path = property(_get_path, _set_path) def _delete_cookie(self): @@ -203,7 +203,7 @@ class Session(dict): self.was_invalidated = True self._create_id() self.load() - + def load(self): "Loads the data from this session from persistent storage" self.namespace = self.namespace_class(self.id, @@ -211,7 +211,7 @@ class Session(dict): **self.namespace_args) now = time.time() self.request['set_cookie'] = True - + self.namespace.acquire_read_lock() timed_out = False try: @@ -233,7 +233,7 @@ class Session(dict): '_accessed_time':now } self.is_new = True - + if self.timeout is not None and \ now - session_data['_accessed_time'] > self.timeout: timed_out= True @@ -244,7 +244,7 @@ class Session(dict): self.last_accessed = None else: self.last_accessed = session_data['_accessed_time'] - + # Update the current _accessed_time session_data['_accessed_time'] = now self.update(session_data) @@ -253,34 +253,34 @@ class Session(dict): self.namespace.release_read_lock() if timed_out: self.invalidate() - + def save(self, accessed_only=False): """Saves the data for this session to persistent storage - + If accessed_only is True, then only the original data loaded at the beginning of the request will be saved, with the updated last accessed time. - + """ # Look to see if its a new session that was only accessed # Don't save it under that case if accessed_only and self.is_new: return None - + if not hasattr(self, 'namespace'): self.namespace = self.namespace_class( self.id, data_dir=self.data_dir, digest_filenames=False, **self.namespace_args) - + self.namespace.acquire_write_lock() try: if accessed_only: data = dict(self.accessed_dict.items()) else: data = dict(self.items()) - + # Save the data if not data and 'session' in self.namespace: del self.namespace['session'] @@ -290,20 +290,20 @@ class Session(dict): self.namespace.release_write_lock() if self.is_new: self.request['set_cookie'] = True - + def revert(self): """Revert the session to its original state from its first access in the request""" self.clear() self.update(self.accessed_dict) - + # TODO: I think both these methods should be removed. They're from # the original mod_python code i was ripping off but they really # have no use here. def lock(self): """Locks this session against other processes/threads. This is automatic when load/save is called. - + ***use with caution*** and always with a corresponding 'unlock' inside a "finally:" block, as a stray lock typically cannot be unlocked without shutting down the whole application. @@ -324,10 +324,10 @@ class Session(dict): class CookieSession(Session): """Pure cookie-based session - + Options recognized when using cookie-based sessions are slightly more restricted than general sessions. - + ``key`` The name the cookie should be set to. ``timeout`` @@ -343,16 +343,16 @@ class CookieSession(Session): Domain to use for the cookie. ``secure`` Whether or not the cookie should only be sent over SSL. - + """ def __init__(self, request, key='beaker.session.id', timeout=None, cookie_expires=True, cookie_domain=None, encrypt_key=None, validate_key=None, secure=False, **kwargs): - + if not crypto.has_aes and encrypt_key: raise InvalidCryptoBackendError("No AES library is installed, can't generate " "encrypted cookie-only Session.") - + self.request = request self.key = key self.timeout = timeout @@ -363,24 +363,24 @@ class CookieSession(Session): self.secure = secure self._domain = cookie_domain self._path = '/' - + try: cookieheader = request['cookie'] except KeyError: cookieheader = '' - + if validate_key is None: raise BeakerException("No validate_key specified for Cookie only " "Session.") - + try: self.cookie = SignedCookie(validate_key, input=cookieheader) except Cookie.CookieError: self.cookie = SignedCookie(validate_key, input=None) - + self['_id'] = self._make_id() self.is_new = True - + # If we have a cookie, load it if self.key in self.cookie and self.cookie[self.key].value is not None: self.is_new = False @@ -393,11 +393,11 @@ class CookieSession(Session): self.clear() self.accessed_dict = self.copy() self._create_cookie() - + def created(self): return self['_creation_time'] created = property(created) - + def id(self): return self['_id'] id = property(id) @@ -405,19 +405,19 @@ class CookieSession(Session): def _set_domain(self, domain): self['_domain'] = domain self._domain = domain - + def _get_domain(self): return self._domain - + domain = property(_get_domain, _set_domain) - + def _set_path(self, path): self['_path'] = path self._path = path - + def _get_path(self): return self._path - + path = property(_get_path, _set_path) def _encrypt_data(self): @@ -431,7 +431,7 @@ class CookieSession(Session): else: data = pickle.dumps(self.copy(), 2) return b64encode(data) - + def _decrypt_data(self): """Bas64, decipher, then un-serialize the data for the session dict""" @@ -445,13 +445,13 @@ class CookieSession(Session): else: data = b64decode(self.cookie[self.key].value) return pickle.loads(data) - + def _make_id(self): return md5(md5( "%f%s%f%s" % (time.time(), id({}), random.random(), getpid()) ).hexdigest() ).hexdigest() - + def save(self, accessed_only=False): """Saves the data for this session to persistent storage""" if accessed_only and self.is_new: @@ -460,19 +460,19 @@ class CookieSession(Session): self.clear() self.update(self.accessed_dict) self._create_cookie() - + def expire(self): """Delete the 'expires' attribute on this Session, if any.""" - + self.pop('_expires', None) - + def _create_cookie(self): if '_creation_time' not in self: self['_creation_time'] = time.time() if '_id' not in self: self['_id'] = self._make_id() self['_accessed_time'] = time.time() - + if self.cookie_expires is not True: if self.cookie_expires is False: expires = datetime.fromtimestamp( 0x7FFFFFFF ) @@ -492,7 +492,7 @@ class CookieSession(Session): val = self._encrypt_data() if len(val) > 4064: raise BeakerException("Cookie value is too long to store") - + self.cookie[self.key] = val if '_domain' in self: self.cookie[self.key]['domain'] = self['_domain'] @@ -500,21 +500,21 @@ class CookieSession(Session): self.cookie[self.key]['domain'] = self._domain if self.secure: self.cookie[self.key]['secure'] = True - + self.cookie[self.key]['path'] = self.get('_path', '/') - + if expires: self.cookie[self.key]['expires'] = \ expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) self.request['cookie_out'] = self.cookie[self.key].output(header='') self.request['set_cookie'] = True - + def delete(self): """Delete the cookie, and clear the session""" # Send a delete cookie request self._delete_cookie() self.clear() - + def invalidate(self): """Clear the contents and start a new session""" self.delete() @@ -523,19 +523,19 @@ class CookieSession(Session): class SessionObject(object): """Session proxy/lazy creator - + This object proxies access to the actual session object, so that in the case that the session hasn't been used before, it will be setup. This avoid creating and loading the session from persistent storage unless its actually used during the request. - + """ def __init__(self, environ, **params): self.__dict__['_params'] = params self.__dict__['_environ'] = environ self.__dict__['_sess'] = None self.__dict__['_headers'] = [] - + def _session(self): """Lazy initial creation of session object""" if self.__dict__['_sess'] is None: @@ -549,35 +549,35 @@ class SessionObject(object): self.__dict__['_sess'] = Session(req, use_cookies=True, **params) return self.__dict__['_sess'] - + def __getattr__(self, attr): return getattr(self._session(), attr) - + def __setattr__(self, attr, value): setattr(self._session(), attr, value) - + def __delattr__(self, name): self._session().__delattr__(name) - + def __getitem__(self, key): return self._session()[key] - + def __setitem__(self, key, value): self._session()[key] = value - + def __delitem__(self, key): self._session().__delitem__(key) - + def __repr__(self): return self._session().__repr__() - + def __iter__(self): """Only works for proxying to a dict""" return iter(self._session().keys()) - + def __contains__(self, key): return self._session().has_key(key) - + def get_by_id(self, id): """Loads a session given a session ID""" params = self.__dict__['_params'] @@ -585,22 +585,22 @@ class SessionObject(object): if session.is_new: return None return session - + def save(self): self.__dict__['_dirty'] = True - + def delete(self): self.__dict__['_dirty'] = True self._session().delete() - + def persist(self): """Persist the session to the storage - + If its set to autosave, then the entire session will be saved regardless of if save() has been called. Otherwise, just the accessed time will be updated if save() was not called, or the session will be saved if save() was called. - + """ if self.__dict__['_params'].get('auto'): self._session().save() @@ -609,10 +609,10 @@ class SessionObject(object): self._session().save() else: self._session().save(accessed_only=True) - + def dirty(self): return self.__dict__.get('_dirty', False) - + def accessed(self): """Returns whether or not the session has been accessed""" return self.__dict__['_sess'] is not None diff --git a/module/lib/beaker/synchronization.py b/module/lib/beaker/synchronization.py index 761303707..6616ca78b 100644 --- a/module/lib/beaker/synchronization.py +++ b/module/lib/beaker/synchronization.py @@ -36,7 +36,7 @@ __all__ = ["file_synchronizer", "mutex_synchronizer", "null_synchronizer", class NameLock(object): """a proxy for an RLock object that is stored in a name based registry. - + Multiple threads can get a reference to the same RLock based on the name alone, and synchronize operations related to that name. @@ -115,7 +115,7 @@ class SynchronizerImpl(object): else: return self._state.get() state = property(state) - + def release_read_lock(self): state = self.state @@ -123,19 +123,19 @@ class SynchronizerImpl(object): raise LockError("lock is in writing state") if not state.reading: raise LockError("lock is not in reading state") - + if state.reentrantcount == 1: self.do_release_read_lock() state.reading = False state.reentrantcount -= 1 - + def acquire_read_lock(self, wait = True): state = self.state if state.writing: raise LockError("lock is in writing state") - + if state.reentrantcount == 0: x = self.do_acquire_read_lock(wait) if (wait or x): @@ -145,7 +145,7 @@ class SynchronizerImpl(object): elif state.reading: state.reentrantcount += 1 return True - + def release_write_lock(self): state = self.state @@ -159,15 +159,15 @@ class SynchronizerImpl(object): state.writing = False state.reentrantcount -= 1 - + release = release_write_lock - + def acquire_write_lock(self, wait = True): state = self.state if state.reading: raise LockError("lock is in reading state") - + if state.reentrantcount == 0: x = self.do_acquire_write_lock(wait) if (wait or x): @@ -182,13 +182,13 @@ class SynchronizerImpl(object): def do_release_read_lock(self): raise NotImplementedError() - + def do_acquire_read_lock(self): raise NotImplementedError() - + def do_release_write_lock(self): raise NotImplementedError() - + def do_acquire_write_lock(self): raise NotImplementedError() @@ -198,18 +198,18 @@ class FileSynchronizer(SynchronizerImpl): Adapted for Python/multithreads from Apache::Session::Lock::File, http://search.cpan.org/src/CWEST/Apache-Session-1.81/Session/Lock/File.pm - + This module does not unlink temporary files, because it interferes with proper locking. This can cause problems on certain systems (Linux) whose file systems (ext2) do not perform well with lots of files in one directory. To prevent this you should use a script to clean out old files from your lock directory. - + """ def __init__(self, identifier, lock_dir): super(FileSynchronizer, self).__init__() self._filedescriptor = util.ThreadLocal() - + if lock_dir is None: lock_dir = tempfile.gettempdir() else: @@ -224,14 +224,14 @@ class FileSynchronizer(SynchronizerImpl): def _filedesc(self): return self._filedescriptor.get() _filedesc = property(_filedesc) - + def _open(self, mode): filedescriptor = self._filedesc if filedescriptor is None: filedescriptor = os.open(self.filename, mode) self._filedescriptor.put(filedescriptor) return filedescriptor - + def do_acquire_read_lock(self, wait): filedescriptor = self._open(os.O_CREAT | os.O_RDONLY) if not wait: @@ -259,13 +259,13 @@ class FileSynchronizer(SynchronizerImpl): else: fcntl.flock(filedescriptor, fcntl.LOCK_EX) return True - + def do_release_read_lock(self): self._release_all_locks() - + def do_release_write_lock(self): self._release_all_locks() - + def _release_all_locks(self): filedescriptor = self._filedesc if filedescriptor is not None: @@ -276,7 +276,7 @@ class FileSynchronizer(SynchronizerImpl): class ConditionSynchronizer(SynchronizerImpl): """a synchronizer using a Condition.""" - + def __init__(self, identifier): super(ConditionSynchronizer, self).__init__() @@ -308,12 +308,12 @@ class ConditionSynchronizer(SynchronizerImpl): if not wait: return True - + def do_release_read_lock(self): self.condition.acquire() try: self.async -= 1 - + # check if we are the last asynchronous reader thread # out the door. if self.async == 0: @@ -326,13 +326,13 @@ class ConditionSynchronizer(SynchronizerImpl): "release_read_locks called") finally: self.condition.release() - + def do_acquire_write_lock(self, wait = True): self.condition.acquire() try: # here, we are not a synchronous reader, and after returning, # assuming waiting or immediate availability, we will be. - + if wait: # if another sync is working, wait while self.current_sync_operation is not None: @@ -342,7 +342,7 @@ class ConditionSynchronizer(SynchronizerImpl): # we dont want to wait, so forget it if self.current_sync_operation is not None: return False - + # establish ourselves as the current sync # this indicates to other read/write operations # that they should wait until this is None again @@ -359,7 +359,7 @@ class ConditionSynchronizer(SynchronizerImpl): return False finally: self.condition.release() - + if not wait: return True diff --git a/module/lib/beaker/util.py b/module/lib/beaker/util.py index 04c9617c5..21a467e42 100644 --- a/module/lib/beaker/util.py +++ b/module/lib/beaker/util.py @@ -45,7 +45,6 @@ def verify_directory(dir): if tries > 5: raise - def deprecated(message): def wrapper(fn): def deprecated_method(*args, **kargs): @@ -56,7 +55,7 @@ def deprecated(message): deprecated_method.__doc__ = "%s\n\n%s" % (message, fn.__doc__) return deprecated_method return wrapper - + class ThreadLocal(object): """stores a value on a per-thread basis""" @@ -64,25 +63,25 @@ class ThreadLocal(object): def __init__(self): self._tlocal = _tlocal() - + def put(self, value): self._tlocal.value = value - + def has(self): return hasattr(self._tlocal, 'value') - + def get(self, default=None): return getattr(self._tlocal, 'value', default) - + def remove(self): del self._tlocal.value - + class SyncDict(object): """ An efficient/threadsafe singleton map algorithm, a.k.a. "get a value based on this key, and create if not found or not valid" paradigm: - + exists && isvalid ? get : create Designed to work with weakref dictionaries to expect items @@ -96,7 +95,7 @@ class SyncDict(object): def __init__(self): self.mutex = _thread.allocate_lock() self.dict = {} - + def get(self, key, createfunc, *args, **kwargs): try: if self.has_key(key): @@ -125,7 +124,7 @@ class SyncDict(object): def has_key(self, key): return self.dict.has_key(key) - + def __contains__(self, key): return self.dict.__contains__(key) def __getitem__(self, key): @@ -146,30 +145,30 @@ class WeakValuedRegistry(SyncDict): sha1 = None def encoded_path(root, identifiers, extension = ".enc", depth = 3, digest_filenames=True): - + """Generate a unique file-accessible path from the given list of identifiers starting at the given root directory.""" ident = "_".join(identifiers) - + global sha1 if sha1 is None: from beaker.crypto import sha1 - + if digest_filenames: if py3k: ident = sha1(ident.encode('utf-8')).hexdigest() else: ident = sha1(ident).hexdigest() - + ident = os.path.basename(ident) tokens = [] for d in range(1, depth): tokens.append(ident[0:d]) - + dir = os.path.join(root, *tokens) verify_directory(dir) - + return os.path.join(dir, ident + extension) @@ -251,7 +250,7 @@ def coerce_cache_params(params): def parse_cache_config_options(config, include_defaults=True): """Parse configuration options and validate for use with the CacheManager""" - + # Load default cache options if include_defaults: options= dict(type='memory', data_dir=None, expire=None, @@ -264,11 +263,11 @@ def parse_cache_config_options(config, include_defaults=True): if key.startswith('cache.'): options[key[6:]] = val coerce_cache_params(options) - + # Set cache to enabled if not turned off if 'enabled' not in options: options['enabled'] = True - + # Configure region dict if regions are available regions = options.pop('regions', None) if regions: @@ -295,7 +294,7 @@ def func_namespace(func): if hasattr(func, 'im_func'): kls = func.im_class func = func.im_func - + if kls: return '%s.%s' % (kls.__module__, kls.__name__) else: diff --git a/module/lib/bottle.py b/module/lib/bottle.py index 2c243278e..f4e93de9d 100644 --- a/module/lib/bottle.py +++ b/module/lib/bottle.py @@ -298,13 +298,13 @@ class Router(object): def path_filter(self, conf): return r'.*?', None, None - + def add_filter(self, name, func): ''' Add a filter. The provided function is called with the configuration string as parameter and must return a (regexp, to_python, to_url) tuple. The first element is a string, the last two are callables or None. ''' self.filters[name] = func - + def parse_rule(self, rule): ''' Parses a rule into a (name, filter, conf) token stream. If mode is None, name contains a static rule part. ''' @@ -2422,10 +2422,10 @@ class FileCheckerThread(threading.Thread): thread.interrupt_main() break time.sleep(self.interval) - + def __enter__(self): self.start() - + def __exit__(self, exc_type, exc_val, exc_tb): if not self.status: self.status = 'exit' # silent exit self.join() diff --git a/module/lib/feedparser.py b/module/lib/feedparser.py index a746ed8f5..32f9d2dd7 100644 --- a/module/lib/feedparser.py +++ b/module/lib/feedparser.py @@ -89,7 +89,7 @@ try: except (NameError, AttributeError): import string _maketrans = string.maketrans - + # base64 support for Atom feeds that contain embedded binary data try: import base64, binascii @@ -334,7 +334,7 @@ class FeedParserDict(UserDict): if not self.has_key(key): self[key] = value return self[key] - + def has_key(self, key): try: return hasattr(self, key) or UserDict.__contains__(self, key) @@ -343,7 +343,7 @@ class FeedParserDict(UserDict): # This alias prevents the 2to3 tool from changing the semantics of the # __contains__ function below and exhausting the maximum recursion depth __has_key = has_key - + def __getattr__(self, key): try: return self.__dict__[key] @@ -451,7 +451,7 @@ class _FeedParserMixin: 'http://purl.org/atom/ns#': '', 'http://www.w3.org/2005/Atom': '', 'http://purl.org/rss/1.0/modules/rss091#': '', - + 'http://webns.net/mvcb/': 'admin', 'http://purl.org/rss/1.0/modules/aggregation/': 'ag', 'http://purl.org/rss/1.0/modules/annotate/': 'annotate', @@ -508,7 +508,7 @@ class _FeedParserMixin: can_contain_relative_uris = ['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description'] can_contain_dangerous_markup = ['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description'] html_types = ['text/html', 'application/xhtml+xml'] - + def __init__(self, baseuri=None, baselang=None, encoding='utf-8'): if _debug: sys.stderr.write('initializing FeedParser\n') if not self._matchnamespaces: @@ -554,7 +554,7 @@ class _FeedParserMixin: # strict xml parsers do -- account for this difference if isinstance(self, _LooseFeedParser): attrs = [(k, v.replace('&', '&')) for k, v in attrs] - + # track xml:base and xml:lang attrsD = dict(attrs) baseuri = attrsD.get('xml:base', attrsD.get('base')) or self.baseuri @@ -582,7 +582,7 @@ class _FeedParserMixin: self.lang = lang self.basestack.append(self.baseuri) self.langstack.append(lang) - + # track namespaces for prefix, uri in attrs: if prefix.startswith('xmlns:'): @@ -620,7 +620,7 @@ class _FeedParserMixin: self.intextinput = 0 if (not prefix) and tag not in ('title', 'link', 'description', 'url', 'href', 'width', 'height'): self.inimage = 0 - + # call special handler (if defined) or default handler methodname = '_start_' + prefix + suffix try: @@ -754,7 +754,7 @@ class _FeedParserMixin: elif contentType == 'xhtml': contentType = 'application/xhtml+xml' return contentType - + def trackNamespace(self, prefix, uri): loweruri = uri.lower() if (prefix, loweruri) == (None, 'http://my.netscape.com/rdf/simple/0.9/') and not self.version: @@ -775,7 +775,7 @@ class _FeedParserMixin: def resolveURI(self, uri): return _urljoin(self.baseuri or '', uri) - + def decodeEntities(self, element, data): return data @@ -788,7 +788,7 @@ class _FeedParserMixin: def pop(self, element, stripWhitespace=1): if not self.elementstack: return if self.elementstack[-1][0] != element: return - + element, expectingText, pieces = self.elementstack.pop() if self.version == 'atom10' and self.contentparams.get('type','text') == 'application/xhtml+xml': @@ -833,11 +833,11 @@ class _FeedParserMixin: # In Python 3, base64 takes and outputs bytes, not str # This may not be the most correct way to accomplish this output = _base64decode(output.encode('utf-8')).decode('utf-8') - + # resolve relative URIs if (element in self.can_be_relative_uri) and output: output = self.resolveURI(output) - + # decode entities within embedded markup if not self.contentparams.get('base64', 0): output = self.decodeEntities(element, output) @@ -860,7 +860,7 @@ class _FeedParserMixin: if is_htmlish and RESOLVE_RELATIVE_URIS: if element in self.can_contain_relative_uris: output = _resolveRelativeURIs(output, self.baseuri, self.encoding, self.contentparams.get('type', 'text/html')) - + # parse microformats # (must do this before sanitizing because some microformats # rely on elements that we sanitize) @@ -876,7 +876,7 @@ class _FeedParserMixin: vcard = mfresults.get('vcard') if vcard: self._getContext()['vcard'] = vcard - + # sanitize embedded markup if is_htmlish and SANITIZE_HTML: if element in self.can_contain_dangerous_markup: @@ -906,7 +906,7 @@ class _FeedParserMixin: if element == 'title' and self.hasTitle: return output - + # store output in appropriate place(s) if self.inentry and not self.insource: if element == 'content': @@ -962,7 +962,7 @@ class _FeedParserMixin: self.incontent -= 1 self.contentparams.clear() return value - + # a number of elements in a number of RSS variants are nominally plain # text, but this is routinely ignored. This is an attempt to detect # the most common cases. As false positives often result in silent @@ -993,7 +993,7 @@ class _FeedParserMixin: prefix = self.namespacemap.get(prefix, prefix) name = prefix + ':' + suffix return name - + def _getAttribute(self, attrsD, name): return attrsD.get(self._mapToStandardPrefix(name)) @@ -1021,7 +1021,7 @@ class _FeedParserMixin: pass attrsD['href'] = href return attrsD - + def _save(self, key, value, overwrite=False): context = self._getContext() if overwrite: @@ -1046,7 +1046,7 @@ class _FeedParserMixin: self.version = 'rss20' else: self.version = 'rss' - + def _start_dlhottitles(self, attrsD): self.version = 'hotrss' @@ -1064,7 +1064,7 @@ class _FeedParserMixin: self._start_link({}) self.elementstack[-1][-1] = attrsD['href'] self._end_link() - + def _start_feed(self, attrsD): self.infeed = 1 versionmap = {'0.1': 'atom01', @@ -1081,7 +1081,7 @@ class _FeedParserMixin: def _end_channel(self): self.infeed = 0 _end_feed = _end_channel - + def _start_image(self, attrsD): context = self._getContext() if not self.inentry: @@ -1089,7 +1089,7 @@ class _FeedParserMixin: self.inimage = 1 self.hasTitle = 0 self.push('image', 0) - + def _end_image(self): self.pop('image') self.inimage = 0 @@ -1101,7 +1101,7 @@ class _FeedParserMixin: self.hasTitle = 0 self.push('textinput', 0) _start_textInput = _start_textinput - + def _end_textinput(self): self.pop('textinput') self.intextinput = 0 @@ -1301,7 +1301,7 @@ class _FeedParserMixin: self.popContent('subtitle') _end_tagline = _end_subtitle _end_itunes_subtitle = _end_subtitle - + def _start_rights(self, attrsD): self.pushContent('rights', attrsD, 'text/plain', 1) _start_dc_rights = _start_rights @@ -1399,7 +1399,7 @@ class _FeedParserMixin: attrsD['rel']='license' if value: attrsD['href']=value context.setdefault('links', []).append(attrsD) - + def _start_creativecommons_license(self, attrsD): self.push('license', 1) _start_creativeCommons_license = _start_creativecommons_license @@ -1420,7 +1420,7 @@ class _FeedParserMixin: value = FeedParserDict({'relationships': relationships, 'href': href, 'name': name}) if value not in xfn: xfn.append(value) - + def _addTag(self, term, scheme, label): context = self._getContext() tags = context.setdefault('tags', []) @@ -1438,7 +1438,7 @@ class _FeedParserMixin: self.push('category', 1) _start_dc_subject = _start_category _start_keywords = _start_category - + def _start_media_category(self, attrsD): attrsD.setdefault('scheme', 'http://search.yahoo.com/mrss/category_schema') self._start_category(attrsD) @@ -1446,11 +1446,11 @@ class _FeedParserMixin: def _end_itunes_keywords(self): for term in self.pop('itunes_keywords').split(): self._addTag(term, 'http://www.itunes.com/', None) - + def _start_itunes_category(self, attrsD): self._addTag(attrsD.get('text'), 'http://www.itunes.com/', None) self.push('category', 1) - + def _end_category(self): value = self.pop('category') if not value: return @@ -1467,7 +1467,7 @@ class _FeedParserMixin: def _start_cloud(self, attrsD): self._getContext()['cloud'] = FeedParserDict(attrsD) - + def _start_link(self, attrsD): attrsD.setdefault('rel', 'alternate') if attrsD['rel'] == 'self': @@ -1568,7 +1568,7 @@ class _FeedParserMixin: context = self._getContext() if context.has_key('generator_detail'): context['generator_detail']['name'] = value - + def _start_admin_generatoragent(self, attrsD): self.push('generator', 1) value = self._getAttribute(attrsD, 'rdf:resource') @@ -1583,7 +1583,7 @@ class _FeedParserMixin: if value: self.elementstack[-1][2].append(value) self.pop('errorreportsto') - + def _start_summary(self, attrsD): context = self._getContext() if context.has_key('summary'): @@ -1601,13 +1601,13 @@ class _FeedParserMixin: self.popContent(self._summaryKey or 'summary') self._summaryKey = None _end_itunes_summary = _end_summary - + def _start_enclosure(self, attrsD): attrsD = self._itsAnHrefDamnIt(attrsD) context = self._getContext() attrsD['rel']='enclosure' context.setdefault('links', []).append(FeedParserDict(attrsD)) - + def _start_source(self, attrsD): if 'url' in attrsD: # This means that we're processing a source element from an RSS 2.0 feed @@ -1659,7 +1659,7 @@ class _FeedParserMixin: if attrsD.get('href'): self._getContext()['image'] = FeedParserDict({'href': attrsD.get('href')}) _start_itunes_link = _start_itunes_image - + def _end_itunes_block(self): value = self.pop('itunes_block', 0) self._getContext()['itunes_block'] = (value == 'yes') and 1 or 0 @@ -1718,12 +1718,12 @@ if _XML_AVAILABLE: self.bozo = 0 self.exc = None self.decls = {} - + def startPrefixMapping(self, prefix, uri): self.trackNamespace(prefix, uri) if uri == 'http://www.w3.org/1999/xlink': self.decls['xmlns:'+prefix] = uri - + def startElementNS(self, name, qname, attrs): namespace, localname = name lowernamespace = str(namespace or '').lower() @@ -1910,7 +1910,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:]) else: self.pieces.append('&#%(ref)s;' % locals()) - + def handle_entityref(self, ref): # called for each entity reference, e.g. for '©', ref will be 'copy' # Reconstruct the original entity reference. @@ -1925,12 +1925,12 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): # Store the original text verbatim. if _debug: sys.stderr.write('_BaseHTMLProcessor, handle_data, text=%s\n' % text) self.pieces.append(text) - + def handle_comment(self, text): # called for each HTML comment, e.g. <!-- insert Javascript code here --> # Reconstruct the original comment. self.pieces.append('<!--%(text)s-->' % locals()) - + def handle_pi(self, text): # called for each processing instruction, e.g. <?instruction> # Reconstruct original processing instruction. @@ -1942,7 +1942,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): # "http://www.w3.org/TR/html4/loose.dtd"> # Reconstruct original DOCTYPE self.pieces.append('<!%(text)s>' % locals()) - + _new_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9:]*\s*').match def _scan_name(self, i, declstartpos): rawdata = self.rawdata @@ -1998,7 +1998,7 @@ class _LooseFeedParser(_FeedParserMixin, _BaseHTMLProcessor): data = data.replace('"', '"') data = data.replace(''', "'") return data - + def strattrs(self, attrs): return ''.join([' %s="%s"' % (n,v.replace('"','"')) for n,v in attrs]) @@ -2022,12 +2022,12 @@ class _MicroformatsParser: self.enclosures = [] self.xfn = [] self.vcard = None - + def vcardEscape(self, s): if type(s) in (type(''), type(u'')): s = s.replace(',', '\\,').replace(';', '\\;').replace('\n', '\\n') return s - + def vcardFold(self, s): s = re.sub(';+$', '', s) sFolded = '' @@ -2043,14 +2043,14 @@ class _MicroformatsParser: def normalize(self, s): return re.sub(r'\s+', ' ', s).strip() - + def unique(self, aList): results = [] for element in aList: if element not in results: results.append(element) return results - + def toISO8601(self, dt): return time.strftime('%Y-%m-%dT%H:%M:%SZ', dt) @@ -2140,21 +2140,21 @@ class _MicroformatsParser: def findVCards(self, elmRoot, bAgentParsing=0): sVCards = '' - + if not bAgentParsing: arCards = self.getPropertyValue(elmRoot, 'vcard', bAllowMultiple=1) else: arCards = [elmRoot] - + for elmCard in arCards: arLines = [] - + def processSingleString(sProperty): sValue = self.getPropertyValue(elmCard, sProperty, self.STRING, bAutoEscape=1).decode(self.encoding) if sValue: arLines.append(self.vcardFold(sProperty.upper() + ':' + sValue)) return sValue or u'' - + def processSingleURI(sProperty): sValue = self.getPropertyValue(elmCard, sProperty, self.URI) if sValue: @@ -2177,7 +2177,7 @@ class _MicroformatsParser: if sContentType: sContentType = ';TYPE=' + sContentType.upper() arLines.append(self.vcardFold(sProperty.upper() + sEncoding + sContentType + sValueKey + ':' + sValue)) - + def processTypeValue(sProperty, arDefaultType, arForceType=None): arResults = self.getPropertyValue(elmCard, sProperty, bAllowMultiple=1) for elmResult in arResults: @@ -2189,7 +2189,7 @@ class _MicroformatsParser: sValue = self.getPropertyValue(elmResult, 'value', self.EMAIL, 0) if sValue: arLines.append(self.vcardFold(sProperty.upper() + ';TYPE=' + ','.join(arType) + ':' + sValue)) - + # AGENT # must do this before all other properties because it is destructive # (removes nested class="vcard" nodes so they don't interfere with @@ -2208,10 +2208,10 @@ class _MicroformatsParser: sAgentValue = self.getPropertyValue(elmAgent, 'value', self.URI, bAutoEscape=1); if sAgentValue: arLines.append(self.vcardFold('AGENT;VALUE=uri:' + sAgentValue)) - + # FN (full name) sFN = processSingleString('fn') - + # N (name) elmName = self.getPropertyValue(elmCard, 'n') if elmName: @@ -2237,25 +2237,25 @@ class _MicroformatsParser: arLines.append(self.vcardFold('N:' + arNames[0] + ';' + arNames[1])) else: arLines.append(self.vcardFold('N:' + arNames[1] + ';' + arNames[0])) - + # SORT-STRING sSortString = self.getPropertyValue(elmCard, 'sort-string', self.STRING, bAutoEscape=1) if sSortString: arLines.append(self.vcardFold('SORT-STRING:' + sSortString)) - + # NICKNAME arNickname = self.getPropertyValue(elmCard, 'nickname', self.STRING, 1, 1) if arNickname: arLines.append(self.vcardFold('NICKNAME:' + ','.join(arNickname))) - + # PHOTO processSingleURI('photo') - + # BDAY dtBday = self.getPropertyValue(elmCard, 'bday', self.DATE) if dtBday: arLines.append(self.vcardFold('BDAY:' + self.toISO8601(dtBday))) - + # ADR (address) arAdr = self.getPropertyValue(elmCard, 'adr', bAllowMultiple=1) for elmAdr in arAdr: @@ -2277,38 +2277,38 @@ class _MicroformatsParser: sRegion + ';' + sPostalCode + ';' + sCountryName)) - + # LABEL processTypeValue('label', ['intl','postal','parcel','work']) - + # TEL (phone number) processTypeValue('tel', ['voice']) - + # EMAIL processTypeValue('email', ['internet'], ['internet']) - + # MAILER processSingleString('mailer') - + # TZ (timezone) processSingleString('tz') - + # GEO (geographical information) elmGeo = self.getPropertyValue(elmCard, 'geo') if elmGeo: sLatitude = self.getPropertyValue(elmGeo, 'latitude', self.STRING, 0, 1) sLongitude = self.getPropertyValue(elmGeo, 'longitude', self.STRING, 0, 1) arLines.append(self.vcardFold('GEO:' + sLatitude + ';' + sLongitude)) - + # TITLE processSingleString('title') - + # ROLE processSingleString('role') # LOGO processSingleURI('logo') - + # ORG (organization) elmOrg = self.getPropertyValue(elmCard, 'org') if elmOrg: @@ -2322,39 +2322,39 @@ class _MicroformatsParser: else: arOrganizationUnit = self.getPropertyValue(elmOrg, 'organization-unit', self.STRING, 1, 1) arLines.append(self.vcardFold('ORG:' + sOrganizationName + ';' + ';'.join(arOrganizationUnit))) - + # CATEGORY arCategory = self.getPropertyValue(elmCard, 'category', self.STRING, 1, 1) + self.getPropertyValue(elmCard, 'categories', self.STRING, 1, 1) if arCategory: arLines.append(self.vcardFold('CATEGORIES:' + ','.join(arCategory))) - + # NOTE processSingleString('note') - + # REV processSingleString('rev') - + # SOUND processSingleURI('sound') - + # UID processSingleString('uid') - + # URL processSingleURI('url') - + # CLASS processSingleString('class') - + # KEY processSingleURI('key') - + if arLines: arLines = [u'BEGIN:vCard',u'VERSION:3.0'] + arLines + [u'END:vCard'] sVCards += u'\n'.join(arLines) + u'\n' - + return sVCards.strip() - + def isProbablyDownloadable(self, elm): attrsD = elm.attrMap if not attrsD.has_key('href'): return 0 @@ -2453,7 +2453,7 @@ class _RelativeURIResolver(_BaseHTMLProcessor): def resolveURI(self, uri): return _makeSafeAbsoluteURI(_urljoin(self.baseuri, uri.strip())) - + def unknown_starttag(self, tag, attrs): if _debug: sys.stderr.write('tag: [%s] with attributes: [%s]\n' % (tag, str(attrs))) @@ -2612,7 +2612,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): self.unacceptablestack = 0 self.mathmlOK = 0 self.svgOK = 0 - + def unknown_starttag(self, tag, attrs): acceptable_attributes = self.acceptable_attributes keymap = {} @@ -2671,7 +2671,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): clean_value = self.sanitize_style(value) if clean_value: clean_attrs.append((key,clean_value)) _BaseHTMLProcessor.unknown_starttag(self, tag, clean_attrs) - + def unknown_endtag(self, tag): if not tag in self.acceptable_elements: if tag in self.unacceptable_elements_with_end_tag: @@ -2791,7 +2791,7 @@ class _FeedURLHandler(urllib2.HTTPDigestAuthHandler, urllib2.HTTPRedirectHandler http_error_300 = http_error_302 http_error_303 = http_error_302 http_error_307 = http_error_302 - + def http_error_401(self, req, fp, code, msg, headers): # Check if # - server requires digest auth, AND @@ -2890,7 +2890,7 @@ def _open_resource(url_file_stream_or_string, etag, modified, agent, referrer, h return opener.open(request) finally: opener.close() # JohnD - + # try to open with native open function (if url_file_stream_or_string is a filename) try: return open(url_file_stream_or_string, 'rb') @@ -2942,7 +2942,7 @@ _date_handlers = [] def registerDateHandler(func): '''Register a date handler function (takes string, returns 9-tuple date in GMT)''' _date_handlers.insert(0, func) - + # ISO-8601 date parsing routines written by Fazal Majid. # The ISO 8601 standard is very convoluted and irregular - a full ISO 8601 # parser is beyond the scope of feedparser and would be a worthwhile addition @@ -3055,7 +3055,7 @@ def _parse_date_iso8601(dateString): # Many implementations have bugs, but we'll pretend they don't. return time.localtime(time.mktime(tuple(tm))) registerDateHandler(_parse_date_iso8601) - + # 8-bit date handling routines written by ytrewq1. _korean_year = u'\ub144' # b3e2 in euc-kr _korean_month = u'\uc6d4' # bff9 in euc-kr @@ -3374,7 +3374,7 @@ def _getCharacterEncoding(http_headers, xml_data): http_headers is a dictionary xml_data is a raw string (not Unicode) - + This is so much trickier than it sounds, it's not even funny. According to RFC 3023 ('XML Media Types'), if the HTTP Content-Type is application/xml, application/*+xml, @@ -3393,12 +3393,12 @@ def _getCharacterEncoding(http_headers, xml_data): served with a Content-Type of text/* and no charset parameter must be treated as us-ascii. (We now do this.) And also that it must always be flagged as non-well-formed. (We now do this too.) - + If Content-Type is unspecified (input was local file or non-HTTP source) or unrecognized (server just got it totally wrong), then go by the encoding given in the XML prefix of the document and default to 'iso-8859-1' as per the HTTP specification (RFC 2616). - + Then, assuming we didn't find a character encoding in the HTTP headers (and the HTTP Content-type allowed us to look in the body), we need to sniff the first few bytes of the XML data and try to determine @@ -3508,7 +3508,7 @@ def _getCharacterEncoding(http_headers, xml_data): if true_encoding.lower() == 'gb2312': true_encoding = 'gb18030' return true_encoding, http_encoding, xml_encoding, sniffed_xml_encoding, acceptable_content_type - + def _toUTF8(data, encoding): '''Changes an XML data stream on the fly to specify a new encoding @@ -3571,7 +3571,7 @@ def _stripDoctype(data): start = re.search(_s2bytes('<\w'), data) start = start and start.start() or -1 head,data = data[:start+1], data[start+1:] - + entity_pattern = re.compile(_s2bytes(r'^\s*<!ENTITY([^>]*?)>'), re.MULTILINE) entity_results=entity_pattern.findall(head) head = entity_pattern.sub(_s2bytes(''), head) @@ -3593,10 +3593,10 @@ def _stripDoctype(data): data = doctype_pattern.sub(replacement, head) + data return version, data, dict(replacement and [(k.decode('utf-8'), v.decode('utf-8')) for k, v in safe_pattern.findall(replacement)]) - + def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, referrer=None, handlers=[], request_headers={}, response_headers={}): '''Parse a feed from a URL, file, stream, or string. - + request_headers, if given, is a dict from http header name to value to add to the request; this overrides internally generated values. ''' @@ -3837,7 +3837,7 @@ class TextSerializer(Serializer): stream.write('\n') except: pass - + class PprintSerializer(Serializer): def write(self, stream=sys.stdout): if self.results.has_key('href'): @@ -3845,7 +3845,7 @@ class PprintSerializer(Serializer): from pprint import pprint pprint(self.results, stream) stream.write('\n') - + if __name__ == '__main__': try: from optparse import OptionParser diff --git a/module/lib/simplejson/__init__.py b/module/lib/simplejson/__init__.py index ef5c0db48..03748041d 100644 --- a/module/lib/simplejson/__init__.py +++ b/module/lib/simplejson/__init__.py @@ -189,7 +189,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, If *namedtuple_as_object* is true (default: ``True``), :class:`tuple` subclasses with ``_asdict()`` methods will be encoded as JSON objects. - + If *tuple_as_array* is true (default: ``True``), :class:`tuple` (and subclasses) will be encoded as JSON arrays. @@ -268,7 +268,7 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, If *namedtuple_as_object* is true (default: ``True``), :class:`tuple` subclasses with ``_asdict()`` methods will be encoded as JSON objects. - + If *tuple_as_array* is true (default: ``True``), :class:`tuple` (and subclasses) will be encoded as JSON arrays. diff --git a/module/lib/simplejson/encoder.py b/module/lib/simplejson/encoder.py index 5ec7440f1..5560c55e6 100644 --- a/module/lib/simplejson/encoder.py +++ b/module/lib/simplejson/encoder.py @@ -157,7 +157,7 @@ class JSONEncoder(object): If namedtuple_as_object is true (the default), tuple subclasses with ``_asdict()`` methods will be encoded as JSON objects. - + If tuple_as_array is true (the default), tuple (and subclasses) will be encoded as JSON arrays. """ diff --git a/module/lib/thrift/protocol/TBase.py b/module/lib/thrift/protocol/TBase.py index e675c7dc0..514ba44e7 100644 --- a/module/lib/thrift/protocol/TBase.py +++ b/module/lib/thrift/protocol/TBase.py @@ -43,10 +43,10 @@ class TBase(object): if my_val != other_val: return False return True - + def __ne__(self, other): return not (self == other) - + def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) @@ -63,10 +63,10 @@ class TExceptionBase(Exception): # old style class so python2.4 can raise exceptions derived from this # This can't inherit from TBase because of that limitation. __slots__ = [] - + __repr__ = TBase.__repr__.im_func __eq__ = TBase.__eq__.im_func __ne__ = TBase.__ne__.im_func read = TBase.read.im_func write = TBase.write.im_func - + diff --git a/module/lib/thrift/protocol/TProtocol.py b/module/lib/thrift/protocol/TProtocol.py index 7338ff68a..f987026d9 100644 --- a/module/lib/thrift/protocol/TProtocol.py +++ b/module/lib/thrift/protocol/TProtocol.py @@ -279,7 +279,7 @@ class TProtocolBase: obj = obj_class() obj.read(self) return obj - + def readContainerMap(self, spec): results = dict() key_ttype, key_spec = spec[0], spec[1] diff --git a/module/lib/thrift/server/TNonblockingServer.py b/module/lib/thrift/server/TNonblockingServer.py index ea348a0b6..26d6af38d 100644 --- a/module/lib/thrift/server/TNonblockingServer.py +++ b/module/lib/thrift/server/TNonblockingServer.py @@ -81,7 +81,7 @@ def socket_exception(func): class Connection: """Basic class is represented connection. - + It can be in state: WAIT_LEN --- connection is reading request len. WAIT_MESSAGE --- connection is reading request. @@ -102,7 +102,7 @@ class Connection: def _read_len(self): """Reads length of request. - + It's really paranoic routine and it may be replaced by self.socket.recv(4).""" read = self.socket.recv(4 - len(self.message)) @@ -162,14 +162,14 @@ class Connection: @locked def ready(self, all_ok, message): """Callback function for switching state and waking up main thread. - + This function is the only function witch can be called asynchronous. - + The ready can switch Connection to three states: WAIT_LEN if request was oneway. SEND_ANSWER if request was processed in normal way. CLOSED if request throws unexpected exception. - + The one wakes up main thread. """ assert self.status == WAIT_PROCESS @@ -243,12 +243,12 @@ class TNonblockingServer: def wake_up(self): """Wake up main thread. - + The server usualy waits in select call in we should terminate one. The simplest way is using socketpair. - + Select always wait to read from the first socket of socketpair. - + In this case, we can just write anything to the second socket from socketpair.""" self._write.send('1') @@ -265,10 +265,10 @@ class TNonblockingServer: if connection.is_closed(): del self.clients[i] return select.select(readable, writable, readable) - + def handle(self): """Handle requests. - + WARNING! You must call prepare BEFORE calling handle. """ assert self.prepared, "You have to call prepare before handle" @@ -302,7 +302,7 @@ class TNonblockingServer: self.tasks.put([None, None, None, None, None]) self.socket.close() self.prepared = False - + def serve(self): """Serve forever.""" self.prepare() diff --git a/module/lib/thrift/transport/TZlibTransport.py b/module/lib/thrift/transport/TZlibTransport.py index 784d4e1e0..4356e4933 100644 --- a/module/lib/thrift/transport/TZlibTransport.py +++ b/module/lib/thrift/transport/TZlibTransport.py @@ -30,14 +30,14 @@ from TTransport import TTransportBase, CReadableTransport class TZlibTransportFactory(object): ''' Factory transport that builds zlib compressed transports. - + This factory caches the last single client/transport that it was passed and returns the same TZlibTransport object that was created. - + This caching means the TServer class will get the _same_ transport object for both input and output transports from this factory. (For non-threaded scenarios only, since the cache only holds one object) - + The purpose of this caching is to allocate only one TZlibTransport where only one is really needed (since it must have separate read/write buffers), and makes the statistics from getCompSavings() and getCompRatio() @@ -52,11 +52,11 @@ class TZlibTransportFactory(object): '''Wrap a transport , trans, with the TZlibTransport compressed transport class, returning a new transport to the caller. - + @param compresslevel: The zlib compression level, ranging from 0 (no compression) to 9 (best compression). Defaults to 9. @type compresslevel: int - + This method returns a TZlibTransport which wraps the passed C{trans} TTransport derived instance. ''' @@ -83,7 +83,7 @@ class TZlibTransport(TTransportBase, CReadableTransport): ''' Create a new TZlibTransport, wrapping C{trans}, another TTransport derived object. - + @param trans: A thrift transport object, i.e. a TSocket() object. @type trans: TTransport @param compresslevel: The zlib compression level, ranging @@ -127,17 +127,17 @@ class TZlibTransport(TTransportBase, CReadableTransport): ''' Get the current measured compression ratios (in,out) from this transport. - + Returns a tuple of: (inbound_compression_ratio, outbound_compression_ratio) - + The compression ratios are computed as: compressed / uncompressed E.g., data that compresses by 10x will have a ratio of: 0.10 and data that compresses to half of ts original size will have a ratio of 0.5 - + None is returned if no bytes have yet been processed in a particular direction. ''' @@ -152,10 +152,10 @@ class TZlibTransport(TTransportBase, CReadableTransport): ''' Get the current count of saved bytes due to data compression. - + Returns a tuple of: (inbound_saved_bytes, outbound_saved_bytes) - + Note: if compression is actually expanding your data (only likely with very tiny thrift objects), then the values returned will be negative. diff --git a/module/lib/wsgiserver/__init__.py b/module/lib/wsgiserver/__init__.py index c380e18b0..f2c2e866a 100644 --- a/module/lib/wsgiserver/__init__.py +++ b/module/lib/wsgiserver/__init__.py @@ -4,28 +4,28 @@ Simplest example on how to use this module directly (without using CherryPy's application machinery): from cherrypy import wsgiserver - + def my_crazy_app(environ, start_response): status = '200 OK' response_headers = [('Content-type','text/plain')] start_response(status, response_headers) return ['Hello world!\n'] - + server = wsgiserver.CherryPyWSGIServer( ('0.0.0.0', 8070), my_crazy_app, server_name='www.cherrypy.example') - + The CherryPy WSGI server can serve as many WSGI applications as you want in one instance by using a WSGIPathInfoDispatcher: - + d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app}) server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d) - + Want SSL support? Just set these attributes: - + server.ssl_certificate = <filename> server.ssl_private_key = <filename> - + if __name__ == '__main__': try: server.start() @@ -109,7 +109,7 @@ import errno def plat_specific_errors(*errnames): """Return error numbers for all errors in errnames on this platform. - + The 'errno' module contains different global constants depending on the specific platform (OS). This function will return the list of numeric values for a given list of potential names. @@ -147,24 +147,24 @@ comma_separated_headers = ['ACCEPT', 'ACCEPT-CHARSET', 'ACCEPT-ENCODING', class WSGIPathInfoDispatcher(object): """A WSGI dispatcher for dispatch based on the PATH_INFO. - + apps: a dict or list of (path_prefix, app) pairs. """ - + def __init__(self, apps): try: apps = apps.items() except AttributeError: pass - + # Sort the apps by len(path), descending apps.sort() apps.reverse() - + # The path_prefix strings must start, but not end, with a slash. # Use "" instead of "/". self.apps = [(p.rstrip("/"), a) for p, a in apps] - + def __call__(self, environ, start_response): path = environ["PATH_INFO"] or "/" for p, app in self.apps: @@ -174,7 +174,7 @@ class WSGIPathInfoDispatcher(object): environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p environ["PATH_INFO"] = path[len(p):] return app(environ, start_response) - + start_response('404 Not Found', [('Content-Type', 'text/plain'), ('Content-Length', '0')]) return [''] @@ -185,29 +185,29 @@ class MaxSizeExceeded(Exception): class SizeCheckWrapper(object): """Wraps a file-like object, raising MaxSizeExceeded if too large.""" - + def __init__(self, rfile, maxlen): self.rfile = rfile self.maxlen = maxlen self.bytes_read = 0 - + def _check_length(self): if self.maxlen and self.bytes_read > self.maxlen: raise MaxSizeExceeded() - + def read(self, size=None): data = self.rfile.read(size) self.bytes_read += len(data) self._check_length() return data - + def readline(self, size=None): if size is not None: data = self.rfile.readline(size) self.bytes_read += len(data) self._check_length() return data - + # User didn't specify a size ... # We read the line in chunks to make sure it's not a 100MB line ! res = [] @@ -219,7 +219,7 @@ class SizeCheckWrapper(object): # See http://www.cherrypy.org/ticket/421 if len(data) < 256 or data[-1:] == "\n": return ''.join(res) - + def readlines(self, sizehint=0): # Shamelessly stolen from StringIO total = 0 @@ -232,13 +232,13 @@ class SizeCheckWrapper(object): break line = self.readline() return lines - + def close(self): self.rfile.close() - + def __iter__(self): return self - + def next(self): data = self.rfile.next() self.bytes_read += len(data) @@ -248,9 +248,9 @@ class SizeCheckWrapper(object): class HTTPRequest(object): """An HTTP Request (and response). - + A single HTTP connection may consist of multiple request/response pairs. - + send: the 'send' method from the connection's socket object. wsgi_app: the WSGI application to call. environ: a partial WSGI environ (server and connection entries). @@ -267,7 +267,7 @@ class HTTPRequest(object): version is less than or equal to the one received in the request. An HTTP server MUST NOT send a version for which it is not at least conditionally compliant." - + outheaders: a list of header tuples to write in the response. ready: when True, the request has been parsed and is ready to begin generating the response. When False, signals the calling Connection @@ -280,16 +280,16 @@ class HTTPRequest(object): transfer-coding. This value is set automatically inside send_headers. """ - + max_request_header_size = 0 max_request_body_size = 0 - + def __init__(self, wfile, environ, wsgi_app): self.rfile = environ['wsgi.input'] self.wfile = wfile self.environ = environ.copy() self.wsgi_app = wsgi_app - + self.ready = False self.started_response = False self.status = "" @@ -297,18 +297,18 @@ class HTTPRequest(object): self.sent_headers = False self.close_connection = False self.chunked_write = False - + def parse_request(self): """Parse the next HTTP request start-line and message-headers.""" self.rfile.maxlen = self.max_request_header_size self.rfile.bytes_read = 0 - + try: self._parse_request() except MaxSizeExceeded: self.simple_response("413 Request Entity Too Large") return - + def _parse_request(self): # HTTP/1.1 connections are persistent by default. If a client # requests a page, then idles (leaves the connection open), @@ -322,7 +322,7 @@ class HTTPRequest(object): # Force self.ready = False so the connection will close. self.ready = False return - + if request_line == "\r\n": # RFC 2616 sec 4.1: "...if the server is reading the protocol # stream at the beginning of a message and receives a CRLF @@ -332,32 +332,32 @@ class HTTPRequest(object): if not request_line: self.ready = False return - + environ = self.environ - + try: method, path, req_protocol = request_line.strip().split(" ", 2) except ValueError: self.simple_response(400, "Malformed Request-Line") return - + environ["REQUEST_METHOD"] = method - + # path may be an abs_path (including "http://host.domain.tld"); scheme, location, path, params, qs, frag = urlparse(path) - + if frag: self.simple_response("400 Bad Request", "Illegal #fragment in Request-URI.") return - + if scheme: environ["wsgi.url_scheme"] = scheme if params: path = path + ";" + params - + environ["SCRIPT_NAME"] = "" - + # Unquote the path+params (e.g. "/this%20path" -> "this path"). # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 # @@ -367,11 +367,11 @@ class HTTPRequest(object): atoms = [unquote(x) for x in quoted_slash.split(path)] path = "%2F".join(atoms) environ["PATH_INFO"] = path - + # Note that, like wsgiref and most other WSGI servers, # we unquote the path but not the query string. environ["QUERY_STRING"] = qs - + # Compare request and server HTTP protocol versions, in case our # server does not support the requested protocol. Limit our output # to min(req, server). We want the following output: @@ -393,23 +393,23 @@ class HTTPRequest(object): # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. environ["SERVER_PROTOCOL"] = req_protocol self.response_protocol = "HTTP/%s.%s" % min(rp, sp) - + # If the Request-URI was an absoluteURI, use its location atom. if location: environ["SERVER_NAME"] = location - + # then all the http headers try: self.read_headers() except ValueError, ex: self.simple_response("400 Bad Request", repr(ex.args)) return - + mrbs = self.max_request_body_size if mrbs and int(environ.get("CONTENT_LENGTH", 0)) > mrbs: self.simple_response("413 Request Entity Too Large") return - + # Persistent connection support if self.response_protocol == "HTTP/1.1": # Both server and client are HTTP/1.1 @@ -419,16 +419,16 @@ class HTTPRequest(object): # Either the server or client (or both) are HTTP/1.0 if environ.get("HTTP_CONNECTION", "") != "Keep-Alive": self.close_connection = True - + # Transfer-Encoding support te = None if self.response_protocol == "HTTP/1.1": te = environ.get("HTTP_TRANSFER_ENCODING") if te: te = [x.strip().lower() for x in te.split(",") if x.strip()] - + self.chunked_read = False - + if te: for enc in te: if enc == "chunked": @@ -439,7 +439,7 @@ class HTTPRequest(object): self.simple_response("501 Unimplemented") self.close_connection = True return - + # From PEP 333: # "Servers and gateways that implement HTTP 1.1 must provide # transparent support for HTTP 1.1's "expect/continue" mechanism. @@ -459,23 +459,23 @@ class HTTPRequest(object): # but it seems like it would be a big slowdown for such a rare case. if environ.get("HTTP_EXPECT", "") == "100-continue": self.simple_response(100) - + self.ready = True - + def read_headers(self): """Read header lines from the incoming stream.""" environ = self.environ - + while True: line = self.rfile.readline() if not line: # No more data--illegal end of headers raise ValueError("Illegal end of headers.") - + if line == '\r\n': # Normal end of headers break - + if line[0] in ' \t': # It's a continuation line. v = line.strip() @@ -483,20 +483,20 @@ class HTTPRequest(object): k, v = line.split(":", 1) k, v = k.strip().upper(), v.strip() envname = "HTTP_" + k.replace("-", "_") - + if k in comma_separated_headers: existing = environ.get(envname) if existing: v = ", ".join((existing, v)) environ[envname] = v - + ct = environ.pop("HTTP_CONTENT_TYPE", None) if ct is not None: environ["CONTENT_TYPE"] = ct cl = environ.pop("HTTP_CONTENT_LENGTH", None) if cl is not None: environ["CONTENT_LENGTH"] = cl - + def decode_chunked(self): """Decode the 'chunked' transfer coding.""" cl = 0 @@ -515,15 +515,15 @@ class HTTPRequest(object): "Bad chunked transfer coding " "(expected '\\r\\n', got %r)" % crlf) return - + # Grab any trailer headers self.read_headers() - + data.seek(0) self.environ["wsgi.input"] = data self.environ["CONTENT_LENGTH"] = str(cl) or "" return True - + def respond(self): """Call the appropriate WSGI app and write its iterable output.""" # Set rfile.maxlen to ensure we don't read past Content-Length. @@ -539,20 +539,20 @@ class HTTPRequest(object): else: self.rfile.maxlen = cl self.rfile.bytes_read = 0 - + try: self._respond() except MaxSizeExceeded: if not self.sent_headers: self.simple_response("413 Request Entity Too Large") return - + def _respond(self): if self.chunked_read: if not self.decode_chunked(): self.close_connection = True return - + response = self.wsgi_app(self.environ, self.start_response) try: for chunk in response: @@ -567,35 +567,35 @@ class HTTPRequest(object): finally: if hasattr(response, "close"): response.close() - + if (self.ready and not self.sent_headers): self.sent_headers = True self.send_headers() if self.chunked_write: self.wfile.sendall("0\r\n\r\n") - + def simple_response(self, status, msg=""): """Write a simple response back to the client.""" status = str(status) buf = ["%s %s\r\n" % (self.environ['ACTUAL_SERVER_PROTOCOL'], status), "Content-Length: %s\r\n" % len(msg), "Content-Type: text/plain\r\n"] - + if status[:3] == "413" and self.response_protocol == 'HTTP/1.1': # Request Entity Too Large self.close_connection = True buf.append("Connection: close\r\n") - + buf.append("\r\n") if msg: buf.append(msg) - + try: self.wfile.sendall("".join(buf)) except socket.error, x: if x.args[0] not in socket_errors_to_ignore: raise - + def start_response(self, status, headers, exc_info = None): """WSGI callable to begin the HTTP response.""" # "The application may call start_response more than once, @@ -603,7 +603,7 @@ class HTTPRequest(object): if self.started_response and not exc_info: raise AssertionError("WSGI start_response called a second " "time with no exc_info.") - + # "if exc_info is provided, and the HTTP headers have already been # sent, start_response must raise an error, and should raise the # exc_info tuple." @@ -612,36 +612,36 @@ class HTTPRequest(object): raise exc_info[0], exc_info[1], exc_info[2] finally: exc_info = None - + self.started_response = True self.status = status self.outheaders.extend(headers) return self.write - + def write(self, chunk): """WSGI callable to write unbuffered data to the client. - + This method is also used internally by start_response (to write data from the iterable returned by the WSGI application). """ if not self.started_response: raise AssertionError("WSGI write called before start_response.") - + if not self.sent_headers: self.sent_headers = True self.send_headers() - + if self.chunked_write and chunk: buf = [hex(len(chunk))[2:], "\r\n", chunk, "\r\n"] self.wfile.sendall("".join(buf)) else: self.wfile.sendall(chunk) - + def send_headers(self): """Assert, process, and send the HTTP response message-headers.""" hkeys = [key.lower() for key, value in self.outheaders] status = int(self.status[:3]) - + if status == 413: # Request Entity Too Large. Close conn to avoid garbage. self.close_connection = True @@ -660,7 +660,7 @@ class HTTPRequest(object): else: # Closing the conn is the only way to determine len. self.close_connection = True - + if "connection" not in hkeys: if self.response_protocol == 'HTTP/1.1': # Both server and client are HTTP/1.1 or better @@ -670,7 +670,7 @@ class HTTPRequest(object): # Server and/or client are HTTP/1.0 if not self.close_connection: self.outheaders.append(("Connection", "Keep-Alive")) - + if (not self.close_connection) and (not self.chunked_read): # Read any remaining request body data on the socket. # "If an origin server receives a request that does not include an @@ -687,13 +687,13 @@ class HTTPRequest(object): size = self.rfile.maxlen - self.rfile.bytes_read if size > 0: self.rfile.read(size) - + if "date" not in hkeys: self.outheaders.append(("Date", rfc822.formatdate())) - + if "server" not in hkeys: self.outheaders.append(("Server", self.environ['SERVER_SOFTWARE'])) - + buf = [self.environ['ACTUAL_SERVER_PROTOCOL'], " ", self.status, "\r\n"] try: buf += [k + ": " + v + "\r\n" for k, v in self.outheaders] @@ -1039,17 +1039,17 @@ else: break buf_len += n return "".join(buffers) - + class SSL_fileobject(CP_fileobject): """SSL file object attached to a socket object.""" - + ssl_timeout = 3 ssl_retry = .01 - + def _safe_call(self, is_reader, call, *args, **kwargs): """Wrap the given call with SSL error-trapping. - + is_reader: if False EOF errors will be raised. If True, EOF errors will return "" (to emulate normal sockets). """ @@ -1068,7 +1068,7 @@ class SSL_fileobject(CP_fileobject): except SSL.SysCallError, e: if is_reader and e.args == (-1, 'Unexpected EOF'): return "" - + errnum = e.args[0] if is_reader and errnum in socket_errors_to_ignore: return "" @@ -1076,20 +1076,20 @@ class SSL_fileobject(CP_fileobject): except SSL.Error, e: if is_reader and e.args == (-1, 'Unexpected EOF'): return "" - + thirdarg = None try: thirdarg = e.args[0][0][2] except IndexError: pass - + if thirdarg == 'http request': # The client is talking HTTP to an HTTPS server. raise NoSSLError() raise FatalSSLAlert(*e.args) except: raise - + if time.time() - start > self.ssl_timeout: raise socket.timeout("timed out") @@ -1102,7 +1102,7 @@ class SSL_fileobject(CP_fileobject): p = self._sock.pending() if not p: return "".join(buf) - + def sendall(self, *args, **kwargs): return self._safe_call(False, super(SSL_fileobject, self).sendall, *args, **kwargs) @@ -1112,15 +1112,15 @@ class SSL_fileobject(CP_fileobject): class HTTPConnection(object): """An HTTP connection (active socket). - + socket: the raw socket object (usually TCP) for this connection. wsgi_app: the WSGI application for this server/connection. environ: a WSGI environ template. This will be copied for each request. - + rfile: a fileobject for reading from the socket. send: a function for writing (+ flush) to the socket. """ - + rbufsize = -1 RequestHandlerClass = HTTPRequest environ = {"wsgi.version": (1, 0), @@ -1130,15 +1130,15 @@ class HTTPConnection(object): "wsgi.run_once": False, "wsgi.errors": sys.stderr, } - + def __init__(self, sock, wsgi_app, environ): self.socket = sock self.wsgi_app = wsgi_app - + # Copy the class environ into self. self.environ = self.environ.copy() self.environ.update(environ) - + if SSL and isinstance(sock, SSL.ConnectionType): timeout = sock.gettimeout() self.rfile = SSL_fileobject(sock, "rb", self.rbufsize) @@ -1148,13 +1148,13 @@ class HTTPConnection(object): else: self.rfile = CP_fileobject(sock, "rb", self.rbufsize) self.wfile = CP_fileobject(sock, "wb", -1) - + # Wrap wsgi.input but not HTTPConnection.rfile itself. # We're also not setting maxlen yet; we'll do that separately # for headers and body for each iteration of self.communicate # (if maxlen is 0 the wrapper doesn't check length). self.environ["wsgi.input"] = SizeCheckWrapper(self.rfile, 0) - + def communicate(self): """Read each request and respond appropriately.""" try: @@ -1165,16 +1165,16 @@ class HTTPConnection(object): req = None req = self.RequestHandlerClass(self.wfile, self.environ, self.wsgi_app) - + # This order of operations should guarantee correct pipelining. req.parse_request() if not req.ready: return - + req.respond() if req.close_connection: return - + except socket.error, e: errnum = e.args[0] if errnum == 'timed out': @@ -1201,13 +1201,13 @@ class HTTPConnection(object): except Exception, e: if req and not req.sent_headers: req.simple_response("500 Internal Server Error", format_exc()) - + linger = False - + def close(self): """Close the socket underlying this connection.""" self.rfile.close() - + if not self.linger: # Python's socket module does NOT call close on the kernel socket # when you call socket.close(). We do so manually here because we @@ -1239,25 +1239,25 @@ _SHUTDOWNREQUEST = None class WorkerThread(threading.Thread): """Thread which continuously polls a Queue for Connection objects. - + server: the HTTP Server which spawned this thread, and which owns the Queue and is placing active connections into it. ready: a simple flag for the calling server to know when this thread has begun polling the Queue. - + Due to the timing issues of polling a Queue, a WorkerThread does not check its own 'ready' flag after it has started. To stop the thread, it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue (one for each running WorkerThread). """ - + conn = None - + def __init__(self, server): self.ready = False self.server = server threading.Thread.__init__(self) - + def run(self): try: self.ready = True @@ -1265,7 +1265,7 @@ class WorkerThread(threading.Thread): conn = self.server.requests.get() if conn is _SHUTDOWNREQUEST: return - + self.conn = conn try: conn.communicate() @@ -1278,11 +1278,11 @@ class WorkerThread(threading.Thread): class ThreadPool(object): """A Request Queue for the CherryPyWSGIServer which pools threads. - + ThreadPool objects must provide min, get(), put(obj), start() and stop(timeout) attributes. """ - + def __init__(self, server, min=10, max=-1): self.server = server self.min = min @@ -1290,7 +1290,7 @@ class ThreadPool(object): self._threads = [] self._queue = Queue.Queue() self.get = self._queue.get - + def start(self): """Start the pool of threads.""" for i in xrange(self.min): @@ -1301,17 +1301,17 @@ class ThreadPool(object): for worker in self._threads: while not worker.ready: time.sleep(.1) - + def _get_idle(self): """Number of worker threads which are idle. Read-only.""" return len([t for t in self._threads if t.conn is None]) idle = property(_get_idle, doc=_get_idle.__doc__) - + def put(self, obj): self._queue.put(obj) if obj is _SHUTDOWNREQUEST: return - + def grow(self, amount): """Spawn new worker threads (not above self.max).""" for i in xrange(amount): @@ -1321,7 +1321,7 @@ class ThreadPool(object): worker.setName("CP WSGIServer " + worker.getName()) self._threads.append(worker) worker.start() - + def shrink(self, amount): """Kill off worker threads (not below self.min).""" # Grow/shrink the pool if necessary. @@ -1330,7 +1330,7 @@ class ThreadPool(object): if not t.isAlive(): self._threads.remove(t) amount -= 1 - + if amount > 0: for i in xrange(min(amount, len(self._threads) - self.min)): # Put a number of shutdown requests on the queue equal @@ -1338,13 +1338,13 @@ class ThreadPool(object): # that worker will terminate and be culled from our list # in self.put. self._queue.put(_SHUTDOWNREQUEST) - + def stop(self, timeout=5): # Must shut down threads here so the code that calls # this method can know when all threads are stopped. for worker in self._threads: self._queue.put(_SHUTDOWNREQUEST) - + # Don't join currentThread (when stop is called inside a request). current = threading.currentThread() while self._threads: @@ -1376,14 +1376,14 @@ class ThreadPool(object): class SSLConnection: """A thread-safe wrapper for an SSL.Connection. - + *args: the arguments to create the wrapped SSL.Connection(*args). """ - + def __init__(self, *args): self._ssl_conn = SSL.Connection(*args) self._lock = threading.RLock() - + for f in ('get_context', 'pending', 'send', 'write', 'recv', 'read', 'renegotiate', 'bind', 'listen', 'connect', 'accept', 'setblocking', 'fileno', 'shutdown', 'close', 'get_cipher_list', @@ -1425,7 +1425,7 @@ else: class CherryPyWSGIServer(object): """An HTTP server for WSGI. - + bind_addr: The interface on which to listen for connections. For TCP sockets, a (host, port) tuple. Host values may be any IPv4 or IPv6 address, or any valid hostname. The string 'localhost' is a @@ -1433,7 +1433,7 @@ class CherryPyWSGIServer(object): The string '0.0.0.0' is a special IPv4 entry meaning "any active interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for IPv6. The empty string or None are not allowed. - + For UNIX sockets, supply the filename as a string. wsgi_app: the WSGI 'application callable'; multiple WSGI applications may be passed as (path_prefix, app) pairs. @@ -1444,47 +1444,46 @@ class CherryPyWSGIServer(object): request_queue_size: the 'backlog' argument to socket.listen(); specifies the maximum number of queued connections (default 5). timeout: the timeout in seconds for accepted connections (default 10). - + nodelay: if True (the default since 3.1), sets the TCP_NODELAY socket option. - + protocol: the version string to write in the Status-Line of all HTTP responses. For example, "HTTP/1.1" (the default). This also limits the supported features used in the response. - - + SSL/HTTPS --------- The OpenSSL module must be importable for SSL functionality. You can obtain it from http://pyopenssl.sourceforge.net/ - + ssl_certificate: the filename of the server SSL certificate. ssl_privatekey: the filename of the server's private key file. - + If either of these is None (both are None by default), this server will not use SSL. If both are given and are valid, they will be read on server start and used in the SSL context for the listening socket. """ - + protocol = "HTTP/1.1" _bind_addr = "127.0.0.1" version = "CherryPy/3.1.2" ready = False _interrupt = None - + nodelay = True - + ConnectionClass = HTTPConnection environ = {} - + # Paths to certificate and private key files ssl_certificate = None ssl_private_key = None - + def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5): self.requests = ThreadPool(self, min=numthreads or 1, max=max) - + if callable(wsgi_app): # We've been handed a single wsgi_app, in CP-2.1 style. # Assume it's mounted at "". @@ -1498,26 +1497,26 @@ class CherryPyWSGIServer(object): "include a WSGIPathInfoDispatcher instead.", DeprecationWarning) self.wsgi_app = WSGIPathInfoDispatcher(wsgi_app) - + self.bind_addr = bind_addr if not server_name: server_name = socket.gethostname() self.server_name = server_name self.request_queue_size = request_queue_size - + self.timeout = timeout self.shutdown_timeout = shutdown_timeout - + def _get_numthreads(self): return self.requests.min def _set_numthreads(self, value): self.requests.min = value numthreads = property(_get_numthreads, _set_numthreads) - + def __str__(self): return "%s.%s(%r)" % (self.__module__, self.__class__.__name__, self.bind_addr) - + def _get_bind_addr(self): return self._bind_addr def _set_bind_addr(self, value): @@ -1538,16 +1537,16 @@ class CherryPyWSGIServer(object): self._bind_addr = value bind_addr = property(_get_bind_addr, _set_bind_addr, doc="""The interface on which to listen for connections. - + For TCP sockets, a (host, port) tuple. Host values may be any IPv4 or IPv6 address, or any valid hostname. The string 'localhost' is a synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). The string '0.0.0.0' is a special IPv4 entry meaning "any active interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for IPv6. The empty string or None are not allowed. - + For UNIX sockets, supply the filename as a string.""") - + def start(self): """Run the server forever.""" # We don't have to trap KeyboardInterrupt or SystemExit here, @@ -1555,19 +1554,19 @@ class CherryPyWSGIServer(object): # If you're using this server with another framework, you should # trap those exceptions in whatever code block calls start(). self._interrupt = None - + # Select the appropriate socket if isinstance(self.bind_addr, basestring): # AF_UNIX socket - + # So we can reuse the socket... try: os.unlink(self.bind_addr) except: pass - + # So everyone can access the socket... try: os.chmod(self.bind_addr, 0777) except: pass - + info = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] else: # AF_INET or AF_INET6 socket @@ -1579,7 +1578,7 @@ class CherryPyWSGIServer(object): except socket.gaierror: # Probably a DNS issue. Assume IPv4. info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", self.bind_addr)] - + self.socket = None msg = "No socket could be created" for res in info: @@ -1594,14 +1593,14 @@ class CherryPyWSGIServer(object): break if not self.socket: raise socket.error, msg - + # Timeout so KeyboardInterrupt can be caught on Win32 self.socket.settimeout(1) self.socket.listen(self.request_queue_size) - + # Create worker threads self.requests.start() - + self.ready = True while self.ready: self.tick() @@ -1611,7 +1610,7 @@ class CherryPyWSGIServer(object): time.sleep(0.1) if self.interrupt: raise self.interrupt - + def bind(self, family, type, proto=0): """Create (or recreate) the actual socket object.""" self.socket = socket.socket(family, type, proto) @@ -1622,14 +1621,14 @@ class CherryPyWSGIServer(object): if self.ssl_certificate and self.ssl_private_key: if SSL is None: raise ImportError("You must install pyOpenSSL to use HTTPS.") - + # See http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/442473 ctx = SSL.Context(SSL.SSLv23_METHOD) ctx.use_privatekey_file(self.ssl_private_key) ctx.use_certificate_file(self.ssl_certificate) self.socket = SSLConnection(ctx, self.socket) self.populate_ssl_environ() - + # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), # activate dual-stack. See http://www.cherrypy.org/ticket/871. if (not isinstance(self.bind_addr, basestring) @@ -1640,9 +1639,9 @@ class CherryPyWSGIServer(object): # Apparently, the socket option is not available in # this machine's TCP stack pass - + self.socket.bind(self.bind_addr) - + def tick(self): """Accept a new connection and put it on the Queue.""" try: @@ -1652,7 +1651,7 @@ class CherryPyWSGIServer(object): return if hasattr(s, 'settimeout'): s.settimeout(self.timeout) - + environ = self.environ.copy() # SERVER_SOFTWARE is common for IIS. It's also helpful for # us to pass a default value for the "Server" response header. @@ -1663,7 +1662,7 @@ class CherryPyWSGIServer(object): # See http://www.faqs.org/rfcs/rfc2145.html. environ["ACTUAL_SERVER_PROTOCOL"] = self.protocol environ["SERVER_NAME"] = self.server_name - + if isinstance(self.bind_addr, basestring): # AF_UNIX. This isn't really allowed by WSGI, which doesn't # address unix domain sockets. But it's better than nothing. @@ -1674,7 +1673,7 @@ class CherryPyWSGIServer(object): # Until we do DNS lookups, omit REMOTE_HOST environ["REMOTE_ADDR"] = addr[0] environ["REMOTE_PORT"] = str(addr[1]) - + conn = self.ConnectionClass(s, self.wsgi_app, environ) self.requests.put(conn) except socket.timeout: @@ -1698,7 +1697,7 @@ class CherryPyWSGIServer(object): # See http://www.cherrypy.org/ticket/686. return raise - + def _get_interrupt(self): return self._interrupt def _set_interrupt(self, interrupt): @@ -1708,11 +1707,11 @@ class CherryPyWSGIServer(object): interrupt = property(_get_interrupt, _set_interrupt, doc="Set this to an Exception instance to " "interrupt the server.") - + def stop(self): """Gracefully shutdown a server that is serving forever.""" self.ready = False - + sock = getattr(self, "socket", None) if sock: if not isinstance(self.bind_addr, basestring): @@ -1744,9 +1743,9 @@ class CherryPyWSGIServer(object): if hasattr(sock, "close"): sock.close() self.socket = None - + self.requests.stop(self.shutdown_timeout) - + def populate_ssl_environ(self): """Create WSGI environ entries to be merged into each request.""" cert = open(self.ssl_certificate, 'rb').read() @@ -1760,7 +1759,7 @@ class CherryPyWSGIServer(object): ## SSL_VERSION_INTERFACE string The mod_ssl program version ## SSL_VERSION_LIBRARY string The OpenSSL program version } - + # Server certificate attributes ssl_environ.update({ 'SSL_SERVER_M_VERSION': cert.get_version(), @@ -1768,17 +1767,17 @@ class CherryPyWSGIServer(object): ## 'SSL_SERVER_V_START': Validity of server's certificate (start time), ## 'SSL_SERVER_V_END': Validity of server's certificate (end time), }) - + for prefix, dn in [("I", cert.get_issuer()), ("S", cert.get_subject())]: # X509Name objects don't seem to have a way to get the # complete DN string. Use str() and slice it instead, # because str(dn) == "<X509Name object '/C=US/ST=...'>" dnstr = str(dn)[18:-2] - + wsgikey = 'SSL_SERVER_%s_DN' % prefix ssl_environ[wsgikey] = dnstr - + # The DN should be of the form: /k1=v1/k2=v2, but we must allow # for any value to contain slashes itself (in a URL). while dnstr: @@ -1789,6 +1788,6 @@ class CherryPyWSGIServer(object): if key and value: wsgikey = 'SSL_SERVER_%s_DN_%s' % (prefix, key) ssl_environ[wsgikey] = value - + self.environ.update(ssl_environ) |