diff options
| author |  Walter Purcaro <vuolter@gmail.com>
| 2014-06-28 15:53:29 +0200 |
|---|---|---|
| committer | Walter Purcaro <vuolter@gmail.com>
| 2014-06-28 20:23:43 +0200 |
| commit | b8942672378fccc7d557dfc2e40a4d90aecf7a0d (patch) | |
| tree | 1d4a4cd78988b0ac3c18fcad8e4f3791b11f3f27 | |
| parent | [Lib] Update bottle.py to version 0.12.7 (diff) | |
| download | pyload-b8942672378fccc7d557dfc2e40a4d90aecf7a0d.tar.xz | |
[Lib] Revert libraries to original status (remove all cosmetics optimizations)
32 files changed, 570 insertions, 558 deletions
diff --git a/module/lib/BeautifulSoup.py b/module/lib/BeautifulSoup.py index 9f61c3ca0..55567f588 100644 --- a/module/lib/BeautifulSoup.py +++ b/module/lib/BeautifulSoup.py @@ -483,15 +483,15 @@ class Tag(PageElement): def _invert(h): "Cheap function to invert a hash." i = {} - for k, v in h.items(): + for k,v in h.items(): i[v] = k return i - XML_ENTITIES_TO_SPECIAL_CHARS = {"apos": "'", - "quot": '"', - "amp": "&", - "lt": "<", - "gt": ">"} + XML_ENTITIES_TO_SPECIAL_CHARS = { "apos" : "'", + "quot" : '"', + "amp" : "&", + "lt" : "<", + "gt" : ">" } XML_SPECIAL_CHARS_TO_ENTITIES = _invert(XML_ENTITIES_TO_SPECIAL_CHARS) @@ -929,7 +929,7 @@ class SoupStrainer: markupAttrMap = markupAttrs else: markupAttrMap = {} - for k, v in markupAttrs: + for k,v in markupAttrs: markupAttrMap[k] = v attrValue = markupAttrMap.get(attr) if not self._matches(attrValue, matchAgainst): @@ -1018,7 +1018,7 @@ def buildTagMap(default, *args): for portion in args: if hasattr(portion, 'items'): #It's a map. Merge it. - for k, v in portion.items(): + for k,v in portion.items(): built[k] = v elif hasattr(portion, '__iter__'): # is a list #It's a list. Map each item to the default. @@ -1150,7 +1150,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): n = int(name) except ValueError: return - if not 0 <= n <= 127: # ASCII ends at 127, not 255 + if not 0 <= n <= 127 : # ASCII ends at 127, not 255 return return self.convert_codepoint(n) @@ -1758,7 +1758,7 @@ class UnicodeDammit: # meta tags to the corresponding Python codec names. It only covers # values that aren't in Python's aliases and can't be determined # by the heuristics in find_codec. - CHARSET_ALIASES = {"macintosh": "mac-roman", + CHARSET_ALIASES = { "macintosh" : "mac-roman", "x-sjis" : "shift-jis" } def __init__(self, markup, overrideEncodings=[], @@ -1947,23 +1947,23 @@ class UnicodeDammit: def _ebcdic_to_ascii(self, s): c = self.__class__ if not c.EBCDIC_TO_ASCII_MAP: - emap = (0, 1, 2, 3, 156, 9, 134, 127, 151, 141, 142, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 157, 133, 8, 135, 24, 25, 146, 143, 28, 29, 30, 31, - 128, 129, 130, 131, 132, 10, 23, 27, 136, 137, 138, 139, 140, 5, 6, 7, - 144, 145, 22, 147, 148, 149, 150, 4, 152, 153, 154, 155, 20, 21, 158, 26, - 32, 160, 161, 162, 163, 164, 165, 166, 167, 168, 91, 46, 60, 40, 43, 33, - 38, 169, 170, 171, 172, 173, 174, 175, 176, 177, 93, 36, 42, 41, 59, 94, - 45, 47, 178, 179, 180, 181, 182, 183, 184, 185, 124, 44, 37, 95, 62, 63, - 186, 187, 188, 189, 190, 191, 192, 193, 194, 96, 58, 35, 64, 39, 61, 34, - 195, 97, 98, 99, 100, 101, 102, 103, 104, 105, 196, 197, 198, 199, 200, - 201, 202, 106, 107, 108, 109, 110, 111, 112, 113, 114, 203, 204, 205, - 206, 207, 208, 209, 126, 115, 116, 117, 118, 119, 120, 121, 122, 210, - 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, 231, 123, 65, 66, 67, 68, 69, 70, 71, 72, - 73, 232, 233, 234, 235, 236, 237, 125, 74, 75, 76, 77, 78, 79, 80, 81, - 82, 238, 239, 240, 241, 242, 243, 92, 159, 83, 84, 85, 86, 87, 88, 89, - 90, 244, 245, 246, 247, 248, 249, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - 250, 251, 252, 253, 254, 255) + emap = (0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15, + 16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31, + 128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7, + 144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26, + 32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33, + 38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94, + 45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63, + 186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34, + 195,97,98,99,100,101,102,103,104,105,196,197,198,199,200, + 201,202,106,107,108,109,110,111,112,113,114,203,204,205, + 206,207,208,209,126,115,116,117,118,119,120,121,122,210, + 211,212,213,214,215,216,217,218,219,220,221,222,223,224, + 225,226,227,228,229,230,231,123,65,66,67,68,69,70,71,72, + 73,232,233,234,235,236,237,125,74,75,76,77,78,79,80,81, + 82,238,239,240,241,242,243,92,159,83,84,85,86,87,88,89, + 90,244,245,246,247,248,249,48,49,50,51,52,53,54,55,56,57, + 250,251,252,253,254,255) import string c.EBCDIC_TO_ASCII_MAP = string.maketrans( \ ''.join(map(chr, range(256))), ''.join(map(chr, emap))) @@ -2000,8 +2000,7 @@ class UnicodeDammit: '\x9c' : ('oelig', '153'), '\x9d' : '?', '\x9e' : ('#x17E', '17E'), - '\x9f' : ('Yuml', '') - } + '\x9f' : ('Yuml', ''),} ####################################################################### diff --git a/module/lib/Getch.py b/module/lib/Getch.py index a052f619e..22b7ea7f8 100644 --- a/module/lib/Getch.py +++ b/module/lib/Getch.py @@ -65,7 +65,7 @@ class _GetchMacCarbon: else: # # The event contains the following info: - # (what, msg, when, where, mod)=Carbon.Evt.GetNextEvent(0x0008)[1] + # (what,msg,when,where,mod)=Carbon.Evt.GetNextEvent(0x0008)[1] # # The message (msg) contains the ASCII char which is # extracted with the 0x000000FF charCodeMask; this @@ -73,4 +73,4 @@ class _GetchMacCarbon: # returned # (what, msg, when, where, mod) = Carbon.Evt.GetNextEvent(0x0008)[1] - return chr(msg) + return chr(msg)
\ No newline at end of file diff --git a/module/lib/MultipartPostHandler.py b/module/lib/MultipartPostHandler.py index 4c0271c25..94aee0193 100644 --- a/module/lib/MultipartPostHandler.py +++ b/module/lib/MultipartPostHandler.py @@ -32,8 +32,8 @@ Example: cookies = cookielib.CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies), MultipartPostHandler.MultipartPostHandler) - params = {"username": "bob", "password": "riviera", - "file": open("filename", "rb") } + params = { "username" : "bob", "password" : "riviera", + "file" : open("filename", "rb") } opener.open("http://wwww.bobsite.com/upload/", params) Further Example: @@ -123,9 +123,9 @@ def main(): def validateFile(url): temp = tempfile.mkstemp(suffix=".html") write(temp[0], opener.open(url).read()) - params = {"ss": "0", # show source - "doctype": "Inline", - "uploaded_file": open(temp[1], "rb") } + params = { "ss" : "0", # show source + "doctype" : "Inline", + "uploaded_file" : open(temp[1], "rb") } print opener.open(validatorURL, params).read() remove(temp[1]) diff --git a/module/lib/beaker/cache.py b/module/lib/beaker/cache.py index f59c1e44d..4a96537ff 100644 --- a/module/lib/beaker/cache.py +++ b/module/lib/beaker/cache.py @@ -7,7 +7,7 @@ specifying an alternate type when used. Advanced users can add new backends in beaker.backends """ - + import warnings import beaker.container as container @@ -63,53 +63,54 @@ try: 2) except ImportError: pass - + def cache_region(region, *deco_args): """Decorate a function to cache itself using a cache region - + The region decorator requires arguments if there are more than 2 of the same named function, in the same module. This is because the namespace used for the functions cache is based on the functions name and the module. - + + Example:: - + # Add cache region settings to beaker: beaker.cache.cache_regions.update(dict_of_config_region_options)) - + @cache_region('short_term', 'some_data') def populate_things(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + return load('rabbits', 20, 0) - + .. note:: - + The function being decorated must only be called with positional arguments. - + """ cache = [None] - + def decorate(func): namespace = util.func_namespace(func) def cached(*args): reg = cache_regions[region] if not reg.get('enabled', True): return func(*args) - + if not cache[0]: if region not in cache_regions: raise BeakerException('Cache region not configured: %s' % region) cache[0] = Cache._get_cache(namespace, reg) - + cache_key = " ".join(map(str, deco_args + args)) def go(): return func(*args) - + return cache[0].get_value(cache_key, createfunc=go) cached._arg_namespace = namespace cached._arg_region = region @@ -119,37 +120,37 @@ def cache_region(region, *deco_args): def region_invalidate(namespace, region, *args): """Invalidate a cache region namespace or decorated function - + This function only invalidates cache spaces created with the cache_region decorator. - + :param namespace: Either the namespace of the result to invalidate, or the cached function reference - + :param region: The region the function was cached to. If the function was cached to a single region then this argument can be None - + :param args: Arguments that were used to differentiate the cached function as well as the arguments passed to the decorated function Example:: - + # Add cache region settings to beaker: beaker.cache.cache_regions.update(dict_of_config_region_options)) - + def populate_things(invalidate=False): - + @cache_region('short_term', 'some_data') def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + # If the results should be invalidated first if invalidate: region_invalidate(load, None, 'some_data', 'rabbits', 20, 0) return load('rabbits', 20, 0) - + """ if callable(namespace): if not region: @@ -161,7 +162,7 @@ def region_invalidate(namespace, region, *args): "namespace is required") else: region = cache_regions[region] - + cache = Cache._get_cache(namespace, region) cache_key = " ".join(str(x) for x in args) cache.remove_value(cache_key) @@ -179,7 +180,7 @@ class Cache(object): :param expiretime: seconds to keep cached data (legacy support) :param starttime: time when cache was cache was - + """ def __init__(self, namespace, type='memory', expiretime=None, starttime=None, expire=None, **nsargs): @@ -189,12 +190,12 @@ class Cache(object): raise cls except KeyError: raise TypeError("Unknown cache implementation %r" % type) - + self.namespace = cls(namespace, **nsargs) self.expiretime = expiretime or expire self.starttime = starttime self.nsargs = nsargs - + @classmethod def _get_cache(cls, namespace, kw): key = namespace + str(kw) @@ -203,16 +204,16 @@ class Cache(object): except KeyError: cache_managers[key] = cache = cls(namespace, **kw) return cache - + def put(self, key, value, **kw): self._get_value(key, **kw).set_value(value) set_value = put - + def get(self, key, **kw): """Retrieve a cached value from the container""" return self._get_value(key, **kw).get_value() get_value = get - + def remove_value(self, key, **kw): mycontainer = self._get_value(key, **kw) if mycontainer.has_current_value(): @@ -228,9 +229,9 @@ class Cache(object): kw.setdefault('expiretime', self.expiretime) kw.setdefault('starttime', self.starttime) - + return container.Value(key, self.namespace, **kw) - + @util.deprecated("Specifying a " "'type' and other namespace configuration with cache.get()/put()/etc. " "is deprecated. Specify 'type' and other namespace configuration to " @@ -244,24 +245,24 @@ class Cache(object): c = Cache(self.namespace.namespace, type=type, **kwargs) return c._get_value(key, expiretime=expiretime, createfunc=createfunc, starttime=starttime) - + def clear(self): """Clear all the values from the namespace""" self.namespace.remove() - + # dict interface def __getitem__(self, key): return self.get(key) - + def __contains__(self, key): return self._get_value(key).has_current_value() - + def has_key(self, key): return key in self - + def __delitem__(self, key): self.remove_value(key) - + def __setitem__(self, key, value): self.put(key, value) @@ -269,91 +270,94 @@ class Cache(object): class CacheManager(object): def __init__(self, **kwargs): """Initialize a CacheManager object with a set of options - + Options should be parsed with the :func:`~beaker.util.parse_cache_config_options` function to ensure only valid options are used. - + """ self.kwargs = kwargs self.regions = kwargs.pop('cache_regions', {}) - + # Add these regions to the module global cache_regions.update(self.regions) - + def get_cache(self, name, **kwargs): kw = self.kwargs.copy() kw.update(kwargs) return Cache._get_cache(name, kw) - + def get_cache_region(self, name, region): if region not in self.regions: raise BeakerException('Cache region not configured: %s' % region) kw = self.regions[region] return Cache._get_cache(name, kw) - + def region(self, region, *args): """Decorate a function to cache itself using a cache region - + The region decorator requires arguments if there are more than 2 of the same named function, in the same module. This is because the namespace used for the functions cache is based on the functions name and the module. - + + Example:: - + # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - + + def populate_things(): - + @cache.region('short_term', 'some_data') def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + return load('rabbits', 20, 0) - + .. note:: - + The function being decorated must only be called with positional arguments. - + """ return cache_region(region, *args) def region_invalidate(self, namespace, region, *args): """Invalidate a cache region namespace or decorated function - + This function only invalidates cache spaces created with the cache_region decorator. - + :param namespace: Either the namespace of the result to invalidate, or the name of the cached function - + :param region: The region the function was cached to. If the function was cached to a single region then this argument can be None - + :param args: Arguments that were used to differentiate the cached function as well as the arguments passed to the decorated function Example:: - + # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - + def populate_things(invalidate=False): - + @cache.region('short_term', 'some_data') def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + # If the results should be invalidated first if invalidate: cache.region_invalidate(load, None, 'some_data', 'rabbits', 20, 0) return load('rabbits', 20, 0) - + + """ return region_invalidate(namespace, region, *args) if callable(namespace): @@ -366,7 +370,7 @@ class CacheManager(object): "namespace is required") else: region = self.regions[region] - + cache = self.get_cache(namespace, **region) cache_key = " ".join(str(x) for x in args) cache.remove_value(cache_key) @@ -383,24 +387,25 @@ class CacheManager(object): # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - + + def populate_things(): - + @cache.cache('mycache', expire=15) def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + return load('rabbits', 20, 0) - + .. note:: - + The function being decorated must only be called with positional arguments. """ cache = [None] key = " ".join(str(x) for x in args) - + def decorate(func): namespace = util.func_namespace(func) def cached(*args): @@ -416,12 +421,12 @@ class CacheManager(object): def invalidate(self, func, *args, **kwargs): """Invalidate a cache decorated function - + This function only invalidates cache spaces created with the cache decorator. - + :param func: Decorated function to invalidate - + :param args: Used to make the key unique for this function, as in region() above. @@ -430,21 +435,22 @@ class CacheManager(object): function Example:: - + # Assuming a cache object is available like: cache = CacheManager(dict_of_config_options) - + + def populate_things(invalidate=False): - + @cache.cache('mycache', type="file", expire=15) def load(search_term, limit, offset): return load_the_data(search_term, limit, offset) - + # If the results should be invalidated first if invalidate: cache.invalidate(load, 'mycache', 'rabbits', 20, 0, type="file") return load('rabbits', 20, 0) - + """ namespace = func._arg_namespace diff --git a/module/lib/beaker/container.py b/module/lib/beaker/container.py index e7a938c6e..515e97af6 100644 --- a/module/lib/beaker/container.py +++ b/module/lib/beaker/container.py @@ -28,34 +28,34 @@ else: class NamespaceManager(object): """Handles dictionary operations and locking for a namespace of values. - + The implementation for setting and retrieving the namespace data is handled by subclasses. - + NamespaceManager may be used alone, or may be privately accessed by one or more Container objects. Container objects provide per-key services like expiration times and automatic recreation of values. - + Multiple NamespaceManagers created with a particular name will all share access to the same underlying datasource and will attempt to synchronize against a common mutex object. The scope of this sharing may be within a single process or across multiple processes, depending on the type of NamespaceManager used. - + The NamespaceManager itself is generally threadsafe, except in the case of the DBMNamespaceManager in conjunction with the gdbm dbm implementation. """ - + @classmethod def _init_dependencies(cls): pass - + def __init__(self, namespace): self._init_dependencies() self.namespace = namespace - + def get_creation_lock(self, key): raise NotImplementedError() @@ -79,37 +79,37 @@ class NamespaceManager(object): def __getitem__(self, key): raise NotImplementedError() - + def __setitem__(self, key, value): raise NotImplementedError() - + def set_value(self, key, value, expiretime=None): """Optional set_value() method called by Value. - + Allows an expiretime to be passed, for namespace implementations which can prune their collections using expiretime. - + """ self[key] = value - + def __contains__(self, key): raise NotImplementedError() def __delitem__(self, key): raise NotImplementedError() - + def keys(self): raise NotImplementedError() - + def remove(self): self.do_remove() - + class OpenResourceNamespaceManager(NamespaceManager): """A NamespaceManager where read/write operations require opening/ closing of a resource which is possibly mutexed. - + """ def __init__(self, namespace): NamespaceManager.__init__(self, namespace) @@ -133,13 +133,13 @@ class OpenResourceNamespaceManager(NamespaceManager): except: self.access_lock.release_read_lock() raise - + def release_read_lock(self): try: self.close(checkcount = True) finally: self.access_lock.release_read_lock() - + def acquire_write_lock(self, wait=True): r = self.access_lock.acquire_write_lock(wait) try: @@ -149,7 +149,7 @@ class OpenResourceNamespaceManager(NamespaceManager): except: self.access_lock.release_write_lock() raise - + def release_write_lock(self): try: self.close(checkcount=True) @@ -258,7 +258,7 @@ class Value(object): except KeyError: # guard against un-mutexed backends raising KeyError has_value = False - + if not self.createfunc: raise KeyError(self.key) finally: @@ -349,29 +349,29 @@ class Value(object): class AbstractDictionaryNSManager(NamespaceManager): """A subclassable NamespaceManager that places data in a dictionary. - + Subclasses should provide a "dictionary" attribute or descriptor which returns a dict-like object. The dictionary will store keys that are local to the "namespace" attribute of this manager, so ensure that the dictionary will not be used by any other namespace. e.g.:: - + import collections cached_data = collections.defaultdict(dict) - + class MyDictionaryManager(AbstractDictionaryNSManager): def __init__(self, namespace): AbstractDictionaryNSManager.__init__(self, namespace) self.dictionary = cached_data[self.namespace] - + The above stores data in a global dictionary called "cached_data", which is structured as a dictionary of dictionaries, keyed first on namespace name to a sub-dictionary, then on actual cache key to value. - + """ - + def get_creation_lock(self, key): return NameLock( identifier="memorynamespace/funclock/%s/%s" % (self.namespace, key), @@ -386,19 +386,19 @@ class AbstractDictionaryNSManager(NamespaceManager): def has_key(self, key): return self.dictionary.__contains__(key) - + def __setitem__(self, key, value): self.dictionary[key] = value - + def __delitem__(self, key): del self.dictionary[key] def do_remove(self): self.dictionary.clear() - + def keys(self): return self.dictionary.keys() - + class MemoryNamespaceManager(AbstractDictionaryNSManager): namespaces = util.SyncDict() @@ -411,7 +411,7 @@ class DBMNamespaceManager(OpenResourceNamespaceManager): def __init__(self, namespace, dbmmodule=None, data_dir=None, dbm_dir=None, lock_dir=None, digest_filenames=True, **kwargs): self.digest_filenames = digest_filenames - + if not dbm_dir and not data_dir: raise MissingCacheParameter("data_dir or dbm_dir is required") elif dbm_dir: @@ -419,7 +419,7 @@ class DBMNamespaceManager(OpenResourceNamespaceManager): else: self.dbm_dir = data_dir + "/container_dbm" util.verify_directory(self.dbm_dir) - + if not lock_dir and not data_dir: raise MissingCacheParameter("data_dir or lock_dir is required") elif lock_dir: @@ -437,14 +437,14 @@ class DBMNamespaceManager(OpenResourceNamespaceManager): identifiers=[self.namespace], extension='.dbm', digest_filenames=self.digest_filenames) - + debug("data file %s", self.file) self._checkfile() def get_access_lock(self): return file_synchronizer(identifier=self.namespace, lock_dir=self.lock_dir) - + def get_creation_lock(self, key): return file_synchronizer( identifier = "dbmcontainer/funclock/%s" % self.namespace, @@ -458,19 +458,19 @@ class DBMNamespaceManager(OpenResourceNamespaceManager): for ext in ('db', 'dat', 'pag', 'dir'): if os.access(file + os.extsep + ext, os.F_OK): return True - + return False - + def _checkfile(self): if not self.file_exists(self.file): g = self.dbmmodule.open(self.file, 'c') g.close() - + def get_filenames(self): list = [] if os.access(self.file, os.F_OK): list.append(self.file) - + for ext in ('pag', 'dir', 'db', 'dat'): if os.access(self.file + os.extsep + ext, os.F_OK): list.append(self.file + os.extsep + ext) @@ -488,17 +488,17 @@ class DBMNamespaceManager(OpenResourceNamespaceManager): if self.dbm is not None: debug("closing dbm file %s", self.file) self.dbm.close() - + def do_remove(self): for f in self.get_filenames(): os.remove(f) - + def __getitem__(self, key): return cPickle.loads(self.dbm[key]) def __contains__(self, key): return self.dbm.has_key(key) - + def __setitem__(self, key, value): self.dbm[key] = cPickle.dumps(value) @@ -513,7 +513,7 @@ class FileNamespaceManager(OpenResourceNamespaceManager): def __init__(self, namespace, data_dir=None, file_dir=None, lock_dir=None, digest_filenames=True, **kwargs): self.digest_filenames = digest_filenames - + if not file_dir and not data_dir: raise MissingCacheParameter("data_dir or file_dir is required") elif file_dir: @@ -536,19 +536,19 @@ class FileNamespaceManager(OpenResourceNamespaceManager): extension='.cache', digest_filenames=self.digest_filenames) self.hash = {} - + debug("data file %s", self.file) def get_access_lock(self): return file_synchronizer(identifier=self.namespace, lock_dir=self.lock_dir) - + def get_creation_lock(self, key): return file_synchronizer( identifier = "filecontainer/funclock/%s" % self.namespace, lock_dir = self.lock_dir ) - + def file_exists(self, file): return os.access(file, os.F_OK) @@ -562,7 +562,7 @@ class FileNamespaceManager(OpenResourceNamespaceManager): fh.close() self.flags = flags - + def do_close(self): if self.flags == 'c' or self.flags == 'w': fh = open(self.file, 'wb') @@ -571,7 +571,7 @@ class FileNamespaceManager(OpenResourceNamespaceManager): self.hash = {} self.flags = None - + def do_remove(self): try: os.remove(self.file) @@ -580,13 +580,13 @@ class FileNamespaceManager(OpenResourceNamespaceManager): # but client code has asked for a clear() operation... pass self.hash = {} - + def __getitem__(self, key): return self.hash[key] def __contains__(self, key): return self.hash.has_key(key) - + def __setitem__(self, key, value): self.hash[key] = value @@ -602,7 +602,7 @@ class FileNamespaceManager(OpenResourceNamespaceManager): namespace_classes = {} ContainerContext = dict - + class ContainerMeta(type): def __init__(cls, classname, bases, dict_): namespace_classes[cls] = cls.namespace_class diff --git a/module/lib/beaker/crypto/pbkdf2.py b/module/lib/beaker/crypto/pbkdf2.py index 5f40a5c78..96dc5fbb2 100644 --- a/module/lib/beaker/crypto/pbkdf2.py +++ b/module/lib/beaker/crypto/pbkdf2.py @@ -78,8 +78,8 @@ def strxor(a, b): return "".join([chr(ord(x) ^ ord(y)) for (x, y) in zip(a, b)]) class PBKDF2(object): - """PBKDF2.py: PKCS#5 v2.0 Password-Based Key Derivation - + """PBKDF2.py : PKCS#5 v2.0 Password-Based Key Derivation + This implementation takes a passphrase and a salt (and optionally an iteration count, a digest module, and a MAC module) and provides a file-like object from which an arbitrarily-sized key can be read. @@ -89,10 +89,10 @@ class PBKDF2(object): The idea behind PBKDF2 is to derive a cryptographic key from a passphrase and a salt. - + PBKDF2 may also be used as a strong salted password hash. The 'crypt' function is provided for that purpose. - + Remember: Keys generated using PBKDF2 are only as strong as the passphrases they are derived from. """ @@ -109,7 +109,7 @@ class PBKDF2(object): """Pseudorandom function. e.g. HMAC-SHA1""" return self.__macmodule(key=key, msg=msg, digestmod=self.__digestmodule).digest() - + def read(self, bytes): """Read the specified number of key bytes.""" if self.closed: @@ -131,7 +131,7 @@ class PBKDF2(object): self.__buf = buf[bytes:] self.__blockNum = i return retval - + def __f(self, i): # i must fit within 32 bits assert (1 <= i <= 0xffffffff) @@ -141,7 +141,7 @@ class PBKDF2(object): U = self.__prf(self.__passphrase, U) result = strxor(result, U) return result - + def hexread(self, octets): """Read the specified number of octets. Return them as hexadecimal. @@ -151,7 +151,7 @@ class PBKDF2(object): def _setup(self, passphrase, salt, iterations, prf): # Sanity checks: - + # passphrase and salt must be str or unicode (in the latter # case, we convert to UTF-8) if isinstance(passphrase, unicode): @@ -168,7 +168,7 @@ class PBKDF2(object): raise TypeError("iterations must be an integer") if iterations < 1: raise ValueError("iterations must be at least 1") - + # prf must be callable if not callable(prf): raise TypeError("prf must be callable") @@ -180,7 +180,7 @@ class PBKDF2(object): self.__blockNum = 0 self.__buf = "" self.closed = False - + def close(self): """Close the stream.""" if not self.closed: @@ -194,13 +194,13 @@ class PBKDF2(object): def crypt(word, salt=None, iterations=None): """PBKDF2-based unix crypt(3) replacement. - + The number of iterations specified in the salt overrides the 'iterations' parameter. The effective hash length is 192 bits. """ - + # Generate a (pseudo-)random salt if the user hasn't provided one. if salt is None: salt = _makesalt() @@ -229,7 +229,7 @@ def crypt(word, salt=None, iterations=None): iterations = converted if not (iterations >= 1): raise ValueError("Invalid salt") - + # Make sure the salt matches the allowed character set allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789./" for ch in salt: @@ -251,7 +251,7 @@ PBKDF2.crypt = staticmethod(crypt) def _makesalt(): """Return a 48-bit pseudorandom salt for crypt(). - + This function is not suitable for generating cryptographic secrets. """ binarysalt = "".join([pack("@H", randint(0, 0xffff)) for i in range(3)]) @@ -260,7 +260,7 @@ def _makesalt(): def test_pbkdf2(): """Module self-test""" from binascii import a2b_hex - + # # Test vectors from RFC 3962 # @@ -284,18 +284,18 @@ def test_pbkdf2(): "c5ec59f1a452f5cc9ad940fea0598ed1") if result != expected: raise RuntimeError("self-test failed") - + # Test 4 result = PBKDF2("X"*65, "pass phrase exceeds block size", 1200).hexread(32) expected = ("9ccad6d468770cd51b10e6a68721be61" "1a8b4d282601db3b36be9246915ec82a") if result != expected: raise RuntimeError("self-test failed") - + # # Other test vectors # - + # Chunked read f = PBKDF2("kickstart", "workbench", 256) result = f.read(17) @@ -306,7 +306,7 @@ def test_pbkdf2(): expected = PBKDF2("kickstart", "workbench", 256).read(40) if result != expected: raise RuntimeError("self-test failed") - + # # crypt() test vectors # @@ -316,7 +316,7 @@ def test_pbkdf2(): expected = '$p5k2$$exec$r1EWMCMk7Rlv3L/RNcFXviDefYa0hlql' if result != expected: raise RuntimeError("self-test failed") - + # crypt 2 result = crypt("gnu", '$p5k2$c$u9HvcT4d$.....') expected = '$p5k2$c$u9HvcT4d$Sd1gwSVCLZYAuqZ25piRnbBEoAesaa/g' @@ -328,7 +328,7 @@ def test_pbkdf2(): expected = "$p5k2$d$tUsch7fU$nqDkaxMDOFBeJsTSfABsyn.PYUXilHwL" if result != expected: raise RuntimeError("self-test failed") - + # crypt 4 (unicode) result = crypt(u'\u0399\u03c9\u03b1\u03bd\u03bd\u03b7\u03c2', '$p5k2$$KosHgqNo$9mjN8gqjt02hDoP0c2J0ABtLIwtot8cQ') diff --git a/module/lib/beaker/crypto/pycrypto.py b/module/lib/beaker/crypto/pycrypto.py index 1939555ce..a3eb4d9db 100644 --- a/module/lib/beaker/crypto/pycrypto.py +++ b/module/lib/beaker/crypto/pycrypto.py @@ -9,16 +9,16 @@ try: def aesEncrypt(data, key): cipher = aes.AES(key) return cipher.process(data) - + # magic. aesDecrypt = aesEncrypt - + except ImportError: from Crypto.Cipher import AES def aesEncrypt(data, key): cipher = AES.new(key) - + data = data + (" " * (16 - (len(data) % 16))) return cipher.encrypt(data) diff --git a/module/lib/beaker/crypto/util.py b/module/lib/beaker/crypto/util.py index 7f96ac856..d97e8ce6f 100644 --- a/module/lib/beaker/crypto/util.py +++ b/module/lib/beaker/crypto/util.py @@ -6,9 +6,9 @@ try: # Use PyCrypto (if available) from Crypto.Hash import HMAC as hmac, SHA as hmac_sha1 sha1 = hmac_sha1.new - + except ImportError: - + # PyCrypto not available. Use the Python standard library. import hmac diff --git a/module/lib/beaker/ext/database.py b/module/lib/beaker/ext/database.py index 7e3db6899..701e6f7d2 100644 --- a/module/lib/beaker/ext/database.py +++ b/module/lib/beaker/ext/database.py @@ -30,12 +30,12 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): except ImportError: raise InvalidCacheBackendError("Database cache backend requires " "the 'sqlalchemy' library") - + def __init__(self, namespace, url=None, sa_opts=None, optimistic=False, table_name='beaker_cache', data_dir=None, lock_dir=None, **params): """Creates a database namespace manager - + ``url`` SQLAlchemy compliant db url ``sa_opts`` @@ -49,7 +49,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): The table name to use in the database for the cache. """ OpenResourceNamespaceManager.__init__(self, namespace) - + if sa_opts is None: sa_opts = params @@ -59,7 +59,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): self.lock_dir = data_dir + "/container_db_lock" if self.lock_dir: verify_directory(self.lock_dir) - + # Check to see if the table's been created before url = url or sa_opts['sa.url'] table_key = url + table_name @@ -90,7 +90,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): self._is_new = False self.loaded = False self.cache = DatabaseNamespaceManager.tables.get(table_key, make_cache) - + def get_access_lock(self): return null_synchronizer() @@ -104,7 +104,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): if self.loaded: self.flags = flags return - + cache = self.cache result = sa.select([cache.c.data], cache.c.namespace==self.namespace @@ -123,7 +123,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): self._is_new = True self.flags = flags self.loaded = True - + def do_close(self): if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): cache = self.cache @@ -136,12 +136,12 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): cache.update(cache.c.namespace==self.namespace).execute( data=self.hash, accessed=datetime.now()) self.flags = None - + def do_remove(self): cache = self.cache cache.delete(cache.c.namespace==self.namespace).execute() self.hash = {} - + # We can retain the fact that we did a load attempt, but since the # file is gone this will be a new namespace should it be saved. self._is_new = True @@ -151,7 +151,7 @@ class DatabaseNamespaceManager(OpenResourceNamespaceManager): def __contains__(self, key): return self.hash.has_key(key) - + def __setitem__(self, key, value): self.hash[key] = value diff --git a/module/lib/beaker/ext/google.py b/module/lib/beaker/ext/google.py index b6d617d6b..dd8380d7f 100644 --- a/module/lib/beaker/ext/google.py +++ b/module/lib/beaker/ext/google.py @@ -23,11 +23,11 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): except ImportError: raise InvalidCacheBackendError("Datastore cache backend requires the " "'google.appengine.ext' library") - + def __init__(self, namespace, table_name='beaker_cache', **params): """Creates a datastore namespace manager""" OpenResourceNamespaceManager.__init__(self, namespace) - + def make_cache(): table_dict = dict(created=db.DateTimeProperty(), accessed=db.DateTimeProperty(), @@ -40,11 +40,11 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): self._is_new = False self.loaded = False self.log_debug = logging.DEBUG >= log.getEffectiveLevel() - + # Google wants namespaces to start with letters, change the namespace # to start with a letter self.namespace = 'p%s' % self.namespace - + def get_access_lock(self): return null_synchronizer() @@ -57,9 +57,9 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): if self.loaded: self.flags = flags return - + item = self.cache.get_by_key_name(self.namespace) - + if not item: self._is_new = True self.hash = {} @@ -74,7 +74,7 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): self._is_new = True self.flags = flags self.loaded = True - + def do_close(self): if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): if self._is_new: @@ -90,12 +90,12 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): item.accessed = datetime.now() item.put() self.flags = None - + def do_remove(self): item = self.cache.get_by_key_name(self.namespace) item.delete() self.hash = {} - + # We can retain the fact that we did a load attempt, but since the # file is gone this will be a new namespace should it be saved. self._is_new = True @@ -105,7 +105,7 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): def __contains__(self, key): return self.hash.has_key(key) - + def __setitem__(self, key, value): self.hash[key] = value @@ -114,7 +114,7 @@ class GoogleNamespaceManager(OpenResourceNamespaceManager): def keys(self): return self.hash.keys() - + class GoogleContainer(Container): namespace_class = GoogleNamespaceManager diff --git a/module/lib/beaker/ext/memcached.py b/module/lib/beaker/ext/memcached.py index 60360ae0a..96516953f 100644 --- a/module/lib/beaker/ext/memcached.py +++ b/module/lib/beaker/ext/memcached.py @@ -8,7 +8,7 @@ memcache = None class MemcachedNamespaceManager(NamespaceManager): clients = SyncDict() - + @classmethod def _init_dependencies(cls): global memcache @@ -27,25 +27,25 @@ class MemcachedNamespaceManager(NamespaceManager): except ImportError: raise InvalidCacheBackendError("Memcached cache backend requires either " "the 'memcache' or 'cmemcache' library") - + def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params): NamespaceManager.__init__(self, namespace) - + if not url: raise MissingCacheParameter("url is required") - + if lock_dir: self.lock_dir = lock_dir elif data_dir: self.lock_dir = data_dir + "/container_mcd_lock" if self.lock_dir: verify_directory(self.lock_dir) - + self.mc = MemcachedNamespaceManager.clients.get(url, memcache.Client, url.split(';')) def get_creation_lock(self, key): return file_synchronizer( - identifier="memcachedcontainer/funclock/%s" % self.namespace, lock_dir = self.lock_dir) + identifier="memcachedcontainer/funclock/%s" % self.namespace,lock_dir = self.lock_dir) def _format_key(self, key): return self.namespace + '_' + key.replace(' ', '\302\267') @@ -68,13 +68,13 @@ class MemcachedNamespaceManager(NamespaceManager): def __setitem__(self, key, value): self.set_value(key, value) - + def __delitem__(self, key): self.mc.delete(self._format_key(key)) def do_remove(self): self.mc.flush_all() - + def keys(self): raise NotImplementedError("Memcache caching does not support iteration of all cache keys") diff --git a/module/lib/beaker/middleware.py b/module/lib/beaker/middleware.py index 785d25377..7ba88b37d 100644 --- a/module/lib/beaker/middleware.py +++ b/module/lib/beaker/middleware.py @@ -16,15 +16,15 @@ from beaker.util import coerce_cache_params, coerce_session_params, \ class CacheMiddleware(object): cache = beaker_cache - + def __init__(self, app, config=None, environ_key='beaker.cache', **kwargs): """Initialize the Cache Middleware - + The Cache middleware will make a Cache instance available every request under the ``environ['beaker.cache']`` key by default. The location in environ can be changed by setting ``environ_key``. - + ``config`` dict All settings should be prefixed by 'cache.'. This method of passing variables is intended for Paste and other @@ -32,11 +32,11 @@ class CacheMiddleware(object): single dictionary. If config contains *no cache. prefixed args*, then *all* of the config options will be used to intialize the Cache objects. - + ``environ_key`` Location where the Cache instance will keyed in the WSGI environ - + ``**kwargs`` All keyword arguments are assumed to be cache settings and will override any settings found in ``config`` @@ -44,26 +44,26 @@ class CacheMiddleware(object): """ self.app = app config = config or {} - + self.options = {} - + # Update the options with the parsed config self.options.update(parse_cache_config_options(config)) - + # Add any options from kwargs, but leave out the defaults this # time self.options.update( parse_cache_config_options(kwargs, include_defaults=False)) - + # Assume all keys are intended for cache if none are prefixed with # 'cache.' if not self.options and config: self.options = config - + self.options.update(kwargs) self.cache_manager = CacheManager(**self.options) self.environ_key = environ_key - + def __call__(self, environ, start_response): if environ.get('paste.registry'): if environ['paste.registry'].reglist: @@ -75,16 +75,16 @@ class CacheMiddleware(object): class SessionMiddleware(object): session = beaker_session - + def __init__(self, wrap_app, config=None, environ_key='beaker.session', **kwargs): """Initialize the Session Middleware - + The Session middleware will make a lazy session instance available every request under the ``environ['beaker.session']`` key by default. The location in environ can be changed by setting ``environ_key``. - + ``config`` dict All settings should be prefixed by 'session.'. This method of passing variables is intended for Paste and other @@ -92,18 +92,18 @@ class SessionMiddleware(object): single dictionary. If config contains *no cache. prefixed args*, then *all* of the config options will be used to intialize the Cache objects. - + ``environ_key`` Location where the Session instance will keyed in the WSGI environ - + ``**kwargs`` All keyword arguments are assumed to be session settings and will override any settings found in ``config`` """ config = config or {} - + # Load up the default params self.options = dict(invalidate_corrupt=True, type=None, data_dir=None, key='beaker.session.id', @@ -120,19 +120,19 @@ class SessionMiddleware(object): warnings.warn('Session options should start with session. ' 'instead of session_.', DeprecationWarning, 2) self.options[key[8:]] = val - + # Coerce and validate session params coerce_session_params(self.options) - + # Assume all keys are intended for cache if none are prefixed with # 'cache.' if not self.options and config: self.options = config - + self.options.update(kwargs) self.wrap_app = wrap_app self.environ_key = environ_key - + def __call__(self, environ, start_response): session = SessionObject(environ, **self.options) if environ.get('paste.registry'): @@ -140,7 +140,7 @@ class SessionMiddleware(object): environ['paste.registry'].register(self.session, session) environ[self.environ_key] = session environ['beaker.get_session'] = self._get_session - + def session_start_response(status, headers, exc_info = None): if session.accessed(): session.persist() @@ -150,7 +150,7 @@ class SessionMiddleware(object): headers.append(('Set-cookie', cookie)) return start_response(status, headers, exc_info) return self.wrap_app(environ, session_start_response) - + def _get_session(self): return Session({}, use_cookies=False, **self.options) diff --git a/module/lib/beaker/session.py b/module/lib/beaker/session.py index a1461ebdd..7d465530b 100644 --- a/module/lib/beaker/session.py +++ b/module/lib/beaker/session.py @@ -3,7 +3,7 @@ import os import random import time from datetime import datetime, timedelta - + from beaker.crypto import hmac as HMAC, hmac_sha1 as SHA1, md5 from beaker.util import pickle @@ -15,32 +15,32 @@ from base64 import b64encode, b64decode __all__ = ['SignedCookie', 'Session'] -getpid = hasattr(os, 'getpid') and os.getpid or (lambda: '') +getpid = hasattr(os, 'getpid') and os.getpid or (lambda : '') class SignedCookie(Cookie.BaseCookie): """Extends python cookie to give digital signature support""" def __init__(self, secret, input=None): self.secret = secret Cookie.BaseCookie.__init__(self, input) - + def value_decode(self, val): val = val.strip('"') sig = HMAC.new(self.secret, val[40:], SHA1).hexdigest() - + # Avoid timing attacks invalid_bits = 0 input_sig = val[:40] if len(sig) != len(input_sig): return None, val - + for a, b in zip(sig, input_sig): invalid_bits += a != b - + if invalid_bits: return None, val else: return val[40:], val - + def value_encode(self, val): sig = HMAC.new(self.secret, val, SHA1).hexdigest() return str(val), ("%s%s" % (sig, val)) @@ -48,7 +48,7 @@ class SignedCookie(Cookie.BaseCookie): class Session(dict): """Session object that uses container package for storage. - + ``key`` The name the cookie should be set to. ``timeout`` @@ -76,15 +76,15 @@ class Session(dict): self.namespace_class = namespace_class or clsmap[self.type] self.namespace_args = namespace_args - + self.request = request self.data_dir = data_dir self.key = key - + self.timeout = timeout self.use_cookies = use_cookies self.cookie_expires = cookie_expires - + # Default cookie domain/path self._domain = cookie_domain self._path = '/' @@ -93,7 +93,7 @@ class Session(dict): self.secure = secure self.id = id self.accessed_dict = {} - + if self.use_cookies: cookieheader = request.get('cookie', '') if secret: @@ -103,10 +103,10 @@ class Session(dict): self.cookie = SignedCookie(secret, input=None) else: self.cookie = Cookie.SimpleCookie(input=cookieheader) - + if not self.id and self.key in self.cookie: self.id = self.cookie[self.key].value - + self.is_new = self.id is None if self.is_new: self._create_id() @@ -119,7 +119,7 @@ class Session(dict): self.invalidate() else: raise - + def _create_id(self): self.id = md5( md5("%f%s%f%s" % (time.time(), id({}), random.random(), @@ -148,31 +148,31 @@ class Session(dict): expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) self.request['cookie_out'] = self.cookie[self.key].output(header='') self.request['set_cookie'] = False - + def created(self): return self['_creation_time'] created = property(created) - + def _set_domain(self, domain): self['_domain'] = domain self.cookie[self.key]['domain'] = domain self.request['cookie_out'] = self.cookie[self.key].output(header='') self.request['set_cookie'] = True - + def _get_domain(self): return self._domain - + domain = property(_get_domain, _set_domain) - + def _set_path(self, path): self['_path'] = path self.cookie[self.key]['path'] = path self.request['cookie_out'] = self.cookie[self.key].output(header='') self.request['set_cookie'] = True - + def _get_path(self): return self._path - + path = property(_get_path, _set_path) def _delete_cookie(self): @@ -203,7 +203,7 @@ class Session(dict): self.was_invalidated = True self._create_id() self.load() - + def load(self): "Loads the data from this session from persistent storage" self.namespace = self.namespace_class(self.id, @@ -211,7 +211,7 @@ class Session(dict): **self.namespace_args) now = time.time() self.request['set_cookie'] = True - + self.namespace.acquire_read_lock() timed_out = False try: @@ -233,7 +233,7 @@ class Session(dict): '_accessed_time':now } self.is_new = True - + if self.timeout is not None and \ now - session_data['_accessed_time'] > self.timeout: timed_out= True @@ -244,7 +244,7 @@ class Session(dict): self.last_accessed = None else: self.last_accessed = session_data['_accessed_time'] - + # Update the current _accessed_time session_data['_accessed_time'] = now self.update(session_data) @@ -253,34 +253,34 @@ class Session(dict): self.namespace.release_read_lock() if timed_out: self.invalidate() - + def save(self, accessed_only=False): """Saves the data for this session to persistent storage - + If accessed_only is True, then only the original data loaded at the beginning of the request will be saved, with the updated last accessed time. - + """ # Look to see if its a new session that was only accessed # Don't save it under that case if accessed_only and self.is_new: return None - + if not hasattr(self, 'namespace'): self.namespace = self.namespace_class( self.id, data_dir=self.data_dir, digest_filenames=False, **self.namespace_args) - + self.namespace.acquire_write_lock() try: if accessed_only: data = dict(self.accessed_dict.items()) else: data = dict(self.items()) - + # Save the data if not data and 'session' in self.namespace: del self.namespace['session'] @@ -290,20 +290,20 @@ class Session(dict): self.namespace.release_write_lock() if self.is_new: self.request['set_cookie'] = True - + def revert(self): """Revert the session to its original state from its first access in the request""" self.clear() self.update(self.accessed_dict) - + # TODO: I think both these methods should be removed. They're from # the original mod_python code i was ripping off but they really # have no use here. def lock(self): """Locks this session against other processes/threads. This is automatic when load/save is called. - + ***use with caution*** and always with a corresponding 'unlock' inside a "finally:" block, as a stray lock typically cannot be unlocked without shutting down the whole application. @@ -324,10 +324,10 @@ class Session(dict): class CookieSession(Session): """Pure cookie-based session - + Options recognized when using cookie-based sessions are slightly more restricted than general sessions. - + ``key`` The name the cookie should be set to. ``timeout`` @@ -343,16 +343,16 @@ class CookieSession(Session): Domain to use for the cookie. ``secure`` Whether or not the cookie should only be sent over SSL. - + """ def __init__(self, request, key='beaker.session.id', timeout=None, cookie_expires=True, cookie_domain=None, encrypt_key=None, validate_key=None, secure=False, **kwargs): - + if not crypto.has_aes and encrypt_key: raise InvalidCryptoBackendError("No AES library is installed, can't generate " "encrypted cookie-only Session.") - + self.request = request self.key = key self.timeout = timeout @@ -363,24 +363,24 @@ class CookieSession(Session): self.secure = secure self._domain = cookie_domain self._path = '/' - + try: cookieheader = request['cookie'] except KeyError: cookieheader = '' - + if validate_key is None: raise BeakerException("No validate_key specified for Cookie only " "Session.") - + try: self.cookie = SignedCookie(validate_key, input=cookieheader) except Cookie.CookieError: self.cookie = SignedCookie(validate_key, input=None) - + self['_id'] = self._make_id() self.is_new = True - + # If we have a cookie, load it if self.key in self.cookie and self.cookie[self.key].value is not None: self.is_new = False @@ -393,11 +393,11 @@ class CookieSession(Session): self.clear() self.accessed_dict = self.copy() self._create_cookie() - + def created(self): return self['_creation_time'] created = property(created) - + def id(self): return self['_id'] id = property(id) @@ -405,19 +405,19 @@ class CookieSession(Session): def _set_domain(self, domain): self['_domain'] = domain self._domain = domain - + def _get_domain(self): return self._domain - + domain = property(_get_domain, _set_domain) - + def _set_path(self, path): self['_path'] = path self._path = path - + def _get_path(self): return self._path - + path = property(_get_path, _set_path) def _encrypt_data(self): @@ -431,7 +431,7 @@ class CookieSession(Session): else: data = pickle.dumps(self.copy(), 2) return b64encode(data) - + def _decrypt_data(self): """Bas64, decipher, then un-serialize the data for the session dict""" @@ -445,13 +445,13 @@ class CookieSession(Session): else: data = b64decode(self.cookie[self.key].value) return pickle.loads(data) - + def _make_id(self): return md5(md5( "%f%s%f%s" % (time.time(), id({}), random.random(), getpid()) ).hexdigest() ).hexdigest() - + def save(self, accessed_only=False): """Saves the data for this session to persistent storage""" if accessed_only and self.is_new: @@ -460,19 +460,19 @@ class CookieSession(Session): self.clear() self.update(self.accessed_dict) self._create_cookie() - + def expire(self): """Delete the 'expires' attribute on this Session, if any.""" - + self.pop('_expires', None) - + def _create_cookie(self): if '_creation_time' not in self: self['_creation_time'] = time.time() if '_id' not in self: self['_id'] = self._make_id() self['_accessed_time'] = time.time() - + if self.cookie_expires is not True: if self.cookie_expires is False: expires = datetime.fromtimestamp( 0x7FFFFFFF ) @@ -492,7 +492,7 @@ class CookieSession(Session): val = self._encrypt_data() if len(val) > 4064: raise BeakerException("Cookie value is too long to store") - + self.cookie[self.key] = val if '_domain' in self: self.cookie[self.key]['domain'] = self['_domain'] @@ -500,21 +500,21 @@ class CookieSession(Session): self.cookie[self.key]['domain'] = self._domain if self.secure: self.cookie[self.key]['secure'] = True - + self.cookie[self.key]['path'] = self.get('_path', '/') - + if expires: self.cookie[self.key]['expires'] = \ expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) self.request['cookie_out'] = self.cookie[self.key].output(header='') self.request['set_cookie'] = True - + def delete(self): """Delete the cookie, and clear the session""" # Send a delete cookie request self._delete_cookie() self.clear() - + def invalidate(self): """Clear the contents and start a new session""" self.delete() @@ -523,19 +523,19 @@ class CookieSession(Session): class SessionObject(object): """Session proxy/lazy creator - + This object proxies access to the actual session object, so that in the case that the session hasn't been used before, it will be setup. This avoid creating and loading the session from persistent storage unless its actually used during the request. - + """ def __init__(self, environ, **params): self.__dict__['_params'] = params self.__dict__['_environ'] = environ self.__dict__['_sess'] = None self.__dict__['_headers'] = [] - + def _session(self): """Lazy initial creation of session object""" if self.__dict__['_sess'] is None: @@ -549,35 +549,35 @@ class SessionObject(object): self.__dict__['_sess'] = Session(req, use_cookies=True, **params) return self.__dict__['_sess'] - + def __getattr__(self, attr): return getattr(self._session(), attr) - + def __setattr__(self, attr, value): setattr(self._session(), attr, value) - + def __delattr__(self, name): self._session().__delattr__(name) - + def __getitem__(self, key): return self._session()[key] - + def __setitem__(self, key, value): self._session()[key] = value - + def __delitem__(self, key): self._session().__delitem__(key) - + def __repr__(self): return self._session().__repr__() - + def __iter__(self): """Only works for proxying to a dict""" return iter(self._session().keys()) - + def __contains__(self, key): return self._session().has_key(key) - + def get_by_id(self, id): """Loads a session given a session ID""" params = self.__dict__['_params'] @@ -585,22 +585,22 @@ class SessionObject(object): if session.is_new: return None return session - + def save(self): self.__dict__['_dirty'] = True - + def delete(self): self.__dict__['_dirty'] = True self._session().delete() - + def persist(self): """Persist the session to the storage - + If its set to autosave, then the entire session will be saved regardless of if save() has been called. Otherwise, just the accessed time will be updated if save() was not called, or the session will be saved if save() was called. - + """ if self.__dict__['_params'].get('auto'): self._session().save() @@ -609,10 +609,10 @@ class SessionObject(object): self._session().save() else: self._session().save(accessed_only=True) - + def dirty(self): return self.__dict__.get('_dirty', False) - + def accessed(self): """Returns whether or not the session has been accessed""" return self.__dict__['_sess'] is not None diff --git a/module/lib/beaker/synchronization.py b/module/lib/beaker/synchronization.py index 6616ca78b..761303707 100644 --- a/module/lib/beaker/synchronization.py +++ b/module/lib/beaker/synchronization.py @@ -36,7 +36,7 @@ __all__ = ["file_synchronizer", "mutex_synchronizer", "null_synchronizer", class NameLock(object): """a proxy for an RLock object that is stored in a name based registry. - + Multiple threads can get a reference to the same RLock based on the name alone, and synchronize operations related to that name. @@ -115,7 +115,7 @@ class SynchronizerImpl(object): else: return self._state.get() state = property(state) - + def release_read_lock(self): state = self.state @@ -123,19 +123,19 @@ class SynchronizerImpl(object): raise LockError("lock is in writing state") if not state.reading: raise LockError("lock is not in reading state") - + if state.reentrantcount == 1: self.do_release_read_lock() state.reading = False state.reentrantcount -= 1 - + def acquire_read_lock(self, wait = True): state = self.state if state.writing: raise LockError("lock is in writing state") - + if state.reentrantcount == 0: x = self.do_acquire_read_lock(wait) if (wait or x): @@ -145,7 +145,7 @@ class SynchronizerImpl(object): elif state.reading: state.reentrantcount += 1 return True - + def release_write_lock(self): state = self.state @@ -159,15 +159,15 @@ class SynchronizerImpl(object): state.writing = False state.reentrantcount -= 1 - + release = release_write_lock - + def acquire_write_lock(self, wait = True): state = self.state if state.reading: raise LockError("lock is in reading state") - + if state.reentrantcount == 0: x = self.do_acquire_write_lock(wait) if (wait or x): @@ -182,13 +182,13 @@ class SynchronizerImpl(object): def do_release_read_lock(self): raise NotImplementedError() - + def do_acquire_read_lock(self): raise NotImplementedError() - + def do_release_write_lock(self): raise NotImplementedError() - + def do_acquire_write_lock(self): raise NotImplementedError() @@ -198,18 +198,18 @@ class FileSynchronizer(SynchronizerImpl): Adapted for Python/multithreads from Apache::Session::Lock::File, http://search.cpan.org/src/CWEST/Apache-Session-1.81/Session/Lock/File.pm - + This module does not unlink temporary files, because it interferes with proper locking. This can cause problems on certain systems (Linux) whose file systems (ext2) do not perform well with lots of files in one directory. To prevent this you should use a script to clean out old files from your lock directory. - + """ def __init__(self, identifier, lock_dir): super(FileSynchronizer, self).__init__() self._filedescriptor = util.ThreadLocal() - + if lock_dir is None: lock_dir = tempfile.gettempdir() else: @@ -224,14 +224,14 @@ class FileSynchronizer(SynchronizerImpl): def _filedesc(self): return self._filedescriptor.get() _filedesc = property(_filedesc) - + def _open(self, mode): filedescriptor = self._filedesc if filedescriptor is None: filedescriptor = os.open(self.filename, mode) self._filedescriptor.put(filedescriptor) return filedescriptor - + def do_acquire_read_lock(self, wait): filedescriptor = self._open(os.O_CREAT | os.O_RDONLY) if not wait: @@ -259,13 +259,13 @@ class FileSynchronizer(SynchronizerImpl): else: fcntl.flock(filedescriptor, fcntl.LOCK_EX) return True - + def do_release_read_lock(self): self._release_all_locks() - + def do_release_write_lock(self): self._release_all_locks() - + def _release_all_locks(self): filedescriptor = self._filedesc if filedescriptor is not None: @@ -276,7 +276,7 @@ class FileSynchronizer(SynchronizerImpl): class ConditionSynchronizer(SynchronizerImpl): """a synchronizer using a Condition.""" - + def __init__(self, identifier): super(ConditionSynchronizer, self).__init__() @@ -308,12 +308,12 @@ class ConditionSynchronizer(SynchronizerImpl): if not wait: return True - + def do_release_read_lock(self): self.condition.acquire() try: self.async -= 1 - + # check if we are the last asynchronous reader thread # out the door. if self.async == 0: @@ -326,13 +326,13 @@ class ConditionSynchronizer(SynchronizerImpl): "release_read_locks called") finally: self.condition.release() - + def do_acquire_write_lock(self, wait = True): self.condition.acquire() try: # here, we are not a synchronous reader, and after returning, # assuming waiting or immediate availability, we will be. - + if wait: # if another sync is working, wait while self.current_sync_operation is not None: @@ -342,7 +342,7 @@ class ConditionSynchronizer(SynchronizerImpl): # we dont want to wait, so forget it if self.current_sync_operation is not None: return False - + # establish ourselves as the current sync # this indicates to other read/write operations # that they should wait until this is None again @@ -359,7 +359,7 @@ class ConditionSynchronizer(SynchronizerImpl): return False finally: self.condition.release() - + if not wait: return True diff --git a/module/lib/beaker/util.py b/module/lib/beaker/util.py index 0bd82bb85..04c9617c5 100644 --- a/module/lib/beaker/util.py +++ b/module/lib/beaker/util.py @@ -16,7 +16,7 @@ import warnings import sys py3k = getattr(sys, 'py3kwarning', False) or sys.version_info >= (3, 0) -py24 = sys.version_info < (2, 5) +py24 = sys.version_info < (2,5) jython = sys.platform.startswith('java') if py3k or jython: @@ -45,6 +45,7 @@ def verify_directory(dir): if tries > 5: raise + def deprecated(message): def wrapper(fn): def deprecated_method(*args, **kargs): @@ -55,7 +56,7 @@ def deprecated(message): deprecated_method.__doc__ = "%s\n\n%s" % (message, fn.__doc__) return deprecated_method return wrapper - + class ThreadLocal(object): """stores a value on a per-thread basis""" @@ -63,25 +64,25 @@ class ThreadLocal(object): def __init__(self): self._tlocal = _tlocal() - + def put(self, value): self._tlocal.value = value - + def has(self): return hasattr(self._tlocal, 'value') - + def get(self, default=None): return getattr(self._tlocal, 'value', default) - + def remove(self): del self._tlocal.value - + class SyncDict(object): """ An efficient/threadsafe singleton map algorithm, a.k.a. "get a value based on this key, and create if not found or not valid" paradigm: - + exists && isvalid ? get : create Designed to work with weakref dictionaries to expect items @@ -95,7 +96,7 @@ class SyncDict(object): def __init__(self): self.mutex = _thread.allocate_lock() self.dict = {} - + def get(self, key, createfunc, *args, **kwargs): try: if self.has_key(key): @@ -124,7 +125,7 @@ class SyncDict(object): def has_key(self, key): return self.dict.has_key(key) - + def __contains__(self, key): return self.dict.__contains__(key) def __getitem__(self, key): @@ -145,30 +146,30 @@ class WeakValuedRegistry(SyncDict): sha1 = None def encoded_path(root, identifiers, extension = ".enc", depth = 3, digest_filenames=True): - + """Generate a unique file-accessible path from the given list of identifiers starting at the given root directory.""" ident = "_".join(identifiers) - + global sha1 if sha1 is None: from beaker.crypto import sha1 - + if digest_filenames: if py3k: ident = sha1(ident.encode('utf-8')).hexdigest() else: ident = sha1(ident).hexdigest() - + ident = os.path.basename(ident) tokens = [] for d in range(1, depth): tokens.append(ident[0:d]) - + dir = os.path.join(root, *tokens) verify_directory(dir) - + return os.path.join(dir, ident + extension) @@ -250,7 +251,7 @@ def coerce_cache_params(params): def parse_cache_config_options(config, include_defaults=True): """Parse configuration options and validate for use with the CacheManager""" - + # Load default cache options if include_defaults: options= dict(type='memory', data_dir=None, expire=None, @@ -263,11 +264,11 @@ def parse_cache_config_options(config, include_defaults=True): if key.startswith('cache.'): options[key[6:]] = val coerce_cache_params(options) - + # Set cache to enabled if not turned off if 'enabled' not in options: options['enabled'] = True - + # Configure region dict if regions are available regions = options.pop('regions', None) if regions: @@ -294,7 +295,7 @@ def func_namespace(func): if hasattr(func, 'im_func'): kls = func.im_class func = func.im_func - + if kls: return '%s.%s' % (kls.__module__, kls.__name__) else: diff --git a/module/lib/feedparser.py b/module/lib/feedparser.py index 8d3fec678..a746ed8f5 100644 --- a/module/lib/feedparser.py +++ b/module/lib/feedparser.py @@ -52,7 +52,7 @@ USER_AGENT = "UniversalFeedParser/%s +http://feedparser.org/" % __version__ # HTTP "Accept" header to send to servers when downloading feeds. If you don't # want to send an Accept header, set this to None. -ACCEPT_HEADER = "application/atom+xml, application/rdf+xml, application/rss+xml, application/x-netcdf, application/xml;q=0.9, text/xml;q=0.2,*/*;q=0.1" +ACCEPT_HEADER = "application/atom+xml,application/rdf+xml,application/rss+xml,application/x-netcdf,application/xml;q=0.9,text/xml;q=0.2,*/*;q=0.1" # List of preferred XML parsers, by SAX driver name. These will be tried first, # but if they're not installed, Python will keep searching through its own list @@ -89,7 +89,7 @@ try: except (NameError, AttributeError): import string _maketrans = string.maketrans - + # base64 support for Atom feeds that contain embedded binary data try: import base64, binascii @@ -167,7 +167,7 @@ try: _XML_AVAILABLE = 1 except: _XML_AVAILABLE = 0 - def _xmlescape(data, entities={}): + def _xmlescape(data,entities={}): data = data.replace('&', '&') data = data.replace('>', '>') data = data.replace('<', '<') @@ -203,7 +203,7 @@ except: import htmlentitydefs name2codepoint={} codepoint2name={} - for (name, codepoint) in htmlentitydefs.entitydefs.iteritems(): + for (name,codepoint) in htmlentitydefs.entitydefs.iteritems(): if codepoint.startswith('&#'): codepoint=unichr(int(codepoint[2:-1])) name2codepoint[name]=ord(codepoint) codepoint2name[ord(codepoint)]=name @@ -235,8 +235,8 @@ if sgmllib.endbracket.search(' <').start(0): # Overriding the built-in sgmllib.endbracket regex allows the # parser to find angle brackets embedded in element attributes. self.endbracket = re.compile('''([^'"<>]|"[^"]*"(?=>|/|\s|\w+=)|'[^']*'(?=>|/|\s|\w+=))*(?=[<>])|.*?(?=[<>])''') - def search(self, string, index=0): - match = self.endbracket.match(string, index) + def search(self,string,index=0): + match = self.endbracket.match(string,index) if match is not None: # Returning a new object in the calling thread's context # resolves a thread-safety. @@ -299,7 +299,7 @@ class FeedParserDict(UserDict): if key == 'category': return UserDict.__getitem__(self, 'tags')[0]['term'] if key == 'enclosures': - norel = lambda link: FeedParserDict([(name, value) for (name, value) in link.items() if name!='rel']) + norel = lambda link: FeedParserDict([(name,value) for (name,value) in link.items() if name!='rel']) return [norel(link) for link in UserDict.__getitem__(self, 'links') if link['rel']=='enclosure'] if key == 'license': for link in UserDict.__getitem__(self, 'links'): @@ -334,7 +334,7 @@ class FeedParserDict(UserDict): if not self.has_key(key): self[key] = value return self[key] - + def has_key(self, key): try: return hasattr(self, key) or UserDict.__contains__(self, key) @@ -343,7 +343,7 @@ class FeedParserDict(UserDict): # This alias prevents the 2to3 tool from changing the semantics of the # __contains__ function below and exhausting the maximum recursion depth __has_key = has_key - + def __getattr__(self, key): try: return self.__dict__[key] @@ -378,22 +378,22 @@ def _ebcdic_to_ascii(s): global _ebcdic_to_ascii_map if not _ebcdic_to_ascii_map: emap = ( - 0, 1, 2, 3, 156, 9, 134, 127, 151, 141, 142, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 157, 133, 8, 135, 24, 25, 146, 143, 28, 29, 30, 31, - 128, 129, 130, 131, 132, 10, 23, 27, 136, 137, 138, 139, 140, 5, 6, 7, - 144, 145, 22, 147, 148, 149, 150, 4, 152, 153, 154, 155, 20, 21, 158, 26, - 32, 160, 161, 162, 163, 164, 165, 166, 167, 168, 91, 46, 60, 40, 43, 33, - 38, 169, 170, 171, 172, 173, 174, 175, 176, 177, 93, 36, 42, 41, 59, 94, - 45, 47, 178, 179, 180, 181, 182, 183, 184, 185, 124, 44, 37, 95, 62, 63, - 186, 187, 188, 189, 190, 191, 192, 193, 194, 96, 58, 35, 64, 39, 61, 34, - 195, 97, 98, 99, 100, 101, 102, 103, 104, 105, 196, 197, 198, 199, 200, 201, - 202, 106, 107, 108, 109, 110, 111, 112, 113, 114, 203, 204, 205, 206, 207, 208, - 209, 126, 115, 116, 117, 118, 119, 120, 121, 122, 210, 211, 212, 213, 214, 215, - 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, - 123, 65, 66, 67, 68, 69, 70, 71, 72, 73, 232, 233, 234, 235, 236, 237, - 125, 74, 75, 76, 77, 78, 79, 80, 81, 82, 238, 239, 240, 241, 242, 243, - 92, 159, 83, 84, 85, 86, 87, 88, 89, 90, 244, 245, 246, 247, 248, 249, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 250, 251, 252, 253, 254, 255 + 0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15, + 16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31, + 128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7, + 144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26, + 32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33, + 38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94, + 45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63, + 186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34, + 195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,201, + 202,106,107,108,109,110,111,112,113,114,203,204,205,206,207,208, + 209,126,115,116,117,118,119,120,121,122,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231, + 123,65,66,67,68,69,70,71,72,73,232,233,234,235,236,237, + 125,74,75,76,77,78,79,80,81,82,238,239,240,241,242,243, + 92,159,83,84,85,86,87,88,89,90,244,245,246,247,248,249, + 48,49,50,51,52,53,54,55,56,57,250,251,252,253,254,255 ) _ebcdic_to_ascii_map = _maketrans( \ _l2bytes(range(256)), _l2bytes(emap)) @@ -451,7 +451,7 @@ class _FeedParserMixin: 'http://purl.org/atom/ns#': '', 'http://www.w3.org/2005/Atom': '', 'http://purl.org/rss/1.0/modules/rss091#': '', - + 'http://webns.net/mvcb/': 'admin', 'http://purl.org/rss/1.0/modules/aggregation/': 'ag', 'http://purl.org/rss/1.0/modules/annotate/': 'annotate', @@ -508,7 +508,7 @@ class _FeedParserMixin: can_contain_relative_uris = ['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description'] can_contain_dangerous_markup = ['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description'] html_types = ['text/html', 'application/xhtml+xml'] - + def __init__(self, baseuri=None, baselang=None, encoding='utf-8'): if _debug: sys.stderr.write('initializing FeedParser\n') if not self._matchnamespaces: @@ -554,7 +554,7 @@ class _FeedParserMixin: # strict xml parsers do -- account for this difference if isinstance(self, _LooseFeedParser): attrs = [(k, v.replace('&', '&')) for k, v in attrs] - + # track xml:base and xml:lang attrsD = dict(attrs) baseuri = attrsD.get('xml:base', attrsD.get('base')) or self.baseuri @@ -582,7 +582,7 @@ class _FeedParserMixin: self.lang = lang self.basestack.append(self.baseuri) self.langstack.append(lang) - + # track namespaces for prefix, uri in attrs: if prefix.startswith('xmlns:'): @@ -620,7 +620,7 @@ class _FeedParserMixin: self.intextinput = 0 if (not prefix) and tag not in ('title', 'link', 'description', 'url', 'href', 'width', 'height'): self.inimage = 0 - + # call special handler (if defined) or default handler methodname = '_start_' + prefix + suffix try: @@ -754,7 +754,7 @@ class _FeedParserMixin: elif contentType == 'xhtml': contentType = 'application/xhtml+xml' return contentType - + def trackNamespace(self, prefix, uri): loweruri = uri.lower() if (prefix, loweruri) == (None, 'http://my.netscape.com/rdf/simple/0.9/') and not self.version: @@ -775,12 +775,12 @@ class _FeedParserMixin: def resolveURI(self, uri): return _urljoin(self.baseuri or '', uri) - + def decodeEntities(self, element, data): return data def strattrs(self, attrs): - return ''.join([' %s="%s"' % (t[0], _xmlescape(t[1], {'"':'"'})) for t in attrs]) + return ''.join([' %s="%s"' % (t[0],_xmlescape(t[1],{'"':'"'})) for t in attrs]) def push(self, element, expectingText): self.elementstack.append([element, expectingText, []]) @@ -788,7 +788,7 @@ class _FeedParserMixin: def pop(self, element, stripWhitespace=1): if not self.elementstack: return if self.elementstack[-1][0] != element: return - + element, expectingText, pieces = self.elementstack.pop() if self.version == 'atom10' and self.contentparams.get('type','text') == 'application/xhtml+xml': @@ -796,9 +796,9 @@ class _FeedParserMixin: # only if all the remaining content is nested underneath it. # This means that the divs would be retained in the following: # <div>foo</div><div>bar</div> - while pieces and len(pieces)> 1 and not pieces[-1].strip(): + while pieces and len(pieces)>1 and not pieces[-1].strip(): del pieces[-1] - while pieces and len(pieces)> 1 and not pieces[0].strip(): + while pieces and len(pieces)>1 and not pieces[0].strip(): del pieces[0] if pieces and (pieces[0] == '<div>' or pieces[0].startswith('<div ')) and pieces[-1]=='</div>': depth = 0 @@ -833,11 +833,11 @@ class _FeedParserMixin: # In Python 3, base64 takes and outputs bytes, not str # This may not be the most correct way to accomplish this output = _base64decode(output.encode('utf-8')).decode('utf-8') - + # resolve relative URIs if (element in self.can_be_relative_uri) and output: output = self.resolveURI(output) - + # decode entities within embedded markup if not self.contentparams.get('base64', 0): output = self.decodeEntities(element, output) @@ -860,7 +860,7 @@ class _FeedParserMixin: if is_htmlish and RESOLVE_RELATIVE_URIS: if element in self.can_contain_relative_uris: output = _resolveRelativeURIs(output, self.baseuri, self.encoding, self.contentparams.get('type', 'text/html')) - + # parse microformats # (must do this before sanitizing because some microformats # rely on elements that we sanitize) @@ -876,7 +876,7 @@ class _FeedParserMixin: vcard = mfresults.get('vcard') if vcard: self._getContext()['vcard'] = vcard - + # sanitize embedded markup if is_htmlish and SANITIZE_HTML: if element in self.can_contain_dangerous_markup: @@ -906,7 +906,7 @@ class _FeedParserMixin: if element == 'title' and self.hasTitle: return output - + # store output in appropriate place(s) if self.inentry and not self.insource: if element == 'content': @@ -962,7 +962,7 @@ class _FeedParserMixin: self.incontent -= 1 self.contentparams.clear() return value - + # a number of elements in a number of RSS variants are nominally plain # text, but this is routinely ignored. This is an attempt to detect # the most common cases. As false positives often result in silent @@ -993,7 +993,7 @@ class _FeedParserMixin: prefix = self.namespacemap.get(prefix, prefix) name = prefix + ':' + suffix return name - + def _getAttribute(self, attrsD, name): return attrsD.get(self._mapToStandardPrefix(name)) @@ -1021,7 +1021,7 @@ class _FeedParserMixin: pass attrsD['href'] = href return attrsD - + def _save(self, key, value, overwrite=False): context = self._getContext() if overwrite: @@ -1046,7 +1046,7 @@ class _FeedParserMixin: self.version = 'rss20' else: self.version = 'rss' - + def _start_dlhottitles(self, attrsD): self.version = 'hotrss' @@ -1064,7 +1064,7 @@ class _FeedParserMixin: self._start_link({}) self.elementstack[-1][-1] = attrsD['href'] self._end_link() - + def _start_feed(self, attrsD): self.infeed = 1 versionmap = {'0.1': 'atom01', @@ -1081,7 +1081,7 @@ class _FeedParserMixin: def _end_channel(self): self.infeed = 0 _end_feed = _end_channel - + def _start_image(self, attrsD): context = self._getContext() if not self.inentry: @@ -1089,7 +1089,7 @@ class _FeedParserMixin: self.inimage = 1 self.hasTitle = 0 self.push('image', 0) - + def _end_image(self): self.pop('image') self.inimage = 0 @@ -1101,7 +1101,7 @@ class _FeedParserMixin: self.hasTitle = 0 self.push('textinput', 0) _start_textInput = _start_textinput - + def _end_textinput(self): self.pop('textinput') self.intextinput = 0 @@ -1271,7 +1271,7 @@ class _FeedParserMixin: else: author, email = context.get(key), None if not author: return - emailmatch = re.search(r'''(([a-zA-Z0-9\_\-\.\+]+)@((\[[0-9]{1, 3}\.[0-9]{1, 3}\.[0-9]{1, 3}\.)|(([a-zA-Z0-9\-]+\.)+))([a-zA-Z]{2, 4}|[0-9]{1, 3})(\]?))(\?subject=\S+)?''', author) + emailmatch = re.search(r'''(([a-zA-Z0-9\_\-\.\+]+)@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.)|(([a-zA-Z0-9\-]+\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\]?))(\?subject=\S+)?''', author) if emailmatch: email = emailmatch.group(0) # probably a better way to do the following, but it passes all the tests @@ -1301,7 +1301,7 @@ class _FeedParserMixin: self.popContent('subtitle') _end_tagline = _end_subtitle _end_itunes_subtitle = _end_subtitle - + def _start_rights(self, attrsD): self.pushContent('rights', attrsD, 'text/plain', 1) _start_dc_rights = _start_rights @@ -1399,7 +1399,7 @@ class _FeedParserMixin: attrsD['rel']='license' if value: attrsD['href']=value context.setdefault('links', []).append(attrsD) - + def _start_creativecommons_license(self, attrsD): self.push('license', 1) _start_creativeCommons_license = _start_creativecommons_license @@ -1420,7 +1420,7 @@ class _FeedParserMixin: value = FeedParserDict({'relationships': relationships, 'href': href, 'name': name}) if value not in xfn: xfn.append(value) - + def _addTag(self, term, scheme, label): context = self._getContext() tags = context.setdefault('tags', []) @@ -1438,7 +1438,7 @@ class _FeedParserMixin: self.push('category', 1) _start_dc_subject = _start_category _start_keywords = _start_category - + def _start_media_category(self, attrsD): attrsD.setdefault('scheme', 'http://search.yahoo.com/mrss/category_schema') self._start_category(attrsD) @@ -1446,11 +1446,11 @@ class _FeedParserMixin: def _end_itunes_keywords(self): for term in self.pop('itunes_keywords').split(): self._addTag(term, 'http://www.itunes.com/', None) - + def _start_itunes_category(self, attrsD): self._addTag(attrsD.get('text'), 'http://www.itunes.com/', None) self.push('category', 1) - + def _end_category(self): value = self.pop('category') if not value: return @@ -1467,7 +1467,7 @@ class _FeedParserMixin: def _start_cloud(self, attrsD): self._getContext()['cloud'] = FeedParserDict(attrsD) - + def _start_link(self, attrsD): attrsD.setdefault('rel', 'alternate') if attrsD['rel'] == 'self': @@ -1568,7 +1568,7 @@ class _FeedParserMixin: context = self._getContext() if context.has_key('generator_detail'): context['generator_detail']['name'] = value - + def _start_admin_generatoragent(self, attrsD): self.push('generator', 1) value = self._getAttribute(attrsD, 'rdf:resource') @@ -1583,7 +1583,7 @@ class _FeedParserMixin: if value: self.elementstack[-1][2].append(value) self.pop('errorreportsto') - + def _start_summary(self, attrsD): context = self._getContext() if context.has_key('summary'): @@ -1601,13 +1601,13 @@ class _FeedParserMixin: self.popContent(self._summaryKey or 'summary') self._summaryKey = None _end_itunes_summary = _end_summary - + def _start_enclosure(self, attrsD): attrsD = self._itsAnHrefDamnIt(attrsD) context = self._getContext() attrsD['rel']='enclosure' context.setdefault('links', []).append(FeedParserDict(attrsD)) - + def _start_source(self, attrsD): if 'url' in attrsD: # This means that we're processing a source element from an RSS 2.0 feed @@ -1659,7 +1659,7 @@ class _FeedParserMixin: if attrsD.get('href'): self._getContext()['image'] = FeedParserDict({'href': attrsD.get('href')}) _start_itunes_link = _start_itunes_image - + def _end_itunes_block(self): value = self.pop('itunes_block', 0) self._getContext()['itunes_block'] = (value == 'yes') and 1 or 0 @@ -1718,12 +1718,12 @@ if _XML_AVAILABLE: self.bozo = 0 self.exc = None self.decls = {} - + def startPrefixMapping(self, prefix, uri): self.trackNamespace(prefix, uri) if uri == 'http://www.w3.org/1999/xlink': self.decls['xmlns:'+prefix] = uri - + def startElementNS(self, name, qname, attrs): namespace, localname = name lowernamespace = str(namespace or '').lower() @@ -1756,7 +1756,7 @@ if _XML_AVAILABLE: if prefix: localname = prefix.lower() + ':' + localname elif namespace and not qname: #Expat - for name, value in self.namespacesInUse.items(): + for name,value in self.namespacesInUse.items(): if name and value == namespace: localname = name + ':' + localname break @@ -1786,7 +1786,7 @@ if _XML_AVAILABLE: if prefix: localname = prefix + ':' + localname elif namespace and not qname: #Expat - for name, value in self.namespacesInUse.items(): + for name,value in self.namespacesInUse.items(): if name and value == namespace: localname = name + ':' + localname break @@ -1827,10 +1827,10 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): else: return '<' + tag + '></' + tag + '>' - def parse_starttag(self, i): + def parse_starttag(self,i): j=sgmllib.SGMLParser.parse_starttag(self, i) if self._type == 'application/xhtml+xml': - if j > 2 and self.rawdata[j-2:j]=='/>': + if j>2 and self.rawdata[j-2:j]=='/>': self.unknown_endtag(self.lasttag) return j @@ -1902,7 +1902,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): # called for each character reference, e.g. for ' ', ref will be '160' # Reconstruct the original character reference. if ref.startswith('x'): - value = unichr(int(ref[1:], 16)) + value = unichr(int(ref[1:],16)) else: value = unichr(int(ref)) @@ -1910,7 +1910,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:]) else: self.pieces.append('&#%(ref)s;' % locals()) - + def handle_entityref(self, ref): # called for each entity reference, e.g. for '©', ref will be 'copy' # Reconstruct the original entity reference. @@ -1925,12 +1925,12 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): # Store the original text verbatim. if _debug: sys.stderr.write('_BaseHTMLProcessor, handle_data, text=%s\n' % text) self.pieces.append(text) - + def handle_comment(self, text): # called for each HTML comment, e.g. <!-- insert Javascript code here --> # Reconstruct the original comment. self.pieces.append('<!--%(text)s-->' % locals()) - + def handle_pi(self, text): # called for each processing instruction, e.g. <?instruction> # Reconstruct original processing instruction. @@ -1942,7 +1942,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): # "http://www.w3.org/TR/html4/loose.dtd"> # Reconstruct original DOCTYPE self.pieces.append('<!%(text)s>' % locals()) - + _new_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9:]*\s*').match def _scan_name(self, i, declstartpos): rawdata = self.rawdata @@ -1998,9 +1998,9 @@ class _LooseFeedParser(_FeedParserMixin, _BaseHTMLProcessor): data = data.replace('"', '"') data = data.replace(''', "'") return data - + def strattrs(self, attrs): - return ''.join([' %s="%s"' % (n, v.replace('"','"')) for n, v in attrs]) + return ''.join([' %s="%s"' % (n,v.replace('"','"')) for n,v in attrs]) class _MicroformatsParser: STRING = 1 @@ -2010,7 +2010,7 @@ class _MicroformatsParser: EMAIL = 5 known_xfn_relationships = ['contact', 'acquaintance', 'friend', 'met', 'co-worker', 'coworker', 'colleague', 'co-resident', 'coresident', 'neighbor', 'child', 'parent', 'sibling', 'brother', 'sister', 'spouse', 'wife', 'husband', 'kin', 'relative', 'muse', 'crush', 'date', 'sweetheart', 'me'] - known_binary_extensions = ['zip','rar','exe','gz','tar','tgz','tbz2','bz2','z','7z','dmg','img','sit','sitx','hqx','deb','rpm','bz2','jar','rar','iso','bin','msi','mp2','mp3','ogg','ogm','mp4','m4v','m4a','avi','wma','wmv'] + known_binary_extensions = ['zip','rar','exe','gz','tar','tgz','tbz2','bz2','z','7z','dmg','img','sit','sitx','hqx','deb','rpm','bz2','jar','rar','iso','bin','msi','mp2','mp3','ogg','ogm','mp4','m4v','m4a','avi','wma','wmv'] def __init__(self, data, baseuri, encoding): self.document = BeautifulSoup.BeautifulSoup(data) @@ -2022,12 +2022,12 @@ class _MicroformatsParser: self.enclosures = [] self.xfn = [] self.vcard = None - + def vcardEscape(self, s): if type(s) in (type(''), type(u'')): s = s.replace(',', '\\,').replace(';', '\\;').replace('\n', '\\n') return s - + def vcardFold(self, s): s = re.sub(';+$', '', s) sFolded = '' @@ -2043,14 +2043,14 @@ class _MicroformatsParser: def normalize(self, s): return re.sub(r'\s+', ' ', s).strip() - + def unique(self, aList): results = [] for element in aList: if element not in results: results.append(element) return results - + def toISO8601(self, dt): return time.strftime('%Y-%m-%dT%H:%M:%SZ', dt) @@ -2140,21 +2140,21 @@ class _MicroformatsParser: def findVCards(self, elmRoot, bAgentParsing=0): sVCards = '' - + if not bAgentParsing: arCards = self.getPropertyValue(elmRoot, 'vcard', bAllowMultiple=1) else: arCards = [elmRoot] - + for elmCard in arCards: arLines = [] - + def processSingleString(sProperty): sValue = self.getPropertyValue(elmCard, sProperty, self.STRING, bAutoEscape=1).decode(self.encoding) if sValue: arLines.append(self.vcardFold(sProperty.upper() + ':' + sValue)) return sValue or u'' - + def processSingleURI(sProperty): sValue = self.getPropertyValue(elmCard, sProperty, self.URI) if sValue: @@ -2177,7 +2177,7 @@ class _MicroformatsParser: if sContentType: sContentType = ';TYPE=' + sContentType.upper() arLines.append(self.vcardFold(sProperty.upper() + sEncoding + sContentType + sValueKey + ':' + sValue)) - + def processTypeValue(sProperty, arDefaultType, arForceType=None): arResults = self.getPropertyValue(elmCard, sProperty, bAllowMultiple=1) for elmResult in arResults: @@ -2189,7 +2189,7 @@ class _MicroformatsParser: sValue = self.getPropertyValue(elmResult, 'value', self.EMAIL, 0) if sValue: arLines.append(self.vcardFold(sProperty.upper() + ';TYPE=' + ','.join(arType) + ':' + sValue)) - + # AGENT # must do this before all other properties because it is destructive # (removes nested class="vcard" nodes so they don't interfere with @@ -2208,10 +2208,10 @@ class _MicroformatsParser: sAgentValue = self.getPropertyValue(elmAgent, 'value', self.URI, bAutoEscape=1); if sAgentValue: arLines.append(self.vcardFold('AGENT;VALUE=uri:' + sAgentValue)) - + # FN (full name) sFN = processSingleString('fn') - + # N (name) elmName = self.getPropertyValue(elmCard, 'n') if elmName: @@ -2237,25 +2237,25 @@ class _MicroformatsParser: arLines.append(self.vcardFold('N:' + arNames[0] + ';' + arNames[1])) else: arLines.append(self.vcardFold('N:' + arNames[1] + ';' + arNames[0])) - + # SORT-STRING sSortString = self.getPropertyValue(elmCard, 'sort-string', self.STRING, bAutoEscape=1) if sSortString: arLines.append(self.vcardFold('SORT-STRING:' + sSortString)) - + # NICKNAME arNickname = self.getPropertyValue(elmCard, 'nickname', self.STRING, 1, 1) if arNickname: arLines.append(self.vcardFold('NICKNAME:' + ','.join(arNickname))) - + # PHOTO processSingleURI('photo') - + # BDAY dtBday = self.getPropertyValue(elmCard, 'bday', self.DATE) if dtBday: arLines.append(self.vcardFold('BDAY:' + self.toISO8601(dtBday))) - + # ADR (address) arAdr = self.getPropertyValue(elmCard, 'adr', bAllowMultiple=1) for elmAdr in arAdr: @@ -2277,38 +2277,38 @@ class _MicroformatsParser: sRegion + ';' + sPostalCode + ';' + sCountryName)) - + # LABEL processTypeValue('label', ['intl','postal','parcel','work']) - + # TEL (phone number) processTypeValue('tel', ['voice']) - + # EMAIL processTypeValue('email', ['internet'], ['internet']) - + # MAILER processSingleString('mailer') - + # TZ (timezone) processSingleString('tz') - + # GEO (geographical information) elmGeo = self.getPropertyValue(elmCard, 'geo') if elmGeo: sLatitude = self.getPropertyValue(elmGeo, 'latitude', self.STRING, 0, 1) sLongitude = self.getPropertyValue(elmGeo, 'longitude', self.STRING, 0, 1) arLines.append(self.vcardFold('GEO:' + sLatitude + ';' + sLongitude)) - + # TITLE processSingleString('title') - + # ROLE processSingleString('role') # LOGO processSingleURI('logo') - + # ORG (organization) elmOrg = self.getPropertyValue(elmCard, 'org') if elmOrg: @@ -2322,39 +2322,39 @@ class _MicroformatsParser: else: arOrganizationUnit = self.getPropertyValue(elmOrg, 'organization-unit', self.STRING, 1, 1) arLines.append(self.vcardFold('ORG:' + sOrganizationName + ';' + ';'.join(arOrganizationUnit))) - + # CATEGORY arCategory = self.getPropertyValue(elmCard, 'category', self.STRING, 1, 1) + self.getPropertyValue(elmCard, 'categories', self.STRING, 1, 1) if arCategory: arLines.append(self.vcardFold('CATEGORIES:' + ','.join(arCategory))) - + # NOTE processSingleString('note') - + # REV processSingleString('rev') - + # SOUND processSingleURI('sound') - + # UID processSingleString('uid') - + # URL processSingleURI('url') - + # CLASS processSingleString('class') - + # KEY processSingleURI('key') - + if arLines: arLines = [u'BEGIN:vCard',u'VERSION:3.0'] + arLines + [u'END:vCard'] sVCards += u'\n'.join(arLines) + u'\n' - + return sVCards.strip() - + def isProbablyDownloadable(self, elm): attrsD = elm.attrMap if not attrsD.has_key('href'): return 0 @@ -2453,7 +2453,7 @@ class _RelativeURIResolver(_BaseHTMLProcessor): def resolveURI(self, uri): return _makeSafeAbsoluteURI(_urljoin(self.baseuri, uri.strip())) - + def unknown_starttag(self, tag, attrs): if _debug: sys.stderr.write('tag: [%s] with attributes: [%s]\n' % (tag, str(attrs))) @@ -2545,7 +2545,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): 'transparent', 'underline', 'white', 'yellow'] valid_css_values = re.compile('^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|' + - '\d{0, 2}\.?\d{0, 2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$') + '\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$') mathml_elements = ['annotation', 'annotation-xml', 'maction', 'math', 'merror', 'mfenced', 'mfrac', 'mi', 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded', @@ -2612,7 +2612,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): self.unacceptablestack = 0 self.mathmlOK = 0 self.svgOK = 0 - + def unknown_starttag(self, tag, attrs): acceptable_attributes = self.acceptable_attributes keymap = {} @@ -2651,27 +2651,27 @@ class _HTMLSanitizer(_BaseHTMLProcessor): self.svg_elements = lower self.svg_elem_map = dict([(a.lower(),a) for a in mix]) acceptable_attributes = self.svg_attributes - tag = self.svg_elem_map.get(tag, tag) + tag = self.svg_elem_map.get(tag,tag) keymap = self.svg_attr_map elif not tag in self.acceptable_elements: return # declare xlink namespace, if needed if self.mathmlOK or self.svgOK: - if filter(lambda (n, v): n.startswith('xlink:'), attrs): + if filter(lambda (n,v): n.startswith('xlink:'),attrs): if not ('xmlns:xlink','http://www.w3.org/1999/xlink') in attrs: attrs.append(('xmlns:xlink','http://www.w3.org/1999/xlink')) clean_attrs = [] for key, value in self.normalize_attrs(attrs): if key in acceptable_attributes: - key=keymap.get(key, key) - clean_attrs.append((key, value)) + key=keymap.get(key,key) + clean_attrs.append((key,value)) elif key=='style': clean_value = self.sanitize_style(value) - if clean_value: clean_attrs.append((key, clean_value)) + if clean_value: clean_attrs.append((key,clean_value)) _BaseHTMLProcessor.unknown_starttag(self, tag, clean_attrs) - + def unknown_endtag(self, tag): if not tag in self.acceptable_elements: if tag in self.unacceptable_elements_with_end_tag: @@ -2679,7 +2679,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): if self.mathmlOK and tag in self.mathml_elements: if tag == 'math' and self.mathmlOK: self.mathmlOK -= 1 elif self.svgOK and tag in self.svg_elements: - tag = self.svg_elem_map.get(tag, tag) + tag = self.svg_elem_map.get(tag,tag) if tag == 'svg' and self.svgOK: self.svgOK -= 1 else: return @@ -2705,7 +2705,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): if re.sub("\s*[-\w]+\s*:\s*[^:;]*;?", '', style).strip(): return '' clean = [] - for prop, value in re.findall("([-\w]+)\s*:\s*([^:;]*)",style): + for prop,value in re.findall("([-\w]+)\s*:\s*([^:;]*)",style): if not value: continue if prop.lower() in self.acceptable_css_properties: clean.append(prop + ': ' + value + ';') @@ -2791,7 +2791,7 @@ class _FeedURLHandler(urllib2.HTTPDigestAuthHandler, urllib2.HTTPRedirectHandler http_error_300 = http_error_302 http_error_303 = http_error_302 http_error_307 = http_error_302 - + def http_error_401(self, req, fp, code, msg, headers): # Check if # - server requires digest auth, AND @@ -2875,7 +2875,7 @@ def _open_resource(url_file_stream_or_string, etag, modified, agent, referrer, h # iri support try: - if isinstance(url_file_stream_or_string, unicode): + if isinstance(url_file_stream_or_string,unicode): url_file_stream_or_string = url_file_stream_or_string.encode('idna').decode('utf-8') else: url_file_stream_or_string = url_file_stream_or_string.decode('utf-8').encode('idna').decode('utf-8') @@ -2890,7 +2890,7 @@ def _open_resource(url_file_stream_or_string, etag, modified, agent, referrer, h return opener.open(request) finally: opener.close() # JohnD - + # try to open with native open function (if url_file_stream_or_string is a filename) try: return open(url_file_stream_or_string, 'rb') @@ -2932,7 +2932,7 @@ def _build_urllib2_request(url, agent, etag, modified, referrer, auth, request_h if ACCEPT_HEADER: request.add_header('Accept', ACCEPT_HEADER) # use this for whatever -- cookies, special headers, etc - # [('Cookie','Something'), ('x-special-header','Another Value')] + # [('Cookie','Something'),('x-special-header','Another Value')] for header_name, header_value in request_headers.items(): request.add_header(header_name, header_value) request.add_header('A-IM', 'feed') # RFC 3229 support @@ -2942,7 +2942,7 @@ _date_handlers = [] def registerDateHandler(func): '''Register a date handler function (takes string, returns 9-tuple date in GMT)''' _date_handlers.insert(0, func) - + # ISO-8601 date parsing routines written by Fazal Majid. # The ISO 8601 standard is very convoluted and irregular - a full ISO 8601 # parser is beyond the scope of feedparser and would be a worthwhile addition @@ -3055,7 +3055,7 @@ def _parse_date_iso8601(dateString): # Many implementations have bugs, but we'll pretend they don't. return time.localtime(time.mktime(tuple(tm))) registerDateHandler(_parse_date_iso8601) - + # 8-bit date handling routines written by ytrewq1. _korean_year = u'\ub144' # b3e2 in euc-kr _korean_month = u'\uc6d4' # bff9 in euc-kr @@ -3374,7 +3374,7 @@ def _getCharacterEncoding(http_headers, xml_data): http_headers is a dictionary xml_data is a raw string (not Unicode) - + This is so much trickier than it sounds, it's not even funny. According to RFC 3023 ('XML Media Types'), if the HTTP Content-Type is application/xml, application/*+xml, @@ -3391,14 +3391,14 @@ def _getCharacterEncoding(http_headers, xml_data): Furthermore, discussion on the atom-syntax mailing list with the author of RFC 3023 leads me to the conclusion that any document served with a Content-Type of text/* and no charset parameter - must be treated as us-ascii. (We now do this.) And also that it - must always be flagged as non-well-formed. (We now do this too.) - + must be treated as us-ascii. (We now do this.) And also that it + must always be flagged as non-well-formed. (We now do this too.) + If Content-Type is unspecified (input was local file or non-HTTP source) or unrecognized (server just got it totally wrong), then go by the encoding given in the XML prefix of the document and default to 'iso-8859-1' as per the HTTP specification (RFC 2616). - + Then, assuming we didn't find a character encoding in the HTTP headers (and the HTTP Content-type allowed us to look in the body), we need to sniff the first few bytes of the XML data and try to determine @@ -3508,7 +3508,7 @@ def _getCharacterEncoding(http_headers, xml_data): if true_encoding.lower() == 'gb2312': true_encoding = 'gb18030' return true_encoding, http_encoding, xml_encoding, sniffed_xml_encoding, acceptable_content_type - + def _toUTF8(data, encoding): '''Changes an XML data stream on the fly to specify a new encoding @@ -3570,8 +3570,8 @@ def _stripDoctype(data): ''' start = re.search(_s2bytes('<\w'), data) start = start and start.start() or -1 - head, data = data[:start+1], data[start+1:] - + head,data = data[:start+1], data[start+1:] + entity_pattern = re.compile(_s2bytes(r'^\s*<!ENTITY([^>]*?)>'), re.MULTILINE) entity_results=entity_pattern.findall(head) head = entity_pattern.sub(_s2bytes(''), head) @@ -3587,16 +3587,16 @@ def _stripDoctype(data): replacement=_s2bytes('') if len(doctype_results)==1 and entity_results: safe_pattern=re.compile(_s2bytes('\s+(\w+)\s+"(&#\w+;|[^&"]*)"')) - safe_entities=filter(lambda e: safe_pattern.match(e), entity_results) + safe_entities=filter(lambda e: safe_pattern.match(e),entity_results) if safe_entities: replacement=_s2bytes('<!DOCTYPE feed [\n <!ENTITY') + _s2bytes('>\n <!ENTITY ').join(safe_entities) + _s2bytes('>\n]>') data = doctype_pattern.sub(replacement, head) + data return version, data, dict(replacement and [(k.decode('utf-8'), v.decode('utf-8')) for k, v in safe_pattern.findall(replacement)]) - + def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, referrer=None, handlers=[], request_headers={}, response_headers={}): '''Parse a feed from a URL, file, stream, or string. - + request_headers, if given, is a dict from http header name to value to add to the request; this overrides internally generated values. ''' @@ -3837,7 +3837,7 @@ class TextSerializer(Serializer): stream.write('\n') except: pass - + class PprintSerializer(Serializer): def write(self, stream=sys.stdout): if self.results.has_key('href'): @@ -3845,7 +3845,7 @@ class PprintSerializer(Serializer): from pprint import pprint pprint(self.results, stream) stream.write('\n') - + if __name__ == '__main__': try: from optparse import OptionParser diff --git a/module/lib/jinja2/filters.py b/module/lib/jinja2/filters.py index 97f06ed7d..d1848e434 100644 --- a/module/lib/jinja2/filters.py +++ b/module/lib/jinja2/filters.py @@ -101,7 +101,7 @@ def do_xmlattr(_eval_ctx, d, autospace=True): .. sourcecode:: html+jinja - <ul{{{'class': 'my_list', 'missing': none, + <ul{{ {'class': 'my_list', 'missing': none, 'id': 'list-%d'|format(variable)}|xmlattr }}> ... </ul> diff --git a/module/lib/jinja2/nodes.py b/module/lib/jinja2/nodes.py index f6cdc0d89..6446c70ea 100644 --- a/module/lib/jinja2/nodes.py +++ b/module/lib/jinja2/nodes.py @@ -507,7 +507,7 @@ class Keyword(Helper): class CondExpr(Expr): - """A conditional expression (inline if expression). (``{{ + """A conditional expression (inline if expression). (``{{ foo if bar else baz }}``) """ fields = ('test', 'expr1', 'expr2') diff --git a/module/lib/jinja2/utils.py b/module/lib/jinja2/utils.py index 186fc6c44..7b77b8eb7 100644 --- a/module/lib/jinja2/utils.py +++ b/module/lib/jinja2/utils.py @@ -269,7 +269,7 @@ def urlize(text, trim_url_limit=None, nofollow=False): attribute. """ trim_url = lambda x, limit=trim_url_limit: limit is not None \ - and (x[:limit] + (len(x) >= limit and '...' + and (x[:limit] + (len(x) >=limit and '...' or '')) or x words = _word_split_re.split(unicode(escape(text))) nofollow_attr = nofollow and ' rel="nofollow"' or '' diff --git a/module/lib/simplejson/__init__.py b/module/lib/simplejson/__init__.py index 368d4202b..ef5c0db48 100644 --- a/module/lib/simplejson/__init__.py +++ b/module/lib/simplejson/__init__.py @@ -31,8 +31,8 @@ Encoding basic Python object hierarchies:: Compact encoding:: >>> import simplejson as json - >>> json.dumps([1, 2, 3, {'4': 5, '6': 7}], separators=(',',':')) - '[1, 2, 3, {"4": 5, "6": 7}]' + >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) + '[1,2,3,{"4":5,"6":7}]' Pretty printing:: @@ -189,7 +189,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, If *namedtuple_as_object* is true (default: ``True``), :class:`tuple` subclasses with ``_asdict()`` methods will be encoded as JSON objects. - + If *tuple_as_array* is true (default: ``True``), :class:`tuple` (and subclasses) will be encoded as JSON arrays. @@ -268,7 +268,7 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, If *namedtuple_as_object* is true (default: ``True``), :class:`tuple` subclasses with ``_asdict()`` methods will be encoded as JSON objects. - + If *tuple_as_array* is true (default: ``True``), :class:`tuple` (and subclasses) will be encoded as JSON arrays. diff --git a/module/lib/simplejson/encoder.py b/module/lib/simplejson/encoder.py index 5560c55e6..5ec7440f1 100644 --- a/module/lib/simplejson/encoder.py +++ b/module/lib/simplejson/encoder.py @@ -157,7 +157,7 @@ class JSONEncoder(object): If namedtuple_as_object is true (the default), tuple subclasses with ``_asdict()`` methods will be encoded as JSON objects. - + If tuple_as_array is true (the default), tuple (and subclasses) will be encoded as JSON arrays. """ diff --git a/module/lib/thrift/TSerialization.py b/module/lib/thrift/TSerialization.py index 05f88f838..b19f98aa8 100644 --- a/module/lib/thrift/TSerialization.py +++ b/module/lib/thrift/TSerialization.py @@ -31,3 +31,4 @@ def deserialize(base, buf, protocol_factory = TBinaryProtocol.TBinaryProtocolFac protocol = protocol_factory.getProtocol(transport) base.read(protocol) return base + diff --git a/module/lib/thrift/Thrift.py b/module/lib/thrift/Thrift.py index 8c270ea28..1d271fcff 100644 --- a/module/lib/thrift/Thrift.py +++ b/module/lib/thrift/Thrift.py @@ -74,8 +74,8 @@ class TException(Exception): """Base class for all thrift exceptions.""" - # BaseException.message is deprecated in Python v[2.6, 3.0) - if (2, 6, 0) <= sys.version_info < (3, 0): + # BaseException.message is deprecated in Python v[2.6,3.0) + if (2,6,0) <= sys.version_info < (3,0): def _get_message(self): return self._message def _set_message(self, message): diff --git a/module/lib/thrift/protocol/TBase.py b/module/lib/thrift/protocol/TBase.py index 08e154026..e675c7dc0 100644 --- a/module/lib/thrift/protocol/TBase.py +++ b/module/lib/thrift/protocol/TBase.py @@ -43,10 +43,10 @@ class TBase(object): if my_val != other_val: return False return True - + def __ne__(self, other): return not (self == other) - + def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) @@ -63,9 +63,10 @@ class TExceptionBase(Exception): # old style class so python2.4 can raise exceptions derived from this # This can't inherit from TBase because of that limitation. __slots__ = [] - + __repr__ = TBase.__repr__.im_func __eq__ = TBase.__eq__.im_func __ne__ = TBase.__ne__.im_func read = TBase.read.im_func write = TBase.write.im_func + diff --git a/module/lib/thrift/protocol/TBinaryProtocol.py b/module/lib/thrift/protocol/TBinaryProtocol.py index 0cc44178c..50c6aa896 100644 --- a/module/lib/thrift/protocol/TBinaryProtocol.py +++ b/module/lib/thrift/protocol/TBinaryProtocol.py @@ -239,7 +239,7 @@ class TBinaryProtocolAccelerated(TBinaryProtocol): our C module to do the encoding, bypassing this object entirely. We inherit from TBinaryProtocol so that the normal TBinaryProtocol encoding can happen if the fastbinary module doesn't work for some - reason. (TODO(dreiss): Make this happen sanely in more cases.) + reason. (TODO(dreiss): Make this happen sanely in more cases.) In order to take advantage of the C module, just use TBinaryProtocolAccelerated instead of TBinaryProtocol. diff --git a/module/lib/thrift/protocol/TProtocol.py b/module/lib/thrift/protocol/TProtocol.py index 595ca5ae6..7338ff68a 100644 --- a/module/lib/thrift/protocol/TProtocol.py +++ b/module/lib/thrift/protocol/TProtocol.py @@ -202,7 +202,7 @@ class TProtocolBase: # tuple of: ( 'reader method' name, is_container boolean, 'writer_method' name ) _TTYPE_HANDLERS = ( - (None, None, False), # 0 == TType, STOP + (None, None, False), # 0 == TType,STOP (None, None, False), # 1 == TType.VOID # TODO: handle void? ('readBool', 'writeBool', False), # 2 == TType.BOOL ('readByte', 'writeByte', False), # 3 == TType.BYTE and I08 @@ -279,7 +279,7 @@ class TProtocolBase: obj = obj_class() obj.read(self) return obj - + def readContainerMap(self, spec): results = dict() key_ttype, key_spec = spec[0], spec[1] @@ -298,7 +298,7 @@ class TProtocolBase: v_val = val_reader() else: v_val = self.readFieldByTType(val_ttype, val_spec) - # this raises a TypeError with unhashable keys types. i.e. d=dict(); d[[0, 1]] = 2 fails + # this raises a TypeError with unhashable keys types. i.e. d=dict(); d[[0,1]] = 2 fails results[k_val] = v_val self.readMapEnd() return results @@ -401,3 +401,4 @@ class TProtocolBase: class TProtocolFactory: def getProtocol(self, trans): pass + diff --git a/module/lib/thrift/server/TNonblockingServer.py b/module/lib/thrift/server/TNonblockingServer.py index 26d6af38d..ea348a0b6 100644 --- a/module/lib/thrift/server/TNonblockingServer.py +++ b/module/lib/thrift/server/TNonblockingServer.py @@ -81,7 +81,7 @@ def socket_exception(func): class Connection: """Basic class is represented connection. - + It can be in state: WAIT_LEN --- connection is reading request len. WAIT_MESSAGE --- connection is reading request. @@ -102,7 +102,7 @@ class Connection: def _read_len(self): """Reads length of request. - + It's really paranoic routine and it may be replaced by self.socket.recv(4).""" read = self.socket.recv(4 - len(self.message)) @@ -162,14 +162,14 @@ class Connection: @locked def ready(self, all_ok, message): """Callback function for switching state and waking up main thread. - + This function is the only function witch can be called asynchronous. - + The ready can switch Connection to three states: WAIT_LEN if request was oneway. SEND_ANSWER if request was processed in normal way. CLOSED if request throws unexpected exception. - + The one wakes up main thread. """ assert self.status == WAIT_PROCESS @@ -243,12 +243,12 @@ class TNonblockingServer: def wake_up(self): """Wake up main thread. - + The server usualy waits in select call in we should terminate one. The simplest way is using socketpair. - + Select always wait to read from the first socket of socketpair. - + In this case, we can just write anything to the second socket from socketpair.""" self._write.send('1') @@ -265,10 +265,10 @@ class TNonblockingServer: if connection.is_closed(): del self.clients[i] return select.select(readable, writable, readable) - + def handle(self): """Handle requests. - + WARNING! You must call prepare BEFORE calling handle. """ assert self.prepared, "You have to call prepare before handle" @@ -302,7 +302,7 @@ class TNonblockingServer: self.tasks.put([None, None, None, None, None]) self.socket.close() self.prepared = False - + def serve(self): """Serve forever.""" self.prepare() diff --git a/module/lib/thrift/server/TProcessPoolServer.py b/module/lib/thrift/server/TProcessPoolServer.py index 254f25340..7ed814a88 100644 --- a/module/lib/thrift/server/TProcessPoolServer.py +++ b/module/lib/thrift/server/TProcessPoolServer.py @@ -122,3 +122,4 @@ class TProcessPoolServer(TServer): self.stopCondition.acquire() self.stopCondition.notify() self.stopCondition.release() + diff --git a/module/lib/thrift/server/TServer.py b/module/lib/thrift/server/TServer.py index 5b5fd6813..8456e2d40 100644 --- a/module/lib/thrift/server/TServer.py +++ b/module/lib/thrift/server/TServer.py @@ -270,3 +270,5 @@ class TForkingServer(TServer): self.children.remove(pid) else: break + + diff --git a/module/lib/thrift/transport/TTransport.py b/module/lib/thrift/transport/TTransport.py index 2b2e99eac..12e51a9bf 100644 --- a/module/lib/thrift/transport/TTransport.py +++ b/module/lib/thrift/transport/TTransport.py @@ -18,7 +18,7 @@ # from cStringIO import StringIO -from struct import pack, unpack +from struct import pack,unpack from thrift.Thrift import TException class TTransportException(TException): @@ -127,7 +127,7 @@ class TBufferedTransportFactory: return buffered -class TBufferedTransport(TTransportBase, CReadableTransport): +class TBufferedTransport(TTransportBase,CReadableTransport): """Class that wraps another transport and buffers its I/O. diff --git a/module/lib/thrift/transport/TTwisted.py b/module/lib/thrift/transport/TTwisted.py index 141e299ca..b6dcb4e0b 100644 --- a/module/lib/thrift/transport/TTwisted.py +++ b/module/lib/thrift/transport/TTwisted.py @@ -79,7 +79,7 @@ class ThriftClientProtocol(basic.Int32StringReceiver): self.started.callback(self.client) def connectionLost(self, reason=connectionDone): - for k, v in self.client._reqs.iteritems(): + for k,v in self.client._reqs.iteritems(): tex = TTransport.TTransportException( type=TTransport.TTransportException.END_OF_FILE, message='Connection closed') diff --git a/module/lib/thrift/transport/TZlibTransport.py b/module/lib/thrift/transport/TZlibTransport.py index 89fa96f61..784d4e1e0 100644 --- a/module/lib/thrift/transport/TZlibTransport.py +++ b/module/lib/thrift/transport/TZlibTransport.py @@ -30,14 +30,14 @@ from TTransport import TTransportBase, CReadableTransport class TZlibTransportFactory(object): ''' Factory transport that builds zlib compressed transports. - + This factory caches the last single client/transport that it was passed and returns the same TZlibTransport object that was created. - + This caching means the TServer class will get the _same_ transport object for both input and output transports from this factory. (For non-threaded scenarios only, since the cache only holds one object) - + The purpose of this caching is to allocate only one TZlibTransport where only one is really needed (since it must have separate read/write buffers), and makes the statistics from getCompSavings() and getCompRatio() @@ -52,11 +52,11 @@ class TZlibTransportFactory(object): '''Wrap a transport , trans, with the TZlibTransport compressed transport class, returning a new transport to the caller. - + @param compresslevel: The zlib compression level, ranging from 0 (no compression) to 9 (best compression). Defaults to 9. @type compresslevel: int - + This method returns a TZlibTransport which wraps the passed C{trans} TTransport derived instance. ''' @@ -83,7 +83,7 @@ class TZlibTransport(TTransportBase, CReadableTransport): ''' Create a new TZlibTransport, wrapping C{trans}, another TTransport derived object. - + @param trans: A thrift transport object, i.e. a TSocket() object. @type trans: TTransport @param compresslevel: The zlib compression level, ranging @@ -125,19 +125,19 @@ class TZlibTransport(TTransportBase, CReadableTransport): def getCompRatio(self): ''' - Get the current measured compression ratios (in, out) from + Get the current measured compression ratios (in,out) from this transport. - + Returns a tuple of: (inbound_compression_ratio, outbound_compression_ratio) - + The compression ratios are computed as: compressed / uncompressed E.g., data that compresses by 10x will have a ratio of: 0.10 and data that compresses to half of ts original size will have a ratio of 0.5 - + None is returned if no bytes have yet been processed in a particular direction. ''' @@ -152,10 +152,10 @@ class TZlibTransport(TTransportBase, CReadableTransport): ''' Get the current count of saved bytes due to data compression. - + Returns a tuple of: (inbound_saved_bytes, outbound_saved_bytes) - + Note: if compression is actually expanding your data (only likely with very tiny thrift objects), then the values returned will be negative. |