diff options
Diffstat (limited to 'module/lib')
49 files changed, 5531 insertions, 726 deletions
diff --git a/module/lib/jinja2/__init__.py b/module/lib/jinja2/__init__.py index f944e11b6..a4f7e9c4e 100644 --- a/module/lib/jinja2/__init__.py +++ b/module/lib/jinja2/__init__.py @@ -27,11 +27,7 @@ :license: BSD, see LICENSE for more details. """ __docformat__ = 'restructuredtext en' -try: - __version__ = __import__('pkg_resources') \ - .get_distribution('Jinja2').version -except: - __version__ = 'unknown' +__version__ = '2.7.3' # high level interface from jinja2.environment import Environment, Template diff --git a/module/lib/jinja2/_compat.py b/module/lib/jinja2/_compat.py new file mode 100644 index 000000000..8fa8a49a0 --- /dev/null +++ b/module/lib/jinja2/_compat.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +""" + jinja2._compat + ~~~~~~~~~~~~~~ + + Some py2/py3 compatibility support based on a stripped down + version of six so we don't have to depend on a specific version + of it. + + :copyright: Copyright 2013 by the Jinja team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import sys + +PY2 = sys.version_info[0] == 2 +PYPY = hasattr(sys, 'pypy_translation_info') +_identity = lambda x: x + + +if not PY2: + unichr = chr + range_type = range + text_type = str + string_types = (str,) + + iterkeys = lambda d: iter(d.keys()) + itervalues = lambda d: iter(d.values()) + iteritems = lambda d: iter(d.items()) + + import pickle + from io import BytesIO, StringIO + NativeStringIO = StringIO + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + + ifilter = filter + imap = map + izip = zip + intern = sys.intern + + implements_iterator = _identity + implements_to_string = _identity + encode_filename = _identity + get_next = lambda x: x.__next__ + +else: + unichr = unichr + text_type = unicode + range_type = xrange + string_types = (str, unicode) + + iterkeys = lambda d: d.iterkeys() + itervalues = lambda d: d.itervalues() + iteritems = lambda d: d.iteritems() + + import cPickle as pickle + from cStringIO import StringIO as BytesIO, StringIO + NativeStringIO = BytesIO + + exec('def reraise(tp, value, tb=None):\n raise tp, value, tb') + + from itertools import imap, izip, ifilter + intern = intern + + def implements_iterator(cls): + cls.next = cls.__next__ + del cls.__next__ + return cls + + def implements_to_string(cls): + cls.__unicode__ = cls.__str__ + cls.__str__ = lambda x: x.__unicode__().encode('utf-8') + return cls + + get_next = lambda x: x.next + + def encode_filename(filename): + if isinstance(filename, unicode): + return filename.encode('utf-8') + return filename + +try: + next = next +except NameError: + def next(it): + return it.next() + + +def with_metaclass(meta, *bases): + # This requires a bit of explanation: the basic idea is to make a + # dummy metaclass for one level of class instanciation that replaces + # itself with the actual metaclass. Because of internal type checks + # we also need to make sure that we downgrade the custom metaclass + # for one level to something closer to type (that's why __call__ and + # __init__ comes back from type etc.). + # + # This has the advantage over six.with_metaclass in that it does not + # introduce dummy classes into the final MRO. + class metaclass(meta): + __call__ = type.__call__ + __init__ = type.__init__ + def __new__(cls, name, this_bases, d): + if this_bases is None: + return type.__new__(cls, name, (), d) + return meta(name, bases, d) + return metaclass('temporary_class', None, {}) + + +try: + from collections import Mapping as mapping_types +except ImportError: + import UserDict + mapping_types = (UserDict.UserDict, UserDict.DictMixin, dict) + + +# common types. These do exist in the special types module too which however +# does not exist in IronPython out of the box. Also that way we don't have +# to deal with implementation specific stuff here +class _C(object): + def method(self): pass +def _func(): + yield None +function_type = type(_func) +generator_type = type(_func()) +method_type = type(_C().method) +code_type = type(_C.method.__code__) +try: + raise TypeError() +except TypeError: + _tb = sys.exc_info()[2] + traceback_type = type(_tb) + frame_type = type(_tb.tb_frame) + + +try: + from urllib.parse import quote_from_bytes as url_quote +except ImportError: + from urllib import quote as url_quote + + +try: + from thread import allocate_lock +except ImportError: + try: + from threading import Lock as allocate_lock + except ImportError: + from dummy_thread import allocate_lock diff --git a/module/lib/jinja2/_markupsafe/_bundle.py b/module/lib/jinja2/_markupsafe/_bundle.py deleted file mode 100644 index e694faf23..000000000 --- a/module/lib/jinja2/_markupsafe/_bundle.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- -""" - jinja2._markupsafe._bundle - ~~~~~~~~~~~~~~~~~~~~~~~~~~ - - This script pulls in markupsafe from a source folder and - bundles it with Jinja2. It does not pull in the speedups - module though. - - :copyright: Copyright 2010 by the Jinja team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" -import sys -import os -import re - - -def rewrite_imports(lines): - for idx, line in enumerate(lines): - new_line = re.sub(r'(import|from)\s+markupsafe\b', - r'\1 jinja2._markupsafe', line) - if new_line != line: - lines[idx] = new_line - - -def main(): - if len(sys.argv) != 2: - print 'error: only argument is path to markupsafe' - sys.exit(1) - basedir = os.path.dirname(__file__) - markupdir = sys.argv[1] - for filename in os.listdir(markupdir): - if filename.endswith('.py'): - f = open(os.path.join(markupdir, filename)) - try: - lines = list(f) - finally: - f.close() - rewrite_imports(lines) - f = open(os.path.join(basedir, filename), 'w') - try: - for line in lines: - f.write(line) - finally: - f.close() - - -if __name__ == '__main__': - main() diff --git a/module/lib/jinja2/_markupsafe/tests.py b/module/lib/jinja2/_markupsafe/tests.py deleted file mode 100644 index c1ce3943a..000000000 --- a/module/lib/jinja2/_markupsafe/tests.py +++ /dev/null @@ -1,80 +0,0 @@ -import gc -import unittest -from jinja2._markupsafe import Markup, escape, escape_silent - - -class MarkupTestCase(unittest.TestCase): - - def test_markup_operations(self): - # adding two strings should escape the unsafe one - unsafe = '<script type="application/x-some-script">alert("foo");</script>' - safe = Markup('<em>username</em>') - assert unsafe + safe == unicode(escape(unsafe)) + unicode(safe) - - # string interpolations are safe to use too - assert Markup('<em>%s</em>') % '<bad user>' == \ - '<em><bad user></em>' - assert Markup('<em>%(username)s</em>') % { - 'username': '<bad user>' - } == '<em><bad user></em>' - - # an escaped object is markup too - assert type(Markup('foo') + 'bar') is Markup - - # and it implements __html__ by returning itself - x = Markup("foo") - assert x.__html__() is x - - # it also knows how to treat __html__ objects - class Foo(object): - def __html__(self): - return '<em>awesome</em>' - def __unicode__(self): - return 'awesome' - assert Markup(Foo()) == '<em>awesome</em>' - assert Markup('<strong>%s</strong>') % Foo() == \ - '<strong><em>awesome</em></strong>' - - # escaping and unescaping - assert escape('"<>&\'') == '"<>&'' - assert Markup("<em>Foo & Bar</em>").striptags() == "Foo & Bar" - assert Markup("<test>").unescape() == "<test>" - - def test_all_set(self): - import jinja2._markupsafe as markup - for item in markup.__all__: - getattr(markup, item) - - def test_escape_silent(self): - assert escape_silent(None) == Markup() - assert escape(None) == Markup(None) - assert escape_silent('<foo>') == Markup(u'<foo>') - - -class MarkupLeakTestCase(unittest.TestCase): - - def test_markup_leaks(self): - counts = set() - for count in xrange(20): - for item in xrange(1000): - escape("foo") - escape("<foo>") - escape(u"foo") - escape(u"<foo>") - counts.add(len(gc.get_objects())) - assert len(counts) == 1, 'ouch, c extension seems to leak objects' - - -def suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(MarkupTestCase)) - - # this test only tests the c extension - if not hasattr(escape, 'func_code'): - suite.addTest(unittest.makeSuite(MarkupLeakTestCase)) - - return suite - - -if __name__ == '__main__': - unittest.main(defaultTest='suite') diff --git a/module/lib/jinja2/_stringdefs.py b/module/lib/jinja2/_stringdefs.py index 1161b7f4a..da5830e9f 100644 --- a/module/lib/jinja2/_stringdefs.py +++ b/module/lib/jinja2/_stringdefs.py @@ -13,6 +13,8 @@ :license: BSD, see LICENSE for details. """ +from jinja2._compat import unichr + Cc = u'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f' Cf = u'\xad\u0600\u0601\u0602\u0603\u06dd\u070f\u17b4\u17b5\u200b\u200c\u200d\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2060\u2061\u2062\u2063\u206a\u206b\u206c\u206d\u206e\u206f\ufeff\ufff9\ufffa\ufffb' diff --git a/module/lib/jinja2/bccache.py b/module/lib/jinja2/bccache.py index 1e2236c3a..2d28ab8b2 100644 --- a/module/lib/jinja2/bccache.py +++ b/module/lib/jinja2/bccache.py @@ -15,20 +15,46 @@ :license: BSD. """ from os import path, listdir +import os +import stat +import sys +import errno import marshal import tempfile -import cPickle as pickle import fnmatch -from cStringIO import StringIO -try: - from hashlib import sha1 -except ImportError: - from sha import new as sha1 +from hashlib import sha1 from jinja2.utils import open_if_exists +from jinja2._compat import BytesIO, pickle, PY2, text_type -bc_version = 1 -bc_magic = 'j2'.encode('ascii') + pickle.dumps(bc_version, 2) +# marshal works better on 3.x, one hack less required +if not PY2: + marshal_dump = marshal.dump + marshal_load = marshal.load +else: + + def marshal_dump(code, f): + if isinstance(f, file): + marshal.dump(code, f) + else: + f.write(marshal.dumps(code)) + + def marshal_load(f): + if isinstance(f, file): + return marshal.load(f) + return marshal.loads(f.read()) + + +bc_version = 2 + +# magic version used to only change with new jinja versions. With 2.6 +# we change this to also take Python version changes into account. The +# reason for this is that Python tends to segfault if fed earlier bytecode +# versions because someone thought it would be a good idea to reuse opcodes +# or make Python incompatible with earlier versions. +bc_magic = 'j2'.encode('ascii') + \ + pickle.dumps(bc_version, 2) + \ + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1]) class Bucket(object): @@ -62,12 +88,7 @@ class Bucket(object): if self.checksum != checksum: self.reset() return - # now load the code. Because marshal is not able to load - # from arbitrary streams we have to work around that - if isinstance(f, file): - self.code = marshal.load(f) - else: - self.code = marshal.loads(f.read()) + self.code = marshal_load(f) def write_bytecode(self, f): """Dump the bytecode into the file or file like object passed.""" @@ -75,18 +96,15 @@ class Bucket(object): raise TypeError('can\'t write empty bucket') f.write(bc_magic) pickle.dump(self.checksum, f, 2) - if isinstance(f, file): - marshal.dump(self.code, f) - else: - f.write(marshal.dumps(self.code)) + marshal_dump(self.code, f) def bytecode_from_string(self, string): """Load bytecode from a string.""" - self.load_bytecode(StringIO(string)) + self.load_bytecode(BytesIO(string)) def bytecode_to_string(self): """Return the bytecode as string.""" - out = StringIO() + out = BytesIO() self.write_bytecode(out) return out.getvalue() @@ -144,9 +162,10 @@ class BytecodeCache(object): """Returns the unique hash key for this template name.""" hash = sha1(name.encode('utf-8')) if filename is not None: - if isinstance(filename, unicode): + filename = '|' + filename + if isinstance(filename, text_type): filename = filename.encode('utf-8') - hash.update('|' + filename) + hash.update(filename) return hash.hexdigest() def get_source_checksum(self, source): @@ -173,7 +192,9 @@ class FileSystemBytecodeCache(BytecodeCache): two arguments: The directory where the cache items are stored and a pattern string that is used to build the filename. - If no directory is specified the system temporary items folder is used. + If no directory is specified a default cache directory is selected. On + Windows the user's temp directory is used, on UNIX systems a directory + is created for the user in the system temp directory. The pattern can be used to have multiple separate caches operate on the same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` @@ -186,10 +207,38 @@ class FileSystemBytecodeCache(BytecodeCache): def __init__(self, directory=None, pattern='__jinja2_%s.cache'): if directory is None: - directory = tempfile.gettempdir() + directory = self._get_default_cache_dir() self.directory = directory self.pattern = pattern + def _get_default_cache_dir(self): + tmpdir = tempfile.gettempdir() + + # On windows the temporary directory is used specific unless + # explicitly forced otherwise. We can just use that. + if os.name == 'nt': + return tmpdir + if not hasattr(os, 'getuid'): + raise RuntimeError('Cannot determine safe temp directory. You ' + 'need to explicitly provide one.') + + dirname = '_jinja2-cache-%d' % os.getuid() + actual_dir = os.path.join(tmpdir, dirname) + try: + os.mkdir(actual_dir, stat.S_IRWXU) # 0o700 + except OSError as e: + if e.errno != errno.EEXIST: + raise + + actual_dir_stat = os.lstat(actual_dir) + if actual_dir_stat.st_uid != os.getuid() \ + or not stat.S_ISDIR(actual_dir_stat.st_mode) \ + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU: + raise RuntimeError('Temporary directory \'%s\' has an incorrect ' + 'owner, permissions, or type.' % actual_dir) + + return actual_dir + def _get_cache_filename(self, bucket): return path.join(self.directory, self.pattern % bucket.key) @@ -261,15 +310,26 @@ class MemcachedBytecodeCache(BytecodeCache): This bytecode cache does not support clearing of used items in the cache. The clear method is a no-operation function. + + .. versionadded:: 2.7 + Added support for ignoring memcache errors through the + `ignore_memcache_errors` parameter. """ - def __init__(self, client, prefix='jinja2/bytecode/', timeout=None): + def __init__(self, client, prefix='jinja2/bytecode/', timeout=None, + ignore_memcache_errors=True): self.client = client self.prefix = prefix self.timeout = timeout + self.ignore_memcache_errors = ignore_memcache_errors def load_bytecode(self, bucket): - code = self.client.get(self.prefix + bucket.key) + try: + code = self.client.get(self.prefix + bucket.key) + except Exception: + if not self.ignore_memcache_errors: + raise + code = None if code is not None: bucket.bytecode_from_string(code) @@ -277,4 +337,8 @@ class MemcachedBytecodeCache(BytecodeCache): args = (self.prefix + bucket.key, bucket.bytecode_to_string()) if self.timeout is not None: args += (self.timeout,) - self.client.set(*args) + try: + self.client.set(*args) + except Exception: + if not self.ignore_memcache_errors: + raise diff --git a/module/lib/jinja2/compiler.py b/module/lib/jinja2/compiler.py index 57641596a..75a60b8d2 100644 --- a/module/lib/jinja2/compiler.py +++ b/module/lib/jinja2/compiler.py @@ -8,14 +8,16 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD, see LICENSE for more details. """ -from cStringIO import StringIO from itertools import chain from copy import deepcopy +from keyword import iskeyword as is_python_keyword from jinja2 import nodes from jinja2.nodes import EvalContext -from jinja2.visitor import NodeVisitor, NodeTransformer +from jinja2.visitor import NodeVisitor from jinja2.exceptions import TemplateAssertionError -from jinja2.utils import Markup, concat, escape, is_python_keyword, next +from jinja2.utils import Markup, concat, escape +from jinja2._compat import range_type, next, text_type, string_types, \ + iteritems, NativeStringIO, imap operators = { @@ -29,14 +31,6 @@ operators = { 'notin': 'not in' } -try: - exec '(0 if 0 else 0)' -except SyntaxError: - have_condexpr = False -else: - have_condexpr = True - - # what method to iterate over items do we want to use for dict iteration # in generated code? on 2.x let's go with iteritems, on 3.x with items if hasattr(dict, 'iteritems'): @@ -51,7 +45,11 @@ def unoptimize_before_dead_code(): def f(): if 0: dummy(x) return f -unoptimize_before_dead_code = bool(unoptimize_before_dead_code().func_closure) + +# The getattr is necessary for pypy which does not set this attribute if +# no closure is on the function +unoptimize_before_dead_code = bool( + getattr(unoptimize_before_dead_code(), '__closure__', None)) def generate(node, environment, name, filename, stream=None, @@ -69,8 +67,8 @@ def has_safe_repr(value): """Does the node have a safe representation?""" if value is None or value is NotImplemented or value is Ellipsis: return True - if isinstance(value, (bool, int, long, float, complex, basestring, - xrange, Markup)): + if isinstance(value, (bool, int, float, complex, range_type, + Markup) + string_types): return True if isinstance(value, (tuple, list, set, frozenset)): for item in value: @@ -78,7 +76,7 @@ def has_safe_repr(value): return False return True elif isinstance(value, dict): - for key, value in value.iteritems(): + for key, value in iteritems(value): if not has_safe_repr(key): return False if not has_safe_repr(value): @@ -127,12 +125,10 @@ class Identifiers(object): self.undeclared.discard(name) self.declared.add(name) - def is_declared(self, name, local_only=False): + def is_declared(self, name): """Check if a name is declared in this or an outer scope.""" if name in self.declared_locally or name in self.declared_parameter: return True - if local_only: - return False return name in self.declared def copy(self): @@ -193,12 +189,12 @@ class Frame(object): rv.identifiers.__dict__.update(self.identifiers.__dict__) return rv - def inspect(self, nodes, hard_scope=False): + def inspect(self, nodes): """Walk the node and check for identifiers. If the scope is hard (eg: enforce on a python level) overrides from outer scopes are tracked differently. """ - visitor = FrameIdentifierVisitor(self.identifiers, hard_scope) + visitor = FrameIdentifierVisitor(self.identifiers) for node in nodes: visitor.visit(node) @@ -275,9 +271,8 @@ class UndeclaredNameVisitor(NodeVisitor): class FrameIdentifierVisitor(NodeVisitor): """A visitor for `Frame.inspect`.""" - def __init__(self, identifiers, hard_scope): + def __init__(self, identifiers): self.identifiers = identifiers - self.hard_scope = hard_scope def visit_Name(self, node): """All assignments to names go through this function.""" @@ -286,7 +281,7 @@ class FrameIdentifierVisitor(NodeVisitor): elif node.ctx == 'param': self.identifiers.declared_parameter.add(node.name) elif node.ctx == 'load' and not \ - self.identifiers.is_declared(node.name, self.hard_scope): + self.identifiers.is_declared(node.name): self.identifiers.undeclared.add(node.name) def visit_If(self, node): @@ -371,7 +366,7 @@ class CodeGenerator(NodeVisitor): def __init__(self, environment, name, filename, stream=None, defer_init=False): if stream is None: - stream = StringIO() + stream = NativeStringIO() self.environment = environment self.name = name self.filename = filename @@ -545,7 +540,7 @@ class CodeGenerator(NodeVisitor): self.write(', ') self.visit(kwarg, frame) if extra_kwargs is not None: - for key, value in extra_kwargs.iteritems(): + for key, value in iteritems(extra_kwargs): self.write(', %s=%s' % (key, value)) if node.dyn_args: self.write(', *') @@ -561,7 +556,7 @@ class CodeGenerator(NodeVisitor): self.visit(kwarg.value, frame) self.write(', ') if extra_kwargs is not None: - for key, value in extra_kwargs.iteritems(): + for key, value in iteritems(extra_kwargs): self.write('%r: %s, ' % (key, value)) if node.dyn_kwargs is not None: self.write('}, **') @@ -628,7 +623,7 @@ class CodeGenerator(NodeVisitor): def pop_scope(self, aliases, frame): """Restore all aliases and delete unused variables.""" - for name, alias in aliases.iteritems(): + for name, alias in iteritems(aliases): self.writeline('l_%s = %s' % (name, alias)) to_delete = set() for name in frame.identifiers.declared_locally: @@ -658,7 +653,7 @@ class CodeGenerator(NodeVisitor): children = node.iter_child_nodes() children = list(children) func_frame = frame.inner() - func_frame.inspect(children, hard_scope=True) + func_frame.inspect(children) # variables that are undeclared (accessed before declaration) and # declared locally *and* part of an outside scope raise a template @@ -666,16 +661,16 @@ class CodeGenerator(NodeVisitor): # it without aliasing all the variables. # this could be fixed in Python 3 where we have the nonlocal # keyword or if we switch to bytecode generation - overriden_closure_vars = ( + overridden_closure_vars = ( func_frame.identifiers.undeclared & func_frame.identifiers.declared & (func_frame.identifiers.declared_locally | func_frame.identifiers.declared_parameter) ) - if overriden_closure_vars: + if overridden_closure_vars: self.fail('It\'s not possible to set and access variables ' 'derived from an outer scope! (affects: %s)' % - ', '.join(sorted(overriden_closure_vars)), node.lineno) + ', '.join(sorted(overridden_closure_vars)), node.lineno) # remove variables from a closure from the frame's undeclared # identifiers. @@ -830,7 +825,7 @@ class CodeGenerator(NodeVisitor): self.outdent(2 + (not self.has_known_extends)) # at this point we now have the blocks collected and can visit them too. - for name, block in self.blocks.iteritems(): + for name, block in iteritems(self.blocks): block_frame = Frame(eval_ctx) block_frame.inspect(block.body) block_frame.block = name @@ -897,12 +892,13 @@ class CodeGenerator(NodeVisitor): self.indent() self.writeline('raise TemplateRuntimeError(%r)' % 'extended multiple times') - self.outdent() # if we have a known extends already we don't need that code here # as we know that the template execution will end here. if self.has_known_extends: raise CompilerExit() + else: + self.outdent() self.writeline('parent_template = environment.get_template(', node) self.visit(node.template, frame) @@ -933,7 +929,7 @@ class CodeGenerator(NodeVisitor): func_name = 'get_or_select_template' if isinstance(node.template, nodes.Const): - if isinstance(node.template.value, basestring): + if isinstance(node.template.value, string_types): func_name = 'get_template' elif isinstance(node.template.value, (tuple, list)): func_name = 'select_template' @@ -1035,7 +1031,7 @@ class CodeGenerator(NodeVisitor): discarded_names[0]) else: self.writeline('context.exported_vars.difference_' - 'update((%s))' % ', '.join(map(repr, discarded_names))) + 'update((%s))' % ', '.join(imap(repr, discarded_names))) def visit_For(self, node, frame): # when calculating the nodes for the inner frame we have to exclude @@ -1063,7 +1059,7 @@ class CodeGenerator(NodeVisitor): # otherwise we set up a buffer and add a function def else: - self.writeline('def loop(reciter, loop_render_func):', node) + self.writeline('def loop(reciter, loop_render_func, depth=0):', node) self.indent() self.buffer(loop_frame) aliases = {} @@ -1071,6 +1067,7 @@ class CodeGenerator(NodeVisitor): # make sure the loop variable is a special one and raise a template # assertion error if a loop tries to write to loop if extended_loop: + self.writeline('l_loop = missing') loop_frame.identifiers.add_special('loop') for name in node.find_all(nodes.Name): if name.ctx == 'store' and name.name == 'loop': @@ -1089,7 +1086,7 @@ class CodeGenerator(NodeVisitor): node.iter_child_nodes(only=('else_', 'test')), ('loop',)): self.writeline("l_loop = environment.undefined(%r, name='loop')" % ("'loop' is undefined. the filter section of a loop as well " - "as the else block doesn't have access to the special 'loop'" + "as the else block don't have access to the special 'loop'" " variable of the current loop. Because there is no parent " "loop it's undefined. Happened in loop on %s" % self.position(node))) @@ -1121,7 +1118,7 @@ class CodeGenerator(NodeVisitor): self.visit(node.iter, loop_frame) if node.recursive: - self.write(', recurse=loop_render_func):') + self.write(', loop_render_func, depth):') else: self.write(extended_loop and '):' or ':') @@ -1219,9 +1216,9 @@ class CodeGenerator(NodeVisitor): return if self.environment.finalize: - finalize = lambda x: unicode(self.environment.finalize(x)) + finalize = lambda x: text_type(self.environment.finalize(x)) else: - finalize = unicode + finalize = text_type # if we are inside a frame that requires output checking, we do so outdent_later = False @@ -1249,7 +1246,7 @@ class CodeGenerator(NodeVisitor): else: const = escape(const) const = finalize(const) - except: + except Exception: # if something goes wrong here we evaluate the node # at runtime for easier debugging body.append(child) @@ -1370,7 +1367,7 @@ class CodeGenerator(NodeVisitor): public_names[0]) else: self.writeline('context.exported_vars.update((%s))' % - ', '.join(map(repr, public_names))) + ', '.join(imap(repr, public_names))) # -- Expression Visitors @@ -1421,19 +1418,31 @@ class CodeGenerator(NodeVisitor): self.visit(item.value, frame) self.write('}') - def binop(operator): + def binop(operator, interceptable=True): def visitor(self, node, frame): - self.write('(') - self.visit(node.left, frame) - self.write(' %s ' % operator) - self.visit(node.right, frame) + if self.environment.sandboxed and \ + operator in self.environment.intercepted_binops: + self.write('environment.call_binop(context, %r, ' % operator) + self.visit(node.left, frame) + self.write(', ') + self.visit(node.right, frame) + else: + self.write('(') + self.visit(node.left, frame) + self.write(' %s ' % operator) + self.visit(node.right, frame) self.write(')') return visitor - def uaop(operator): + def uaop(operator, interceptable=True): def visitor(self, node, frame): - self.write('(' + operator) - self.visit(node.node, frame) + if self.environment.sandboxed and \ + operator in self.environment.intercepted_unops: + self.write('environment.call_unop(context, %r, ' % operator) + self.visit(node.node, frame) + else: + self.write('(' + operator) + self.visit(node.node, frame) self.write(')') return visitor @@ -1444,11 +1453,11 @@ class CodeGenerator(NodeVisitor): visit_FloorDiv = binop('//') visit_Pow = binop('**') visit_Mod = binop('%') - visit_And = binop('and') - visit_Or = binop('or') + visit_And = binop('and', interceptable=False) + visit_Or = binop('or', interceptable=False) visit_Pos = uaop('+') visit_Neg = uaop('-') - visit_Not = uaop('not ') + visit_Not = uaop('not ', interceptable=False) del binop, uaop def visit_Concat(self, node, frame): @@ -1546,22 +1555,13 @@ class CodeGenerator(NodeVisitor): 'expression on %s evaluated to false and ' 'no else section was defined.' % self.position(node))) - if not have_condexpr: - self.write('((') - self.visit(node.test, frame) - self.write(') and (') - self.visit(node.expr1, frame) - self.write(',) or (') - write_expr2() - self.write(',))[0]') - else: - self.write('(') - self.visit(node.expr1, frame) - self.write(' if ') - self.visit(node.test, frame) - self.write(' else ') - write_expr2() - self.write(')') + self.write('(') + self.visit(node.expr1, frame) + self.write(' if ') + self.visit(node.test, frame) + self.write(' else ') + write_expr2() + self.write(')') def visit_Call(self, node, frame, forward_caller=False): if self.environment.sandboxed: diff --git a/module/lib/jinja2/debug.py b/module/lib/jinja2/debug.py index eb15456d1..815cc18a4 100644 --- a/module/lib/jinja2/debug.py +++ b/module/lib/jinja2/debug.py @@ -12,13 +12,21 @@ """ import sys import traceback -from jinja2.utils import CodeType, missing, internal_code +from types import TracebackType +from jinja2.utils import missing, internal_code from jinja2.exceptions import TemplateSyntaxError +from jinja2._compat import iteritems, reraise, code_type + +# on pypy we can take advantage of transparent proxies +try: + from __pypy__ import tproxy +except ImportError: + tproxy = None # how does the raise helper look like? try: - exec "raise TypeError, 'foo'" + exec("raise TypeError, 'foo'") except SyntaxError: raise_helper = 'raise __jinja_exception__[1]' except TypeError: @@ -30,17 +38,22 @@ class TracebackFrameProxy(object): def __init__(self, tb): self.tb = tb + self._tb_next = None - def _set_tb_next(self, next): - if tb_set_next is not None: - tb_set_next(self.tb, next and next.tb or None) - self._tb_next = next - - def _get_tb_next(self): + @property + def tb_next(self): return self._tb_next - tb_next = property(_get_tb_next, _set_tb_next) - del _get_tb_next, _set_tb_next + def set_next(self, next): + if tb_set_next is not None: + try: + tb_set_next(self.tb, next and next.tb or None) + except Exception: + # this function can fail due to all the hackery it does + # on various python implementations. We just catch errors + # down and ignore them if necessary. + pass + self._tb_next = next @property def is_jinja_frame(self): @@ -50,8 +63,22 @@ class TracebackFrameProxy(object): return getattr(self.tb, name) +def make_frame_proxy(frame): + proxy = TracebackFrameProxy(frame) + if tproxy is None: + return proxy + def operation_handler(operation, *args, **kwargs): + if operation in ('__getattribute__', '__getattr__'): + return getattr(proxy, args[0]) + elif operation == '__setattr__': + proxy.__setattr__(*args, **kwargs) + else: + return getattr(proxy, operation)(*args, **kwargs) + return tproxy(TracebackType, operation_handler) + + class ProcessedTraceback(object): - """Holds a Jinja preprocessed traceback for priting or reraising.""" + """Holds a Jinja preprocessed traceback for printing or reraising.""" def __init__(self, exc_type, exc_value, frames): assert frames, 'no frames for this traceback?' @@ -59,14 +86,13 @@ class ProcessedTraceback(object): self.exc_value = exc_value self.frames = frames - def chain_frames(self): - """Chains the frames. Requires ctypes or the debugsupport extension.""" + # newly concatenate the frames (which are proxies) prev_tb = None for tb in self.frames: if prev_tb is not None: - prev_tb.tb_next = tb + prev_tb.set_next(tb) prev_tb = tb - prev_tb.tb_next = None + prev_tb.set_next(None) def render_as_text(self, limit=None): """Return a string with the traceback.""" @@ -95,7 +121,12 @@ class ProcessedTraceback(object): @property def standard_exc_info(self): """Standard python exc_info for re-raising""" - return self.exc_type, self.exc_value, self.frames[0].tb + tb = self.frames[0] + # the frame will be an actual traceback (or transparent proxy) if + # we are on pypy or a python implementation with support for tproxy + if type(tb) is not TracebackType: + tb = tb.tb + return self.exc_type, self.exc_value, tb def make_traceback(exc_info, source_hint=None): @@ -128,7 +159,7 @@ def translate_exception(exc_info, initial_skip=0): frames = [] # skip some internal frames if wanted - for x in xrange(initial_skip): + for x in range(initial_skip): if tb is not None: tb = tb.tb_next initial_tb = tb @@ -152,19 +183,16 @@ def translate_exception(exc_info, initial_skip=0): tb = fake_exc_info(exc_info[:2] + (tb,), template.filename, lineno)[2] - frames.append(TracebackFrameProxy(tb)) + frames.append(make_frame_proxy(tb)) tb = next # if we don't have any exceptions in the frames left, we have to # reraise it unchanged. # XXX: can we backup here? when could this happen? if not frames: - raise exc_info[0], exc_info[1], exc_info[2] + reraise(exc_info[0], exc_info[1], exc_info[2]) - traceback = ProcessedTraceback(exc_info[0], exc_info[1], frames) - if tb_set_next is not None: - traceback.chain_frames() - return traceback + return ProcessedTraceback(exc_info[0], exc_info[1], frames) def fake_exc_info(exc_info, filename, lineno): @@ -179,7 +207,7 @@ def fake_exc_info(exc_info, filename, lineno): locals = ctx.get_all() else: locals = {} - for name, value in real_locals.iteritems(): + for name, value in iteritems(real_locals): if name.startswith('l_') and value is not missing: locals[name[2:]] = value @@ -217,17 +245,17 @@ def fake_exc_info(exc_info, filename, lineno): location = 'block "%s"' % function[6:] else: location = 'template' - code = CodeType(0, code.co_nlocals, code.co_stacksize, - code.co_flags, code.co_code, code.co_consts, - code.co_names, code.co_varnames, filename, - location, code.co_firstlineno, - code.co_lnotab, (), ()) + code = code_type(0, code.co_nlocals, code.co_stacksize, + code.co_flags, code.co_code, code.co_consts, + code.co_names, code.co_varnames, filename, + location, code.co_firstlineno, + code.co_lnotab, (), ()) except: pass # execute the code and catch the new traceback try: - exec code in globals, locals + exec(code, globals, locals) except: exc_info = sys.exc_info() new_tb = exc_info[2].tb_next @@ -239,7 +267,8 @@ def fake_exc_info(exc_info, filename, lineno): def _init_ugly_crap(): """This function implements a few ugly things so that we can patch the traceback objects. The function returned allows resetting `tb_next` on - any python traceback object. + any python traceback object. Do not attempt to use this on non cpython + interpreters """ import ctypes from types import TracebackType @@ -297,12 +326,12 @@ def _init_ugly_crap(): return tb_set_next -# try to get a tb_set_next implementation -try: - from jinja2._debugsupport import tb_set_next -except ImportError: +# try to get a tb_set_next implementation if we don't have transparent +# proxies. +tb_set_next = None +if tproxy is None: try: tb_set_next = _init_ugly_crap() except: - tb_set_next = None -del _init_ugly_crap + pass + del _init_ugly_crap diff --git a/module/lib/jinja2/defaults.py b/module/lib/jinja2/defaults.py index d2d45443a..a27cb80cb 100644 --- a/module/lib/jinja2/defaults.py +++ b/module/lib/jinja2/defaults.py @@ -8,6 +8,7 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD, see LICENSE for more details. """ +from jinja2._compat import range_type from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner @@ -21,14 +22,16 @@ COMMENT_END_STRING = '#}' LINE_STATEMENT_PREFIX = None LINE_COMMENT_PREFIX = None TRIM_BLOCKS = False +LSTRIP_BLOCKS = False NEWLINE_SEQUENCE = '\n' +KEEP_TRAILING_NEWLINE = False # default filters, tests and namespace from jinja2.filters import FILTERS as DEFAULT_FILTERS from jinja2.tests import TESTS as DEFAULT_TESTS DEFAULT_NAMESPACE = { - 'range': xrange, + 'range': range_type, 'dict': lambda **kw: kw, 'lipsum': generate_lorem_ipsum, 'cycler': Cycler, diff --git a/module/lib/jinja2/environment.py b/module/lib/jinja2/environment.py index ac74a5c68..45fabada2 100644 --- a/module/lib/jinja2/environment.py +++ b/module/lib/jinja2/environment.py @@ -11,16 +11,26 @@ import os import sys from jinja2 import nodes -from jinja2.defaults import * +from jinja2.defaults import BLOCK_START_STRING, \ + BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ + COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ + LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ + DEFAULT_FILTERS, DEFAULT_TESTS, DEFAULT_NAMESPACE, \ + KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS from jinja2.lexer import get_lexer, TokenStream from jinja2.parser import Parser +from jinja2.nodes import EvalContext from jinja2.optimizer import optimize from jinja2.compiler import generate from jinja2.runtime import Undefined, new_context from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \ - TemplatesNotFound + TemplatesNotFound, TemplateRuntimeError from jinja2.utils import import_string, LRUCache, Markup, missing, \ - concat, consume, internalcode, _encode_filename + concat, consume, internalcode +from jinja2._compat import imap, ifilter, string_types, iteritems, \ + text_type, reraise, implements_iterator, implements_to_string, \ + get_next, encode_filename, PY2, PYPY +from functools import reduce # for direct template usage we have up to ten living environments @@ -67,11 +77,11 @@ def copy_cache(cache): def load_extensions(environment, extensions): """Load the extensions from the list and bind it to the environment. - Returns a dict of instanciated environments. + Returns a dict of instantiated environments. """ result = {} for extension in extensions: - if isinstance(extension, basestring): + if isinstance(extension, string_types): extension = import_string(extension) result[extension.identifier] = extension(environment) return result @@ -134,12 +144,23 @@ class Environment(object): If this is set to ``True`` the first newline after a block is removed (block, not variable tag!). Defaults to `False`. + `lstrip_blocks` + If this is set to ``True`` leading spaces and tabs are stripped + from the start of a line to a block. Defaults to `False`. + `newline_sequence` The sequence that starts a newline. Must be one of ``'\r'``, ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a useful default for Linux and OS X systems as well as web applications. + `keep_trailing_newline` + Preserve the trailing newline when rendering templates. + The default is ``False``, which causes a single newline, + if present, to be stripped from the end of the template. + + .. versionadded:: 2.7 + `extensions` List of Jinja extensions to use. This can either be import paths as strings or extension classes. For more information have a @@ -196,7 +217,8 @@ class Environment(object): #: if this environment is sandboxed. Modifying this variable won't make #: the environment sandboxed though. For a real sandboxed environment - #: have a look at jinja2.sandbox + #: have a look at jinja2.sandbox. This flag alone controls the code + #: generation by the compiler. sandboxed = False #: True if the environment is just an overlay @@ -223,7 +245,9 @@ class Environment(object): line_statement_prefix=LINE_STATEMENT_PREFIX, line_comment_prefix=LINE_COMMENT_PREFIX, trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, extensions=(), optimized=True, undefined=Undefined, @@ -238,7 +262,7 @@ class Environment(object): # passed by keyword rather than position. However it's important to # not change the order of arguments because it's used at least # internally in those cases: - # - spontaneus environments (i18n extension and Template) + # - spontaneous environments (i18n extension and Template) # - unittests # If parameter changes are required only add parameters at the end # and don't change the arguments (or the defaults!) of the arguments @@ -254,7 +278,9 @@ class Environment(object): self.line_statement_prefix = line_statement_prefix self.line_comment_prefix = line_comment_prefix self.trim_blocks = trim_blocks + self.lstrip_blocks = lstrip_blocks self.newline_sequence = newline_sequence + self.keep_trailing_newline = keep_trailing_newline # runtime information self.undefined = undefined @@ -269,7 +295,6 @@ class Environment(object): # set the loader provided self.loader = loader - self.bytecode_cache = None self.cache = create_cache(cache_size) self.bytecode_cache = bytecode_cache self.auto_reload = auto_reload @@ -291,7 +316,7 @@ class Environment(object): yet. This is used by :ref:`extensions <writing-extensions>` to register callbacks and configuration values without breaking inheritance. """ - for key, value in attributes.iteritems(): + for key, value in iteritems(attributes): if not hasattr(self, key): setattr(self, key, value) @@ -299,7 +324,8 @@ class Environment(object): variable_start_string=missing, variable_end_string=missing, comment_start_string=missing, comment_end_string=missing, line_statement_prefix=missing, line_comment_prefix=missing, - trim_blocks=missing, extensions=missing, optimized=missing, + trim_blocks=missing, lstrip_blocks=missing, + extensions=missing, optimized=missing, undefined=missing, finalize=missing, autoescape=missing, loader=missing, cache_size=missing, auto_reload=missing, bytecode_cache=missing): @@ -322,7 +348,7 @@ class Environment(object): rv.overlayed = True rv.linked_to = self - for key, value in args.iteritems(): + for key, value in iteritems(args): if value is not missing: setattr(rv, key, value) @@ -332,7 +358,7 @@ class Environment(object): rv.cache = copy_cache(self.cache) rv.extensions = {} - for key, value in self.extensions.iteritems(): + for key, value in iteritems(self.extensions): rv.extensions[key] = value.bind(rv) if extensions is not missing: rv.extensions.update(load_extensions(rv, extensions)) @@ -351,10 +377,10 @@ class Environment(object): try: return obj[argument] except (TypeError, LookupError): - if isinstance(argument, basestring): + if isinstance(argument, string_types): try: attr = str(argument) - except: + except Exception: pass else: try: @@ -376,6 +402,42 @@ class Environment(object): except (TypeError, LookupError, AttributeError): return self.undefined(obj=obj, name=attribute) + def call_filter(self, name, value, args=None, kwargs=None, + context=None, eval_ctx=None): + """Invokes a filter on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.filters.get(name) + if func is None: + raise TemplateRuntimeError('no filter named %r' % name) + args = [value] + list(args or ()) + if getattr(func, 'contextfilter', False): + if context is None: + raise TemplateRuntimeError('Attempted to invoke context ' + 'filter without context') + args.insert(0, context) + elif getattr(func, 'evalcontextfilter', False): + if eval_ctx is None: + if context is not None: + eval_ctx = context.eval_ctx + else: + eval_ctx = EvalContext(self) + args.insert(0, eval_ctx) + elif getattr(func, 'environmentfilter', False): + args.insert(0, self) + return func(*args, **(kwargs or {})) + + def call_test(self, name, value, args=None, kwargs=None): + """Invokes a test on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.tests.get(name) + if func is None: + raise TemplateRuntimeError('no test named %r' % name) + return func(value, *(args or ()), **(kwargs or {})) + @internalcode def parse(self, source, name=None, filename=None): """Parse the sourcecode and return the abstract syntax tree. This @@ -394,7 +456,7 @@ class Environment(object): def _parse(self, source, name, filename): """Internal parsing function used by `parse` and `compile`.""" - return Parser(self, source, name, _encode_filename(filename)).parse() + return Parser(self, source, name, encode_filename(filename)).parse() def lex(self, source, name=None, filename=None): """Lex the given sourcecode and return a generator that yields @@ -406,7 +468,7 @@ class Environment(object): of the extensions to be applied you have to filter source through the :meth:`preprocess` method. """ - source = unicode(source) + source = text_type(source) try: return self.lexer.tokeniter(source, name, filename) except TemplateSyntaxError: @@ -419,7 +481,7 @@ class Environment(object): because there you usually only want the actual source tokenized. """ return reduce(lambda s, e: e.preprocess(s, name, filename), - self.iter_extensions(), unicode(source)) + self.iter_extensions(), text_type(source)) def _tokenize(self, source, name, filename=None, state=None): """Called by the parser to do the preprocessing and filtering @@ -434,7 +496,7 @@ class Environment(object): return stream def _generate(self, source, name, filename, defer_init=False): - """Internal hook that can be overriden to hook a different generate + """Internal hook that can be overridden to hook a different generate method in. .. versionadded:: 2.5 @@ -442,7 +504,7 @@ class Environment(object): return generate(source, self, name, filename, defer_init=defer_init) def _compile(self, source, filename): - """Internal hook that can be overriden to hook a different compile + """Internal hook that can be overridden to hook a different compile method in. .. versionadded:: 2.5 @@ -473,7 +535,7 @@ class Environment(object): """ source_hint = None try: - if isinstance(source, basestring): + if isinstance(source, string_types): source_hint = source source = self._parse(source, name, filename) if self.optimized: @@ -485,7 +547,7 @@ class Environment(object): if filename is None: filename = '<template>' else: - filename = _encode_filename(filename) + filename = encode_filename(filename) return self._compile(source, filename) except TemplateSyntaxError: exc_info = sys.exc_info() @@ -539,7 +601,7 @@ class Environment(object): def compile_templates(self, target, extensions=None, filter_func=None, zip='deflated', log_function=None, ignore_errors=True, py_compile=False): - """Compiles all the templates the loader can find, compiles them + """Finds all the templates the loader can find, compiles them and stores them in `target`. If `zip` is `None`, instead of in a zipfile, the templates will be will be stored in a directory. By default a deflate zip algorithm is used, to switch to @@ -555,7 +617,9 @@ class Environment(object): to `False` and you will get an exception on syntax errors. If `py_compile` is set to `True` .pyc files will be written to the - target instead of standard .py files. + target instead of standard .py files. This flag does not do anything + on pypy and Python 3 where pyc files are not picked up by itself and + don't give much benefit. .. versionadded:: 2.4 """ @@ -565,14 +629,23 @@ class Environment(object): log_function = lambda x: None if py_compile: - import imp, struct, marshal - py_header = imp.get_magic() + \ - u'\xff\xff\xff\xff'.encode('iso-8859-15') + if not PY2 or PYPY: + from warnings import warn + warn(Warning('py_compile has no effect on pypy or Python 3')) + py_compile = False + else: + import imp, marshal + py_header = imp.get_magic() + \ + u'\xff\xff\xff\xff'.encode('iso-8859-15') + + # Python 3.3 added a source filesize to the header + if sys.version_info >= (3, 3): + py_header += u'\x00\x00\x00\x00'.encode('iso-8859-15') def write_file(filename, data, mode): if zip: info = ZipInfo(filename) - info.external_attr = 0755 << 16L + info.external_attr = 0o755 << 16 zip_file.writestr(info, data) else: f = open(os.path.join(target, filename), mode) @@ -596,7 +669,7 @@ class Environment(object): source, filename, _ = self.loader.get_source(self, name) try: code = self.compile(source, name, filename, True, True) - except TemplateSyntaxError, e: + except TemplateSyntaxError as e: if not ignore_errors: raise log_function('Could not compile "%s": %s' % (name, e)) @@ -605,7 +678,7 @@ class Environment(object): filename = ModuleLoader.get_module_filename(name) if py_compile: - c = self._compile(code, _encode_filename(filename)) + c = self._compile(code, encode_filename(filename)) write_file(filename + 'c', py_header + marshal.dumps(c), 'wb') log_function('Byte-compiled "%s" as %s' % @@ -632,6 +705,8 @@ class Environment(object): in the result list. If the loader does not support that, a :exc:`TypeError` is raised. + + .. versionadded:: 2.4 """ x = self.loader.list_templates() if extensions is not None: @@ -641,7 +716,7 @@ class Environment(object): filter_func = lambda x: '.' in x and \ x.rsplit('.', 1)[1] in extensions if filter_func is not None: - x = filter(filter_func, x) + x = ifilter(filter_func, x) return x def handle_exception(self, exc_info=None, rendered=False, source_hint=None): @@ -664,7 +739,7 @@ class Environment(object): if self.exception_handler is not None: self.exception_handler(traceback) exc_type, exc_value, tb = traceback.standard_exc_info - raise exc_type, exc_value, tb + reraise(exc_type, exc_value, tb) def join_path(self, template, parent): """Join a template with the parent. By default all the lookups are @@ -751,7 +826,7 @@ class Environment(object): .. versionadded:: 2.3 """ - if isinstance(template_name_or_list, basestring): + if isinstance(template_name_or_list, string_types): return self.get_template(template_name_or_list, parent, globals) elif isinstance(template_name_or_list, Template): return template_name_or_list @@ -813,7 +888,9 @@ class Template(object): line_statement_prefix=LINE_STATEMENT_PREFIX, line_comment_prefix=LINE_COMMENT_PREFIX, trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, extensions=(), optimized=True, undefined=Undefined, @@ -823,8 +900,9 @@ class Template(object): block_start_string, block_end_string, variable_start_string, variable_end_string, comment_start_string, comment_end_string, line_statement_prefix, line_comment_prefix, trim_blocks, - newline_sequence, frozenset(extensions), optimized, undefined, - finalize, autoescape, None, 0, False, None) + lstrip_blocks, newline_sequence, keep_trailing_newline, + frozenset(extensions), optimized, undefined, finalize, autoescape, + None, 0, False, None) return env.from_string(source, template_class=cls) @classmethod @@ -836,7 +914,7 @@ class Template(object): 'environment': environment, '__file__': code.co_filename } - exec code in namespace + exec(code, namespace) rv = cls._from_namespace(environment, namespace, globals) rv._uptodate = uptodate return rv @@ -886,7 +964,7 @@ class Template(object): vars = dict(*args, **kwargs) try: return concat(self.root_render_func(self.new_context(vars))) - except: + except Exception: exc_info = sys.exc_info() return self.environment.handle_exception(exc_info, True) @@ -908,7 +986,7 @@ class Template(object): try: for event in self.root_render_func(self.new_context(vars)): yield event - except: + except Exception: exc_info = sys.exc_info() else: return @@ -970,7 +1048,7 @@ class Template(object): @property def debug_info(self): """The debug info mapping.""" - return [tuple(map(int, x.split('='))) for x in + return [tuple(imap(int, x.split('='))) for x in self._debug_info.split('&')] def __repr__(self): @@ -981,6 +1059,7 @@ class Template(object): return '<%s %s>' % (self.__class__.__name__, name) +@implements_to_string class TemplateModule(object): """Represents an imported template. All the exported names of the template are available as attributes on this object. Additionally @@ -996,13 +1075,6 @@ class TemplateModule(object): return Markup(concat(self._body_stream)) def __str__(self): - return unicode(self).encode('utf-8') - - # unicode goes after __str__ because we configured 2to3 to rename - # __unicode__ to __str__. because the 2to3 tree is not designed to - # remove nodes from it, we leave the above __str__ around and let - # it override at runtime. - def __unicode__(self): return concat(self._body_stream) def __repr__(self): @@ -1032,6 +1104,7 @@ class TemplateExpression(object): return rv +@implements_iterator class TemplateStream(object): """A template stream works pretty much like an ordinary python generator but it can buffer multiple items to reduce the number of total iterations. @@ -1050,15 +1123,15 @@ class TemplateStream(object): def dump(self, fp, encoding=None, errors='strict'): """Dump the complete stream into a file or file-like object. Per default unicode strings are written, if you want to encode - before writing specifiy an `encoding`. + before writing specify an `encoding`. Example usage:: Template('Hello {{ name }}!').stream(name='foo').dump('hello.html') """ close = False - if isinstance(fp, basestring): - fp = file(fp, 'w') + if isinstance(fp, string_types): + fp = open(fp, encoding is None and 'w' or 'wb') close = True try: if encoding is not None: @@ -1076,7 +1149,7 @@ class TemplateStream(object): def disable_buffering(self): """Disable the output buffering.""" - self._next = self._gen.next + self._next = get_next(self._gen) self.buffered = False def enable_buffering(self, size=5): @@ -1104,12 +1177,12 @@ class TemplateStream(object): c_size = 0 self.buffered = True - self._next = generator(self._gen.next).next + self._next = get_next(generator(get_next(self._gen))) def __iter__(self): return self - def next(self): + def __next__(self): return self._next() diff --git a/module/lib/jinja2/exceptions.py b/module/lib/jinja2/exceptions.py index 771f6a8d7..c9df6dc7c 100644 --- a/module/lib/jinja2/exceptions.py +++ b/module/lib/jinja2/exceptions.py @@ -8,24 +8,40 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD, see LICENSE for more details. """ +from jinja2._compat import imap, text_type, PY2, implements_to_string class TemplateError(Exception): """Baseclass for all template errors.""" - def __init__(self, message=None): - if message is not None: - message = unicode(message).encode('utf-8') - Exception.__init__(self, message) - - @property - def message(self): - if self.args: - message = self.args[0] + if PY2: + def __init__(self, message=None): if message is not None: - return message.decode('utf-8', 'replace') - - + message = text_type(message).encode('utf-8') + Exception.__init__(self, message) + + @property + def message(self): + if self.args: + message = self.args[0] + if message is not None: + return message.decode('utf-8', 'replace') + + def __unicode__(self): + return self.message or u'' + else: + def __init__(self, message=None): + Exception.__init__(self, message) + + @property + def message(self): + if self.args: + message = self.args[0] + if message is not None: + return message + + +@implements_to_string class TemplateNotFound(IOError, LookupError, TemplateError): """Raised if a template does not exist.""" @@ -42,13 +58,6 @@ class TemplateNotFound(IOError, LookupError, TemplateError): self.templates = [name] def __str__(self): - return self.message.encode('utf-8') - - # unicode goes after __str__ because we configured 2to3 to rename - # __unicode__ to __str__. because the 2to3 tree is not designed to - # remove nodes from it, we leave the above __str__ around and let - # it override at runtime. - def __unicode__(self): return self.message @@ -62,12 +71,13 @@ class TemplatesNotFound(TemplateNotFound): def __init__(self, names=(), message=None): if message is None: - message = u'non of the templates given were found: ' + \ - u', '.join(map(unicode, names)) + message = u'none of the templates given were found: ' + \ + u', '.join(imap(text_type, names)) TemplateNotFound.__init__(self, names and names[-1] or None, message) self.templates = list(names) +@implements_to_string class TemplateSyntaxError(TemplateError): """Raised to tell the user that there is a problem with the template.""" @@ -83,13 +93,6 @@ class TemplateSyntaxError(TemplateError): self.translated = False def __str__(self): - return unicode(self).encode('utf-8') - - # unicode goes after __str__ because we configured 2to3 to rename - # __unicode__ to __str__. because the 2to3 tree is not designed to - # remove nodes from it, we leave the above __str__ around and let - # it override at runtime. - def __unicode__(self): # for translated errors we only return the message if self.translated: return self.message diff --git a/module/lib/jinja2/ext.py b/module/lib/jinja2/ext.py index ceb38953a..c2df12d55 100644 --- a/module/lib/jinja2/ext.py +++ b/module/lib/jinja2/ext.py @@ -10,13 +10,17 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD. """ -from collections import deque from jinja2 import nodes -from jinja2.defaults import * +from jinja2.defaults import BLOCK_START_STRING, \ + BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ + COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ + LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ + KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS from jinja2.environment import Environment -from jinja2.runtime import Undefined, concat +from jinja2.runtime import concat from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError -from jinja2.utils import contextfunction, import_string, Markup, next +from jinja2.utils import contextfunction, import_string, Markup +from jinja2._compat import next, with_metaclass, string_types, iteritems # the only real useful gettext functions for a Jinja template. Note @@ -34,7 +38,7 @@ class ExtensionRegistry(type): return rv -class Extension(object): +class Extension(with_metaclass(ExtensionRegistry, object)): """Extensions can be used to add extra functionality to the Jinja template system at the parser level. Custom extensions are bound to an environment but may not store environment specific data on `self`. The reason for @@ -52,7 +56,6 @@ class Extension(object): is a terrible name, ``fragment_cache_prefix`` on the other hand is a good name as includes the name of the extension (fragment cache). """ - __metaclass__ = ExtensionRegistry #: if this extension parses this is the list of tags it's listening to. tags = set() @@ -103,7 +106,9 @@ class Extension(object): def attr(self, name, lineno=None): """Return an attribute node for the current extension. This is useful - to pass constants on extensions to generated template code:: + to pass constants on extensions to generated template code. + + :: self.attr('_my_attribute', lineno=lineno) """ @@ -203,7 +208,7 @@ class InternationalizationExtension(Extension): self.environment.globals.pop(key, None) def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS): - if isinstance(source, basestring): + if isinstance(source, string_types): source = self.environment.parse(source) return extract_from_ast(source, gettext_functions) @@ -216,6 +221,7 @@ class InternationalizationExtension(Extension): # defined in the body of the trans block too, but this is checked at # a later state. plural_expr = None + plural_expr_assignment = None variables = {} while parser.stream.current.type != 'block_end': if variables: @@ -239,7 +245,13 @@ class InternationalizationExtension(Extension): variables[name.value] = var = nodes.Name(name.value, 'load') if plural_expr is None: - plural_expr = var + if isinstance(var, nodes.Call): + plural_expr = nodes.Name('_trans', 'load') + variables[name.value] = plural_expr + plural_expr_assignment = nodes.Assign( + nodes.Name('_trans', 'store'), var) + else: + plural_expr = var num_called_num = name.value == 'num' parser.stream.expect('block_end') @@ -289,7 +301,10 @@ class InternationalizationExtension(Extension): bool(referenced), num_called_num and have_plural) node.set_lineno(lineno) - return node + if plural_expr_assignment is not None: + return [plural_expr_assignment, node] + else: + return node def _parse_block(self, parser, allow_pluralize): """Parse until the next block tag with a given name.""" @@ -352,7 +367,7 @@ class InternationalizationExtension(Extension): # enough to handle the variable expansion and autoescape # handling itself if self.environment.newstyle_gettext: - for key, value in variables.iteritems(): + for key, value in iteritems(variables): # the function adds that later anyways in case num was # called num, so just skip it. if num_called_num and key == 'num': @@ -474,7 +489,7 @@ def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, strings = [] for arg in node.args: if isinstance(arg, nodes.Const) and \ - isinstance(arg.value, basestring): + isinstance(arg.value, string_types): strings.append(arg.value) else: strings.append(None) @@ -550,6 +565,10 @@ def babel_extract(fileobj, keywords, comment_tags, options): The `newstyle_gettext` flag can be set to `True` to enable newstyle gettext calls. + .. versionchanged:: 2.7 + A `silent` option can now be provided. If set to `False` template + syntax errors are propagated instead of being ignored. + :param fileobj: the file-like object the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognized as translation functions @@ -569,8 +588,10 @@ def babel_extract(fileobj, keywords, comment_tags, options): extensions.add(InternationalizationExtension) def getbool(options, key, default=False): - options.get(key, str(default)).lower() in ('1', 'on', 'yes', 'true') + return options.get(key, str(default)).lower() in \ + ('1', 'on', 'yes', 'true') + silent = getbool(options, 'silent', True) environment = Environment( options.get('block_start_string', BLOCK_START_STRING), options.get('block_end_string', BLOCK_END_STRING), @@ -581,7 +602,10 @@ def babel_extract(fileobj, keywords, comment_tags, options): options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX, options.get('line_comment_prefix') or LINE_COMMENT_PREFIX, getbool(options, 'trim_blocks', TRIM_BLOCKS), - NEWLINE_SEQUENCE, frozenset(extensions), + getbool(options, 'lstrip_blocks', LSTRIP_BLOCKS), + NEWLINE_SEQUENCE, + getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE), + frozenset(extensions), cache_size=0, auto_reload=False ) @@ -593,7 +617,9 @@ def babel_extract(fileobj, keywords, comment_tags, options): try: node = environment.parse(source) tokens = list(environment.lex(environment.preprocess(source))) - except TemplateSyntaxError, e: + except TemplateSyntaxError as e: + if not silent: + raise # skip templates with syntax errors return diff --git a/module/lib/jinja2/filters.py b/module/lib/jinja2/filters.py index d1848e434..fd0db04aa 100644 --- a/module/lib/jinja2/filters.py +++ b/module/lib/jinja2/filters.py @@ -10,12 +10,15 @@ """ import re import math + from random import choice from operator import itemgetter -from itertools import imap, groupby -from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode +from itertools import groupby +from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode, \ + unicode_urlencode from jinja2.runtime import Undefined -from jinja2.exceptions import FilterArgumentError, SecurityError +from jinja2.exceptions import FilterArgumentError +from jinja2._compat import next, imap, string_types, text_type, iteritems _word_re = re.compile(r'\w+(?u)') @@ -48,11 +51,50 @@ def environmentfilter(f): return f +def make_attrgetter(environment, attribute): + """Returns a callable that looks up the given attribute from a + passed object with the rules of the environment. Dots are allowed + to access attributes of attributes. Integer parts in paths are + looked up as integers. + """ + if not isinstance(attribute, string_types) \ + or ('.' not in attribute and not attribute.isdigit()): + return lambda x: environment.getitem(x, attribute) + attribute = attribute.split('.') + def attrgetter(item): + for part in attribute: + if part.isdigit(): + part = int(part) + item = environment.getitem(item, part) + return item + return attrgetter + + def do_forceescape(value): """Enforce HTML escaping. This will probably double escape variables.""" if hasattr(value, '__html__'): value = value.__html__() - return escape(unicode(value)) + return escape(text_type(value)) + + +def do_urlencode(value): + """Escape strings for use in URLs (uses UTF-8 encoding). It accepts both + dictionaries and regular strings as well as pairwise iterables. + + .. versionadded:: 2.7 + """ + itemiter = None + if isinstance(value, dict): + itemiter = iteritems(value) + elif not isinstance(value, string_types): + try: + itemiter = iter(value) + except TypeError: + pass + if itemiter is None: + return unicode_urlencode(value) + return u'&'.join(unicode_urlencode(k) + '=' + + unicode_urlencode(v) for k, v in itemiter) @evalcontextfilter @@ -74,7 +116,7 @@ def do_replace(eval_ctx, s, old, new, count=None): if count is None: count = -1 if not eval_ctx.autoescape: - return unicode(s).replace(unicode(old), unicode(new), count) + return text_type(s).replace(text_type(old), text_type(new), count) if hasattr(old, '__html__') or hasattr(new, '__html__') and \ not hasattr(s, '__html__'): s = escape(s) @@ -119,7 +161,7 @@ def do_xmlattr(_eval_ctx, d, autospace=True): """ rv = u' '.join( u'%s="%s"' % (escape(key), escape(value)) - for key, value in d.iteritems() + for key, value in iteritems(d) if value is not None and not isinstance(value, Undefined) ) if autospace and rv: @@ -140,7 +182,12 @@ def do_title(s): """Return a titlecased version of the value. I.e. words will start with uppercase letters, all remaining characters are lowercase. """ - return soft_unicode(s).title() + rv = [] + for item in re.compile(r'([-\s]+)(?u)').split(s): + if not item: + continue + rv.append(item[0].upper() + item[1:].lower()) + return ''.join(rv) def do_dictsort(value, case_sensitive=False, by='key'): @@ -153,7 +200,7 @@ def do_dictsort(value, case_sensitive=False, by='key'): {% for item in mydict|dictsort %} sort the dict by key, case insensitive - {% for item in mydict|dicsort(true) %} + {% for item in mydict|dictsort(true) %} sort the dict by key, case sensitive {% for item in mydict|dictsort(false, 'value') %} @@ -169,14 +216,16 @@ def do_dictsort(value, case_sensitive=False, by='key'): '"key" or "value"') def sort_func(item): value = item[pos] - if isinstance(value, basestring) and not case_sensitive: + if isinstance(value, string_types) and not case_sensitive: value = value.lower() return value return sorted(value.items(), key=sort_func) -def do_sort(value, reverse=False, case_sensitive=False): +@environmentfilter +def do_sort(environment, value, reverse=False, case_sensitive=False, + attribute=None): """Sort an iterable. Per default it sorts ascending, if you pass it true as first argument it will reverse the sorting. @@ -189,14 +238,30 @@ def do_sort(value, reverse=False, case_sensitive=False): {% for item in iterable|sort %} ... {% endfor %} + + It is also possible to sort by an attribute (for example to sort + by the date of an object) by specifying the `attribute` parameter: + + .. sourcecode:: jinja + + {% for item in iterable|sort(attribute='date') %} + ... + {% endfor %} + + .. versionchanged:: 2.6 + The `attribute` parameter was added. """ if not case_sensitive: def sort_func(item): - if isinstance(item, basestring): + if isinstance(item, string_types): item = item.lower() return item else: sort_func = None + if attribute is not None: + getter = make_attrgetter(environment, attribute) + def sort_func(item, processor=sort_func or (lambda x: x)): + return processor(getter(item)) return sorted(value, key=sort_func, reverse=reverse) @@ -217,13 +282,13 @@ def do_default(value, default_value=u'', boolean=False): {{ ''|default('the string was empty', true) }} """ - if (boolean and not value) or isinstance(value, Undefined): + if isinstance(value, Undefined) or (boolean and not value): return default_value return value @evalcontextfilter -def do_join(eval_ctx, value, d=u''): +def do_join(eval_ctx, value, d=u'', attribute=None): """Return a string which is the concatenation of the strings in the sequence. The separator between elements is an empty string per default, you can define it with the optional parameter: @@ -235,10 +300,22 @@ def do_join(eval_ctx, value, d=u''): {{ [1, 2, 3]|join }} -> 123 + + It is also possible to join certain attributes of an object: + + .. sourcecode:: jinja + + {{ users|join(', ', attribute='username') }} + + .. versionadded:: 2.6 + The `attribute` parameter was added. """ + if attribute is not None: + value = imap(make_attrgetter(eval_ctx.environment, attribute), value) + # no automatic escaping? joining is a lot eaiser then if not eval_ctx.autoescape: - return unicode(d).join(imap(unicode, value)) + return text_type(d).join(imap(text_type, value)) # if the delimiter doesn't have an html representation we check # if any of the items has. If yes we do a coercion to Markup @@ -249,11 +326,11 @@ def do_join(eval_ctx, value, d=u''): if hasattr(item, '__html__'): do_escape = True else: - value[idx] = unicode(item) + value[idx] = text_type(item) if do_escape: d = escape(d) else: - d = unicode(d) + d = text_type(d) return d.join(value) # no html involved, to normal joining @@ -262,14 +339,14 @@ def do_join(eval_ctx, value, d=u''): def do_center(value, width=80): """Centers the value in a field of a given width.""" - return unicode(value).center(width) + return text_type(value).center(width) @environmentfilter def do_first(environment, seq): """Return the first item of a sequence.""" try: - return iter(seq).next() + return next(iter(seq)) except StopIteration: return environment.undefined('No first item, sequence was empty.') @@ -278,7 +355,7 @@ def do_first(environment, seq): def do_last(environment, seq): """Return the last item of a sequence.""" try: - return iter(reversed(seq)).next() + return next(iter(reversed(seq))) except StopIteration: return environment.undefined('No last item, sequence was empty.') @@ -293,21 +370,33 @@ def do_random(environment, seq): def do_filesizeformat(value, binary=False): - """Format the value like a 'human-readable' file size (i.e. 13 KB, - 4.1 MB, 102 bytes, etc). Per default decimal prefixes are used (mega, - giga, etc.), if the second parameter is set to `True` the binary - prefixes are used (mebi, gibi). + """Format the value like a 'human-readable' file size (i.e. 13 kB, + 4.1 MB, 102 Bytes, etc). Per default decimal prefixes are used (Mega, + Giga, etc.), if the second parameter is set to `True` the binary + prefixes are used (Mebi, Gibi). """ bytes = float(value) base = binary and 1024 or 1000 - middle = binary and 'i' or '' - if bytes < base: - return "%d Byte%s" % (bytes, bytes != 1 and 's' or '') - elif bytes < base * base: - return "%.1f K%sB" % (bytes / base, middle) - elif bytes < base * base * base: - return "%.1f M%sB" % (bytes / (base * base), middle) - return "%.1f G%sB" % (bytes / (base * base * base), middle) + prefixes = [ + (binary and 'KiB' or 'kB'), + (binary and 'MiB' or 'MB'), + (binary and 'GiB' or 'GB'), + (binary and 'TiB' or 'TB'), + (binary and 'PiB' or 'PB'), + (binary and 'EiB' or 'EB'), + (binary and 'ZiB' or 'ZB'), + (binary and 'YiB' or 'YB') + ] + if bytes == 1: + return '1 Byte' + elif bytes < base: + return '%d Bytes' % bytes + else: + for i, prefix in enumerate(prefixes): + unit = base ** (i + 2) + if bytes < unit: + return '%.1f %s' % ((base * bytes / unit), prefix) + return '%.1f %s' % ((base * bytes / unit), prefix) def do_pprint(value, verbose=False): @@ -360,16 +449,17 @@ def do_truncate(s, length=255, killwords=False, end='...'): """Return a truncated copy of the string. The length is specified with the first parameter which defaults to ``255``. If the second parameter is ``true`` the filter will cut the text at length. Otherwise - it will try to save the last word. If the text was in fact + it will discard the last word. If the text was in fact truncated it will append an ellipsis sign (``"..."``). If you want a different ellipsis sign than ``"..."`` you can specify it using the third parameter. - .. sourcecode jinja:: + .. sourcecode:: jinja - {{ mytext|truncate(300, false, '»') }} - truncate mytext to 300 chars, don't split up words, use a - right pointing double arrow as ellipsis sign. + {{ "foo bar"|truncate(5) }} + -> "foo ..." + {{ "foo bar"|truncate(5, True) }} + -> "foo b..." """ if len(s) <= length: return s @@ -386,16 +476,24 @@ def do_truncate(s, length=255, killwords=False, end='...'): result.append(end) return u' '.join(result) - -def do_wordwrap(s, width=79, break_long_words=True): +@environmentfilter +def do_wordwrap(environment, s, width=79, break_long_words=True, + wrapstring=None): """ Return a copy of the string passed to the filter wrapped after ``79`` characters. You can override this default using the first parameter. If you set the second parameter to `false` Jinja will not - split words apart if they are longer than `width`. + split words apart if they are longer than `width`. By default, the newlines + will be the default newlines for the environment, but this can be changed + using the wrapstring keyword argument. + + .. versionadded:: 2.7 + Added support for the `wrapstring` parameter. """ + if not wrapstring: + wrapstring = environment.newline_sequence import textwrap - return u'\n'.join(textwrap.wrap(s, width=width, expand_tabs=False, + return wrapstring.join(textwrap.wrap(s, width=width, expand_tabs=False, replace_whitespace=False, break_long_words=break_long_words)) @@ -456,7 +554,7 @@ def do_striptags(value): """ if hasattr(value, '__html__'): value = value.__html__() - return Markup(unicode(value)).striptags() + return Markup(text_type(value)).striptags() def do_slice(value, slices, fill_with=None): @@ -484,7 +582,7 @@ def do_slice(value, slices, fill_with=None): items_per_slice = length // slices slices_with_extra = length % slices offset = 0 - for slice_number in xrange(slices): + for slice_number in range(slices): start = offset + slice_number * items_per_slice if slice_number < slices_with_extra: offset += 1 @@ -500,7 +598,7 @@ def do_batch(value, linecount, fill_with=None): A filter that batches items. It works pretty much like `slice` just the other way round. It returns a list of lists with the given number of items. If you provide a second parameter this - is used to fill missing items. See this example: + is used to fill up missing items. See this example: .. sourcecode:: html+jinja @@ -595,8 +693,12 @@ def do_groupby(environment, value, attribute): As you can see the item we're grouping by is stored in the `grouper` attribute and the `list` contains all the objects that have this grouper in common. + + .. versionchanged:: 2.6 + It's now possible to use dotted notation to group by the child + attribute of another attribute. """ - expr = lambda x: environment.getitem(x, attribute) + expr = make_attrgetter(environment, attribute) return sorted(map(_GroupTuple, groupby(sorted(value, key=expr), expr))) @@ -605,10 +707,32 @@ class _GroupTuple(tuple): grouper = property(itemgetter(0)) list = property(itemgetter(1)) - def __new__(cls, (key, value)): + def __new__(cls, xxx_todo_changeme): + (key, value) = xxx_todo_changeme return tuple.__new__(cls, (key, list(value))) +@environmentfilter +def do_sum(environment, iterable, attribute=None, start=0): + """Returns the sum of a sequence of numbers plus the value of parameter + 'start' (which defaults to 0). When the sequence is empty it returns + start. + + It is also possible to sum up only certain attributes: + + .. sourcecode:: jinja + + Total: {{ items|sum(attribute='price') }} + + .. versionchanged:: 2.6 + The `attribute` parameter was added to allow suming up over + attributes. Also the `start` parameter was moved on to the right. + """ + if attribute is not None: + iterable = imap(make_attrgetter(environment, attribute), iterable) + return sum(iterable, start) + + def do_list(value): """Convert the value into a list. If it was a string the returned list will be a list of characters. @@ -625,14 +749,14 @@ def do_mark_safe(value): def do_mark_unsafe(value): """Mark a value as unsafe. This is the reverse operation for :func:`safe`.""" - return unicode(value) + return text_type(value) def do_reverse(value): """Reverse the object or return an iterator the iterates over it the other way round. """ - if isinstance(value, basestring): + if isinstance(value, string_types): return value[::-1] try: return reversed(value) @@ -670,6 +794,144 @@ def do_attr(environment, obj, name): return environment.undefined(obj=obj, name=name) +@contextfilter +def do_map(*args, **kwargs): + """Applies a filter on a sequence of objects or looks up an attribute. + This is useful when dealing with lists of objects but you are really + only interested in a certain value of it. + + The basic usage is mapping on an attribute. Imagine you have a list + of users but you are only interested in a list of usernames: + + .. sourcecode:: jinja + + Users on this page: {{ users|map(attribute='username')|join(', ') }} + + Alternatively you can let it invoke a filter by passing the name of the + filter and the arguments afterwards. A good example would be applying a + text conversion filter on a sequence: + + .. sourcecode:: jinja + + Users on this page: {{ titles|map('lower')|join(', ') }} + + .. versionadded:: 2.7 + """ + context = args[0] + seq = args[1] + + if len(args) == 2 and 'attribute' in kwargs: + attribute = kwargs.pop('attribute') + if kwargs: + raise FilterArgumentError('Unexpected keyword argument %r' % + next(iter(kwargs))) + func = make_attrgetter(context.environment, attribute) + else: + try: + name = args[2] + args = args[3:] + except LookupError: + raise FilterArgumentError('map requires a filter argument') + func = lambda item: context.environment.call_filter( + name, item, args, kwargs, context=context) + + if seq: + for item in seq: + yield func(item) + + +@contextfilter +def do_select(*args, **kwargs): + """Filters a sequence of objects by appying a test to either the object + or the attribute and only selecting the ones with the test succeeding. + + Example usage: + + .. sourcecode:: jinja + + {{ numbers|select("odd") }} + + .. versionadded:: 2.7 + """ + return _select_or_reject(args, kwargs, lambda x: x, False) + + +@contextfilter +def do_reject(*args, **kwargs): + """Filters a sequence of objects by appying a test to either the object + or the attribute and rejecting the ones with the test succeeding. + + Example usage: + + .. sourcecode:: jinja + + {{ numbers|reject("odd") }} + + .. versionadded:: 2.7 + """ + return _select_or_reject(args, kwargs, lambda x: not x, False) + + +@contextfilter +def do_selectattr(*args, **kwargs): + """Filters a sequence of objects by appying a test to either the object + or the attribute and only selecting the ones with the test succeeding. + + Example usage: + + .. sourcecode:: jinja + + {{ users|selectattr("is_active") }} + {{ users|selectattr("email", "none") }} + + .. versionadded:: 2.7 + """ + return _select_or_reject(args, kwargs, lambda x: x, True) + + +@contextfilter +def do_rejectattr(*args, **kwargs): + """Filters a sequence of objects by appying a test to either the object + or the attribute and rejecting the ones with the test succeeding. + + .. sourcecode:: jinja + + {{ users|rejectattr("is_active") }} + {{ users|rejectattr("email", "none") }} + + .. versionadded:: 2.7 + """ + return _select_or_reject(args, kwargs, lambda x: not x, True) + + +def _select_or_reject(args, kwargs, modfunc, lookup_attr): + context = args[0] + seq = args[1] + if lookup_attr: + try: + attr = args[2] + except LookupError: + raise FilterArgumentError('Missing parameter for attribute name') + transfunc = make_attrgetter(context.environment, attr) + off = 1 + else: + off = 0 + transfunc = lambda x: x + + try: + name = args[2 + off] + args = args[3 + off:] + func = lambda item: context.environment.call_test( + name, item, args, kwargs) + except LookupError: + func = bool + + if seq: + for item in seq: + if modfunc(func(transfunc(item))): + yield item + + FILTERS = { 'attr': do_attr, 'replace': do_replace, @@ -694,7 +956,10 @@ FILTERS = { 'capitalize': do_capitalize, 'first': do_first, 'last': do_last, + 'map': do_map, 'random': do_random, + 'reject': do_reject, + 'rejectattr': do_rejectattr, 'filesizeformat': do_filesizeformat, 'pprint': do_pprint, 'truncate': do_truncate, @@ -708,12 +973,15 @@ FILTERS = { 'format': do_format, 'trim': do_trim, 'striptags': do_striptags, + 'select': do_select, + 'selectattr': do_selectattr, 'slice': do_slice, 'batch': do_batch, - 'sum': sum, + 'sum': do_sum, 'abs': abs, 'round': do_round, 'groupby': do_groupby, 'safe': do_mark_safe, - 'xmlattr': do_xmlattr + 'xmlattr': do_xmlattr, + 'urlencode': do_urlencode } diff --git a/module/lib/jinja2/lexer.py b/module/lib/jinja2/lexer.py index 0d3f69617..a50128507 100644 --- a/module/lib/jinja2/lexer.py +++ b/module/lib/jinja2/lexer.py @@ -15,10 +15,13 @@ :license: BSD, see LICENSE for more details. """ import re + from operator import itemgetter from collections import deque from jinja2.exceptions import TemplateSyntaxError -from jinja2.utils import LRUCache, next +from jinja2.utils import LRUCache +from jinja2._compat import next, iteritems, implements_iterator, text_type, \ + intern # cache for the lexers. Exists in order to be able to have multiple @@ -126,7 +129,7 @@ operators = { ';': TOKEN_SEMICOLON } -reverse_operators = dict([(v, k) for k, v in operators.iteritems()]) +reverse_operators = dict([(v, k) for k, v in iteritems(operators)]) assert len(operators) == len(reverse_operators), 'operators dropped' operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x)))) @@ -197,7 +200,7 @@ def compile_rules(environment): if environment.line_statement_prefix is not None: rules.append((len(environment.line_statement_prefix), 'linestatement', - r'^\s*' + e(environment.line_statement_prefix))) + r'^[ \t\v]*' + e(environment.line_statement_prefix))) if environment.line_comment_prefix is not None: rules.append((len(environment.line_comment_prefix), 'linecomment', r'(?:^|(?<=\S))[^\S\r\n]*' + @@ -262,6 +265,7 @@ class Token(tuple): ) +@implements_iterator class TokenStreamIterator(object): """The iterator for tokenstreams. Iterate over the stream until the eof token is reached. @@ -273,7 +277,7 @@ class TokenStreamIterator(object): def __iter__(self): return self - def next(self): + def __next__(self): token = self.stream.current if token.type is TOKEN_EOF: self.stream.close() @@ -282,6 +286,7 @@ class TokenStreamIterator(object): return token +@implements_iterator class TokenStream(object): """A token stream is an iterable that yields :class:`Token`\s. The parser however does not iterate over it but calls :meth:`next` to go @@ -289,7 +294,7 @@ class TokenStream(object): """ def __init__(self, generator, name, filename): - self._next = iter(generator).next + self._iter = iter(generator) self._pushed = deque() self.name = name self.filename = filename @@ -300,8 +305,9 @@ class TokenStream(object): def __iter__(self): return TokenStreamIterator(self) - def __nonzero__(self): + def __bool__(self): return bool(self._pushed) or self.current.type is not TOKEN_EOF + __nonzero__ = __bool__ # py2 eos = property(lambda x: not x, doc="Are we at the end of the stream?") @@ -319,7 +325,7 @@ class TokenStream(object): def skip(self, n=1): """Got n tokens ahead.""" - for x in xrange(n): + for x in range(n): next(self) def next_if(self, expr): @@ -333,14 +339,14 @@ class TokenStream(object): """Like :meth:`next_if` but only returns `True` or `False`.""" return self.next_if(expr) is not None - def next(self): + def __next__(self): """Go one token ahead and return the old one""" rv = self.current if self._pushed: self.current = self._pushed.popleft() elif self.current.type is not TOKEN_EOF: try: - self.current = self._next() + self.current = next(self._iter) except StopIteration: self.close() return rv @@ -348,7 +354,7 @@ class TokenStream(object): def close(self): """Close the stream.""" self.current = Token(self.current.lineno, TOKEN_EOF, '') - self._next = None + self._iter = None self.closed = True def expect(self, expr): @@ -383,7 +389,9 @@ def get_lexer(environment): environment.line_statement_prefix, environment.line_comment_prefix, environment.trim_blocks, - environment.newline_sequence) + environment.lstrip_blocks, + environment.newline_sequence, + environment.keep_trailing_newline) lexer = _lexer_cache.get(key) if lexer is None: lexer = Lexer(environment) @@ -414,7 +422,7 @@ class Lexer(object): (operator_re, TOKEN_OPERATOR, None) ] - # assamble the root lexing rule. because "|" is ungreedy + # assemble the root lexing rule. because "|" is ungreedy # we have to sort by length so that the lexer continues working # as expected when we have parsing rules like <% for block and # <%= for variables. (if someone wants asp like syntax) @@ -425,7 +433,44 @@ class Lexer(object): # block suffix if trimming is enabled block_suffix_re = environment.trim_blocks and '\\n?' or '' + # strip leading spaces if lstrip_blocks is enabled + prefix_re = {} + if environment.lstrip_blocks: + # use '{%+' to manually disable lstrip_blocks behavior + no_lstrip_re = e('+') + # detect overlap between block and variable or comment strings + block_diff = c(r'^%s(.*)' % e(environment.block_start_string)) + # make sure we don't mistake a block for a variable or a comment + m = block_diff.match(environment.comment_start_string) + no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' + m = block_diff.match(environment.variable_start_string) + no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' + + # detect overlap between comment and variable strings + comment_diff = c(r'^%s(.*)' % e(environment.comment_start_string)) + m = comment_diff.match(environment.variable_start_string) + no_variable_re = m and r'(?!%s)' % e(m.group(1)) or '' + + lstrip_re = r'^[ \t]*' + block_prefix_re = r'%s%s(?!%s)|%s\+?' % ( + lstrip_re, + e(environment.block_start_string), + no_lstrip_re, + e(environment.block_start_string), + ) + comment_prefix_re = r'%s%s%s|%s\+?' % ( + lstrip_re, + e(environment.comment_start_string), + no_variable_re, + e(environment.comment_start_string), + ) + prefix_re['block'] = block_prefix_re + prefix_re['comment'] = comment_prefix_re + else: + block_prefix_re = '%s' % e(environment.block_start_string) + self.newline_sequence = environment.newline_sequence + self.keep_trailing_newline = environment.keep_trailing_newline # global lexing rules self.rules = { @@ -434,11 +479,11 @@ class Lexer(object): (c('(.*?)(?:%s)' % '|'.join( [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % ( e(environment.block_start_string), - e(environment.block_start_string), + block_prefix_re, e(environment.block_end_string), e(environment.block_end_string) )] + [ - r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, r) + r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, prefix_re.get(n,r)) for n, r in root_tag_rules ])), (TOKEN_DATA, '#bygroup'), '#bygroup'), # data @@ -472,7 +517,7 @@ class Lexer(object): TOKEN_RAW_BEGIN: [ (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % ( e(environment.block_start_string), - e(environment.block_start_string), + block_prefix_re, e(environment.block_end_string), e(environment.block_end_string), block_suffix_re @@ -491,7 +536,7 @@ class Lexer(object): } def _normalize_newlines(self, value): - """Called for strings and template data to normlize it to unicode.""" + """Called for strings and template data to normalize it to unicode.""" return newline_re.sub(self.newline_sequence, value) def tokenize(self, source, name=None, filename=None, state=None): @@ -526,7 +571,7 @@ class Lexer(object): value = self._normalize_newlines(value[1:-1]) \ .encode('ascii', 'backslashreplace') \ .decode('unicode-escape') - except Exception, e: + except Exception as e: msg = str(e).split(':')[-1].strip() raise TemplateSyntaxError(msg, lineno, name, filename) # if we can express it as bytestring (ascii only) @@ -549,7 +594,14 @@ class Lexer(object): """This method tokenizes the text and returns the tokens in a generator. Use this method if you just want to tokenize a template. """ - source = '\n'.join(unicode(source).splitlines()) + source = text_type(source) + lines = source.splitlines() + if self.keep_trailing_newline and source: + for newline in ('\r\n', '\r', '\n'): + if source.endswith(newline): + lines.append('') + break + source = '\n'.join(lines) pos = 0 lineno = 1 stack = ['root'] @@ -571,7 +623,7 @@ class Lexer(object): if m is None: continue - # we only match blocks and variables if brances / parentheses + # we only match blocks and variables if braces / parentheses # are balanced. continue parsing with the lower rule which # is the operator rule. do this only if the end tags look # like operators @@ -590,7 +642,7 @@ class Lexer(object): # yield for the current token the first named # group that matched elif token == '#bygroup': - for key, value in m.groupdict().iteritems(): + for key, value in iteritems(m.groupdict()): if value is not None: yield lineno, key, value lineno += value.count('\n') @@ -647,7 +699,7 @@ class Lexer(object): stack.pop() # resolve the new state by group checking elif new_state == '#bygroup': - for key, value in m.groupdict().iteritems(): + for key, value in iteritems(m.groupdict()): if value is not None: stack.append(key) break @@ -669,7 +721,7 @@ class Lexer(object): # publish new function and start again pos = pos2 break - # if loop terminated without break we havn't found a single match + # if loop terminated without break we haven't found a single match # either we are at the end of the file or we have a problem else: # end of text diff --git a/module/lib/jinja2/loaders.py b/module/lib/jinja2/loaders.py index bd435e8b0..cc9c6836e 100644 --- a/module/lib/jinja2/loaders.py +++ b/module/lib/jinja2/loaders.py @@ -13,12 +13,10 @@ import sys import weakref from types import ModuleType from os import path -try: - from hashlib import sha1 -except ImportError: - from sha import new as sha1 +from hashlib import sha1 from jinja2.exceptions import TemplateNotFound -from jinja2.utils import LRUCache, open_if_exists, internalcode +from jinja2.utils import open_if_exists, internalcode +from jinja2._compat import string_types, iteritems def split_template_path(template): @@ -153,7 +151,7 @@ class FileSystemLoader(BaseLoader): """ def __init__(self, searchpath, encoding='utf-8'): - if isinstance(searchpath, basestring): + if isinstance(searchpath, string_types): searchpath = [searchpath] self.searchpath = list(searchpath) self.encoding = encoding @@ -251,8 +249,7 @@ class PackageLoader(BaseLoader): for filename in self.provider.resource_listdir(path): fullname = path + '/' + filename if self.provider.resource_isdir(fullname): - for item in _walk(fullname): - results.append(item) + _walk(fullname) else: results.append(fullname[offset:].lstrip('/')) _walk(path) @@ -275,7 +272,7 @@ class DictLoader(BaseLoader): def get_source(self, environment, template): if template in self.mapping: source = self.mapping[template] - return source, None, lambda: source != self.mapping.get(template) + return source, None, lambda: source == self.mapping.get(template) raise TemplateNotFound(template) def list_templates(self): @@ -307,7 +304,7 @@ class FunctionLoader(BaseLoader): rv = self.load_func(template) if rv is None: raise TemplateNotFound(template) - elif isinstance(rv, basestring): + elif isinstance(rv, string_types): return rv, None, None return rv @@ -331,12 +328,16 @@ class PrefixLoader(BaseLoader): self.mapping = mapping self.delimiter = delimiter - def get_source(self, environment, template): + def get_loader(self, template): try: prefix, name = template.split(self.delimiter, 1) loader = self.mapping[prefix] except (ValueError, KeyError): raise TemplateNotFound(template) + return loader, name + + def get_source(self, environment, template): + loader, name = self.get_loader(template) try: return loader.get_source(environment, name) except TemplateNotFound: @@ -344,9 +345,19 @@ class PrefixLoader(BaseLoader): # (the one that includes the prefix) raise TemplateNotFound(template) + @internalcode + def load(self, environment, name, globals=None): + loader, local_name = self.get_loader(name) + try: + return loader.load(environment, local_name, globals) + except TemplateNotFound: + # re-raise the exception with the correct fileame here. + # (the one that includes the prefix) + raise TemplateNotFound(name) + def list_templates(self): result = [] - for prefix, loader in self.mapping.iteritems(): + for prefix, loader in iteritems(self.mapping): for template in loader.list_templates(): result.append(prefix + self.delimiter + template) return result @@ -377,6 +388,15 @@ class ChoiceLoader(BaseLoader): pass raise TemplateNotFound(template) + @internalcode + def load(self, environment, name, globals=None): + for loader in self.loaders: + try: + return loader.load(environment, name, globals) + except TemplateNotFound: + pass + raise TemplateNotFound(name) + def list_templates(self): found = set() for loader in self.loaders: @@ -397,6 +417,8 @@ class ModuleLoader(BaseLoader): ... ModuleLoader('/path/to/compiled/templates'), ... FileSystemLoader('/path/to/templates') ... ]) + + Templates can be precompiled with :meth:`Environment.compile_templates`. """ has_source_access = False @@ -407,7 +429,7 @@ class ModuleLoader(BaseLoader): # create a fake module that looks for the templates in the # path given. mod = _TemplateModule(package_name) - if isinstance(path, basestring): + if isinstance(path, string_types): path = [path] else: path = list(path) diff --git a/module/lib/jinja2/meta.py b/module/lib/jinja2/meta.py index 3a779a5e9..3110cff60 100644 --- a/module/lib/jinja2/meta.py +++ b/module/lib/jinja2/meta.py @@ -11,6 +11,7 @@ """ from jinja2 import nodes from jinja2.compiler import CodeGenerator +from jinja2._compat import string_types class TrackingCodeGenerator(CodeGenerator): @@ -77,7 +78,7 @@ def find_referenced_templates(ast): # something const, only yield the strings and ignore # non-string consts that really just make no sense if isinstance(template_name, nodes.Const): - if isinstance(template_name.value, basestring): + if isinstance(template_name.value, string_types): yield template_name.value # something dynamic in there else: @@ -87,7 +88,7 @@ def find_referenced_templates(ast): yield None continue # constant is a basestring, direct template name - if isinstance(node.template.value, basestring): + if isinstance(node.template.value, string_types): yield node.template.value # a tuple or list (latter *should* not happen) made of consts, # yield the consts that are strings. We could warn here for @@ -95,7 +96,7 @@ def find_referenced_templates(ast): elif isinstance(node, nodes.Include) and \ isinstance(node.template.value, (tuple, list)): for template_name in node.template.value: - if isinstance(template_name, basestring): + if isinstance(template_name, string_types): yield template_name # something else we don't care about, we could warn here else: diff --git a/module/lib/jinja2/nodes.py b/module/lib/jinja2/nodes.py index 6446c70ea..c5697e6b5 100644 --- a/module/lib/jinja2/nodes.py +++ b/module/lib/jinja2/nodes.py @@ -13,13 +13,15 @@ :license: BSD, see LICENSE for more details. """ import operator -from itertools import chain, izip + from collections import deque -from jinja2.utils import Markup, MethodType, FunctionType +from jinja2.utils import Markup +from jinja2._compat import next, izip, with_metaclass, text_type, \ + method_type, function_type #: the types we support for context functions -_context_function_types = (FunctionType, MethodType) +_context_function_types = (function_type, method_type) _binop_to_func = { @@ -77,6 +79,7 @@ class EvalContext(object): """ def __init__(self, environment, template_name=None): + self.environment = environment if callable(environment.autoescape): self.autoescape = environment.autoescape(template_name) else: @@ -101,9 +104,9 @@ def get_eval_context(node, ctx): return ctx -class Node(object): +class Node(with_metaclass(NodeType, object)): """Baseclass for all Jinja2 nodes. There are a number of nodes available - of different types. There are three major types: + of different types. There are four major types: - :class:`Stmt`: statements - :class:`Expr`: expressions @@ -117,7 +120,6 @@ class Node(object): The `environment` attribute is set at the end of the parsing process for all nodes automatically. """ - __metaclass__ = NodeType fields = () attributes = ('lineno', 'environment') abstract = True @@ -141,7 +143,7 @@ class Node(object): setattr(self, attr, attributes.pop(attr, None)) if attributes: raise TypeError('unknown attribute %r' % - iter(attributes).next()) + next(iter(attributes))) def iter_fields(self, exclude=None, only=None): """This method iterates over all fields that are defined and yields @@ -230,6 +232,9 @@ class Node(object): def __ne__(self, other): return not self.__eq__(other) + # Restore Python 2 hashing behavior on Python 3 + __hash__ = object.__hash__ + def __repr__(self): return '%s(%s)' % ( self.__class__.__name__, @@ -372,10 +377,14 @@ class BinExpr(Expr): def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) + # intercepted operators cannot be folded at compile time + if self.environment.sandboxed and \ + self.operator in self.environment.intercepted_binops: + raise Impossible() f = _binop_to_func[self.operator] try: return f(self.left.as_const(eval_ctx), self.right.as_const(eval_ctx)) - except: + except Exception: raise Impossible() @@ -387,10 +396,14 @@ class UnaryExpr(Expr): def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) + # intercepted operators cannot be folded at compile time + if self.environment.sandboxed and \ + self.operator in self.environment.intercepted_unops: + raise Impossible() f = _uaop_to_func[self.operator] try: return f(self.node.as_const(eval_ctx)) - except: + except Exception: raise Impossible() @@ -431,7 +444,7 @@ class Const(Literal): constant value in the generated code, otherwise it will raise an `Impossible` exception. """ - from compiler import has_safe_repr + from .compiler import has_safe_repr if not has_safe_repr(value): raise Impossible() return cls(value, lineno=lineno, environment=environment) @@ -555,16 +568,16 @@ class Filter(Expr): if self.dyn_args is not None: try: args.extend(self.dyn_args.as_const(eval_ctx)) - except: + except Exception: raise Impossible() if self.dyn_kwargs is not None: try: kwargs.update(self.dyn_kwargs.as_const(eval_ctx)) - except: + except Exception: raise Impossible() try: return filter_(obj, *args, **kwargs) - except: + except Exception: raise Impossible() @@ -604,16 +617,16 @@ class Call(Expr): if self.dyn_args is not None: try: args.extend(self.dyn_args.as_const(eval_ctx)) - except: + except Exception: raise Impossible() if self.dyn_kwargs is not None: try: kwargs.update(self.dyn_kwargs.as_const(eval_ctx)) - except: + except Exception: raise Impossible() try: return obj(*args, **kwargs) - except: + except Exception: raise Impossible() @@ -628,7 +641,7 @@ class Getitem(Expr): try: return self.environment.getitem(self.node.as_const(eval_ctx), self.arg.as_const(eval_ctx)) - except: + except Exception: raise Impossible() def can_assign(self): @@ -648,7 +661,7 @@ class Getattr(Expr): eval_ctx = get_eval_context(self, eval_ctx) return self.environment.getattr(self.node.as_const(eval_ctx), self.attr) - except: + except Exception: raise Impossible() def can_assign(self): @@ -678,7 +691,7 @@ class Concat(Expr): def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) - return ''.join(unicode(x.as_const(eval_ctx)) for x in self.nodes) + return ''.join(text_type(x.as_const(eval_ctx)) for x in self.nodes) class Compare(Expr): @@ -695,7 +708,7 @@ class Compare(Expr): new_value = op.expr.as_const(eval_ctx) result = _cmpop_to_func[op.op](value, new_value) value = new_value - except: + except Exception: raise Impossible() return result diff --git a/module/lib/jinja2/parser.py b/module/lib/jinja2/parser.py index d44229ad0..f60cd018c 100644 --- a/module/lib/jinja2/parser.py +++ b/module/lib/jinja2/parser.py @@ -10,8 +10,8 @@ """ from jinja2 import nodes from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError -from jinja2.utils import next from jinja2.lexer import describe_token, describe_token_expr +from jinja2._compat import next, imap #: statements that callinto @@ -53,7 +53,7 @@ class Parser(object): def _fail_ut_eof(self, name, end_token_stack, lineno): expected = [] for exprs in end_token_stack: - expected.extend(map(describe_token_expr, exprs)) + expected.extend(imap(describe_token_expr, exprs)) if end_token_stack: currently_looking = ' or '.join( "'%s'" % describe_token_expr(expr) @@ -223,7 +223,7 @@ class Parser(object): # raise a nicer error message in that case. if self.stream.current.type == 'sub': self.fail('Block names in Jinja have to be valid Python ' - 'identifiers and may not contain hypens, use an ' + 'identifiers and may not contain hyphens, use an ' 'underscore instead.') node.body = self.parse_statements(('name:endblock',), drop_needle=True) @@ -698,7 +698,6 @@ class Parser(object): arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) return nodes.Getitem(node, arg, 'load', lineno=token.lineno) if token.type == 'lbracket': - priority_on_attribute = False args = [] while self.stream.current.type != 'rbracket': if args: diff --git a/module/lib/jinja2/runtime.py b/module/lib/jinja2/runtime.py index 6fea3aa4f..7791c645a 100644 --- a/module/lib/jinja2/runtime.py +++ b/module/lib/jinja2/runtime.py @@ -8,13 +8,14 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD. """ -import sys -from itertools import chain, imap +from itertools import chain from jinja2.nodes import EvalContext, _context_function_types -from jinja2.utils import Markup, partial, soft_unicode, escape, missing, \ - concat, internalcode, next, object_type_repr +from jinja2.utils import Markup, soft_unicode, escape, missing, concat, \ + internalcode, object_type_repr from jinja2.exceptions import UndefinedError, TemplateRuntimeError, \ TemplateNotFound +from jinja2._compat import next, imap, text_type, iteritems, \ + implements_iterator, implements_to_string, string_types, PY2 # these variables are exported to the template runtime @@ -24,13 +25,14 @@ __all__ = ['LoopContext', 'TemplateReference', 'Macro', 'Markup', 'TemplateNotFound'] #: the name of the function that is used to convert something into -#: a string. 2to3 will adopt that automatically and the generated -#: code can take advantage of it. -to_string = unicode +#: a string. We can just use the text type here. +to_string = text_type #: the identity function. Useful for certain things in the environment identity = lambda x: x +_last_iteration = object() + def markup_join(seq): """Concatenation that escapes if necessary and converts to unicode.""" @@ -45,7 +47,7 @@ def markup_join(seq): def unicode_join(seq): """Simple args to unicode conversion and concatenation.""" - return concat(imap(unicode, seq)) + return concat(imap(text_type, seq)) def new_context(environment, template_name, blocks, vars=None, @@ -62,7 +64,7 @@ def new_context(environment, template_name, blocks, vars=None, # we don't want to modify the dict passed if shared: parent = dict(parent) - for key, value in locals.iteritems(): + for key, value in iteritems(locals): if key[:2] == 'l_' and value is not missing: parent[key[2:]] = value return Context(environment, parent, template_name, blocks) @@ -76,8 +78,6 @@ class TemplateReference(object): def __getitem__(self, name): blocks = self.__context.blocks[name] - wrap = self.__context.eval_ctx.autoescape and \ - Markup or (lambda x: x) return BlockReference(name, self.__context, blocks, 0) def __repr__(self): @@ -120,7 +120,7 @@ class Context(object): # create the initial mapping of blocks. Whenever template inheritance # takes place the runtime will update this mapping with the new blocks # from the template. - self.blocks = dict((k, [v]) for k, v in blocks.iteritems()) + self.blocks = dict((k, [v]) for k, v in iteritems(blocks)) def super(self, name, current): """Render a parent block.""" @@ -172,6 +172,16 @@ class Context(object): """ if __debug__: __traceback_hide__ = True + + # Allow callable classes to take a context + fn = __obj.__call__ + for fn_type in ('contextfunction', + 'evalcontextfunction', + 'environmentfunction'): + if hasattr(fn, fn_type): + __obj = fn + break + if isinstance(__obj, _context_function_types): if getattr(__obj, 'contextfunction', 0): args = (__self,) + args @@ -190,8 +200,9 @@ class Context(object): """Internal helper function to create a derived context.""" context = new_context(self.environment, self.name, {}, self.parent, True, None, locals) + context.vars.update(self.vars) context.eval_ctx = self.eval_ctx - context.blocks.update((k, list(v)) for k, v in self.blocks.iteritems()) + context.blocks.update((k, list(v)) for k, v in iteritems(self.blocks)) return context def _all(meth): @@ -205,7 +216,7 @@ class Context(object): items = _all('items') # not available on python 3 - if hasattr(dict, 'iterkeys'): + if PY2: iterkeys = _all('iterkeys') itervalues = _all('itervalues') iteritems = _all('iteritems') @@ -269,10 +280,12 @@ class BlockReference(object): class LoopContext(object): """A loop context for dynamic iteration.""" - def __init__(self, iterable, recurse=None): + def __init__(self, iterable, recurse=None, depth0=0): self._iterator = iter(iterable) self._recurse = recurse + self._after = self._safe_next() self.index0 = -1 + self.depth0 = depth0 # try to get the length of the iterable early. This must be done # here because there are some broken iterators around where there @@ -290,10 +303,11 @@ class LoopContext(object): return args[self.index0 % len(args)] first = property(lambda x: x.index0 == 0) - last = property(lambda x: x.index0 + 1 == x.length) + last = property(lambda x: x._after is _last_iteration) index = property(lambda x: x.index0 + 1) revindex = property(lambda x: x.length - x.index0) revindex0 = property(lambda x: x.length - x.index) + depth = property(lambda x: x.depth0 + 1) def __len__(self): return self.length @@ -301,12 +315,18 @@ class LoopContext(object): def __iter__(self): return LoopContextIterator(self) + def _safe_next(self): + try: + return next(self._iterator) + except StopIteration: + return _last_iteration + @internalcode def loop(self, iterable): if self._recurse is None: raise TypeError('Tried to call non recursive loop. Maybe you ' "forgot the 'recursive' modifier.") - return self._recurse(iterable, self._recurse) + return self._recurse(iterable, self._recurse, self.depth0 + 1) # a nifty trick to enhance the error message if someone tried to call # the the loop without or with too many arguments. @@ -333,6 +353,7 @@ class LoopContext(object): ) +@implements_iterator class LoopContextIterator(object): """The iterator for a loop context.""" __slots__ = ('context',) @@ -343,10 +364,14 @@ class LoopContextIterator(object): def __iter__(self): return self - def next(self): + def __next__(self): ctx = self.context ctx.index0 += 1 - return next(ctx._iterator), ctx + if ctx._after is _last_iteration: + raise StopIteration() + next_elem = ctx._after + ctx._after = ctx._safe_next() + return next_elem, ctx class Macro(object): @@ -413,6 +438,7 @@ class Macro(object): ) +@implements_to_string class Undefined(object): """The default undefined type. This undefined type can be printed and iterated over, but every other access will raise an :exc:`UndefinedError`: @@ -444,7 +470,7 @@ class Undefined(object): if self._undefined_hint is None: if self._undefined_obj is missing: hint = '%r is undefined' % self._undefined_name - elif not isinstance(self._undefined_name, basestring): + elif not isinstance(self._undefined_name, string_types): hint = '%s has no element %r' % ( object_type_repr(self._undefined_obj), self._undefined_name @@ -458,21 +484,29 @@ class Undefined(object): hint = self._undefined_hint raise self._undefined_exception(hint) + @internalcode + def __getattr__(self, name): + if name[:2] == '__': + raise AttributeError(name) + return self._fail_with_undefined_error() + __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \ __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ - __getattr__ = __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = \ - __int__ = __float__ = __complex__ = __pow__ = __rpow__ = \ + __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = __int__ = \ + __float__ = __complex__ = __pow__ = __rpow__ = \ _fail_with_undefined_error - def __str__(self): - return unicode(self).encode('utf-8') + def __eq__(self, other): + return type(self) is type(other) + + def __ne__(self, other): + return not self.__eq__(other) - # unicode goes after __str__ because we configured 2to3 to rename - # __unicode__ to __str__. because the 2to3 tree is not designed to - # remove nodes from it, we leave the above __str__ around and let - # it override at runtime. - def __unicode__(self): + def __hash__(self): + return id(type(self)) + + def __str__(self): return u'' def __len__(self): @@ -489,6 +523,7 @@ class Undefined(object): return 'Undefined' +@implements_to_string class DebugUndefined(Undefined): """An undefined that returns the debug info when printed. @@ -504,7 +539,7 @@ class DebugUndefined(Undefined): """ __slots__ = () - def __unicode__(self): + def __str__(self): if self._undefined_hint is None: if self._undefined_obj is missing: return u'{{ %s }}' % self._undefined_name @@ -515,6 +550,7 @@ class DebugUndefined(Undefined): return u'{{ undefined value printed: %s }}' % self._undefined_hint +@implements_to_string class StrictUndefined(Undefined): """An undefined that barks on print and iteration as well as boolean tests and all kinds of comparisons. In other words: you can do nothing @@ -535,8 +571,9 @@ class StrictUndefined(Undefined): UndefinedError: 'foo' is undefined """ __slots__ = () - __iter__ = __unicode__ = __str__ = __len__ = __nonzero__ = __eq__ = \ - __ne__ = __bool__ = Undefined._fail_with_undefined_error + __iter__ = __str__ = __len__ = __nonzero__ = __eq__ = \ + __ne__ = __bool__ = __hash__ = \ + Undefined._fail_with_undefined_error # remove remaining slots attributes, after the metaclass did the magic they diff --git a/module/lib/jinja2/sandbox.py b/module/lib/jinja2/sandbox.py index 749719548..da479c1ba 100644 --- a/module/lib/jinja2/sandbox.py +++ b/module/lib/jinja2/sandbox.py @@ -13,11 +13,10 @@ :license: BSD. """ import operator -from jinja2.runtime import Undefined from jinja2.environment import Environment from jinja2.exceptions import SecurityError -from jinja2.utils import FunctionType, MethodType, TracebackType, CodeType, \ - FrameType, GeneratorType +from jinja2._compat import string_types, function_type, method_type, \ + traceback_type, code_type, frame_type, generator_type, PY2 #: maximum number of items a range may produce @@ -30,6 +29,13 @@ UNSAFE_FUNCTION_ATTRIBUTES = set(['func_closure', 'func_code', 'func_dict', #: unsafe method attributes. function attributes are unsafe for methods too UNSAFE_METHOD_ATTRIBUTES = set(['im_class', 'im_func', 'im_self']) +#: unsafe generator attirbutes. +UNSAFE_GENERATOR_ATTRIBUTES = set(['gi_frame', 'gi_code']) + +# On versions > python 2 the special attributes on functions are gone, +# but they remain on methods and generators for whatever reason. +if not PY2: + UNSAFE_FUNCTION_ATTRIBUTES = set() import warnings @@ -91,7 +97,7 @@ def safe_range(*args): """A range that can't generate ranges with a length of more than MAX_RANGE items. """ - rng = xrange(*args) + rng = range(*args) if len(rng) > MAX_RANGE: raise OverflowError('range too big, maximum size for range is %d' % MAX_RANGE) @@ -99,8 +105,9 @@ def safe_range(*args): def unsafe(f): - """ - Mark a function or method as unsafe:: + """Marks a function or method as unsafe. + + :: @unsafe def delete(self): @@ -114,7 +121,7 @@ def is_internal_attribute(obj, attr): """Test if the attribute given is an internal python attribute. For example this function returns `True` for the `func_code` attribute of python objects. This is useful if the environment method - :meth:`~SandboxedEnvironment.is_safe_attribute` is overriden. + :meth:`~SandboxedEnvironment.is_safe_attribute` is overridden. >>> from jinja2.sandbox import is_internal_attribute >>> is_internal_attribute(lambda: None, "func_code") @@ -124,20 +131,20 @@ def is_internal_attribute(obj, attr): >>> is_internal_attribute(str, "upper") False """ - if isinstance(obj, FunctionType): + if isinstance(obj, function_type): if attr in UNSAFE_FUNCTION_ATTRIBUTES: return True - elif isinstance(obj, MethodType): + elif isinstance(obj, method_type): if attr in UNSAFE_FUNCTION_ATTRIBUTES or \ attr in UNSAFE_METHOD_ATTRIBUTES: return True elif isinstance(obj, type): if attr == 'mro': return True - elif isinstance(obj, (CodeType, TracebackType, FrameType)): + elif isinstance(obj, (code_type, traceback_type, frame_type)): return True - elif isinstance(obj, GeneratorType): - if attr == 'gi_frame': + elif isinstance(obj, generator_type): + if attr in UNSAFE_GENERATOR_ATTRIBUTES: return True return attr.startswith('__') @@ -182,9 +189,81 @@ class SandboxedEnvironment(Environment): """ sandboxed = True + #: default callback table for the binary operators. A copy of this is + #: available on each instance of a sandboxed environment as + #: :attr:`binop_table` + default_binop_table = { + '+': operator.add, + '-': operator.sub, + '*': operator.mul, + '/': operator.truediv, + '//': operator.floordiv, + '**': operator.pow, + '%': operator.mod + } + + #: default callback table for the unary operators. A copy of this is + #: available on each instance of a sandboxed environment as + #: :attr:`unop_table` + default_unop_table = { + '+': operator.pos, + '-': operator.neg + } + + #: a set of binary operators that should be intercepted. Each operator + #: that is added to this set (empty by default) is delegated to the + #: :meth:`call_binop` method that will perform the operator. The default + #: operator callback is specified by :attr:`binop_table`. + #: + #: The following binary operators are interceptable: + #: ``//``, ``%``, ``+``, ``*``, ``-``, ``/``, and ``**`` + #: + #: The default operation form the operator table corresponds to the + #: builtin function. Intercepted calls are always slower than the native + #: operator call, so make sure only to intercept the ones you are + #: interested in. + #: + #: .. versionadded:: 2.6 + intercepted_binops = frozenset() + + #: a set of unary operators that should be intercepted. Each operator + #: that is added to this set (empty by default) is delegated to the + #: :meth:`call_unop` method that will perform the operator. The default + #: operator callback is specified by :attr:`unop_table`. + #: + #: The following unary operators are interceptable: ``+``, ``-`` + #: + #: The default operation form the operator table corresponds to the + #: builtin function. Intercepted calls are always slower than the native + #: operator call, so make sure only to intercept the ones you are + #: interested in. + #: + #: .. versionadded:: 2.6 + intercepted_unops = frozenset() + + def intercept_unop(self, operator): + """Called during template compilation with the name of a unary + operator to check if it should be intercepted at runtime. If this + method returns `True`, :meth:`call_unop` is excuted for this unary + operator. The default implementation of :meth:`call_unop` will use + the :attr:`unop_table` dictionary to perform the operator with the + same logic as the builtin one. + + The following unary operators are interceptable: ``+`` and ``-`` + + Intercepted calls are always slower than the native operator call, + so make sure only to intercept the ones you are interested in. + + .. versionadded:: 2.6 + """ + return False + + def __init__(self, *args, **kwargs): Environment.__init__(self, *args, **kwargs) self.globals['range'] = safe_range + self.binop_table = self.default_binop_table.copy() + self.unop_table = self.default_unop_table.copy() def is_safe_attribute(self, obj, attr, value): """The sandboxed environment will call this method to check if the @@ -201,18 +280,36 @@ class SandboxedEnvironment(Environment): True. Override this method to alter the behavior, but this won't affect the `unsafe` decorator from this module. """ - return not (getattr(obj, 'unsafe_callable', False) or \ + return not (getattr(obj, 'unsafe_callable', False) or getattr(obj, 'alters_data', False)) + def call_binop(self, context, operator, left, right): + """For intercepted binary operator calls (:meth:`intercepted_binops`) + this function is executed instead of the builtin operator. This can + be used to fine tune the behavior of certain operators. + + .. versionadded:: 2.6 + """ + return self.binop_table[operator](left, right) + + def call_unop(self, context, operator, arg): + """For intercepted unary operator calls (:meth:`intercepted_unops`) + this function is executed instead of the builtin operator. This can + be used to fine tune the behavior of certain operators. + + .. versionadded:: 2.6 + """ + return self.unop_table[operator](arg) + def getitem(self, obj, argument): """Subscribe an object from sandboxed code.""" try: return obj[argument] except (TypeError, LookupError): - if isinstance(argument, basestring): + if isinstance(argument, string_types): try: attr = str(argument) - except: + except Exception: pass else: try: diff --git a/module/lib/jinja2/tests.py b/module/lib/jinja2/tests.py index d257eca0a..48a3e0618 100644 --- a/module/lib/jinja2/tests.py +++ b/module/lib/jinja2/tests.py @@ -10,20 +10,14 @@ """ import re from jinja2.runtime import Undefined - -# nose, nothing here to test -__test__ = False +from jinja2._compat import text_type, string_types, mapping_types number_re = re.compile(r'^-?\d+(\.\d+)?$') regex_type = type(number_re) -try: - test_callable = callable -except NameError: - def test_callable(x): - return hasattr(x, '__call__') +test_callable = callable def test_odd(value): @@ -70,22 +64,30 @@ def test_none(value): def test_lower(value): """Return true if the variable is lowercased.""" - return unicode(value).islower() + return text_type(value).islower() def test_upper(value): """Return true if the variable is uppercased.""" - return unicode(value).isupper() + return text_type(value).isupper() def test_string(value): """Return true if the object is a string.""" - return isinstance(value, basestring) + return isinstance(value, string_types) + + +def test_mapping(value): + """Return true if the object is a mapping (dict etc.). + + .. versionadded:: 2.6 + """ + return isinstance(value, mapping_types) def test_number(value): """Return true if the variable is a number.""" - return isinstance(value, (int, long, float, complex)) + return isinstance(value, (int, float, complex)) def test_sequence(value): @@ -137,6 +139,7 @@ TESTS = { 'lower': test_lower, 'upper': test_upper, 'string': test_string, + 'mapping': test_mapping, 'number': test_number, 'sequence': test_sequence, 'iterable': test_iterable, diff --git a/module/lib/jinja2/testsuite/__init__.py b/module/lib/jinja2/testsuite/__init__.py new file mode 100644 index 000000000..635c83e5d --- /dev/null +++ b/module/lib/jinja2/testsuite/__init__.py @@ -0,0 +1,156 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite + ~~~~~~~~~~~~~~~~ + + All the unittests of Jinja2. These tests can be executed by + either running run-tests.py using multiple Python versions at + the same time. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import os +import re +import sys +import unittest +from traceback import format_exception +from jinja2 import loaders +from jinja2._compat import PY2 + + +here = os.path.dirname(os.path.abspath(__file__)) + +dict_loader = loaders.DictLoader({ + 'justdict.html': 'FOO' +}) +package_loader = loaders.PackageLoader('jinja2.testsuite.res', 'templates') +filesystem_loader = loaders.FileSystemLoader(here + '/res/templates') +function_loader = loaders.FunctionLoader({'justfunction.html': 'FOO'}.get) +choice_loader = loaders.ChoiceLoader([dict_loader, package_loader]) +prefix_loader = loaders.PrefixLoader({ + 'a': filesystem_loader, + 'b': dict_loader +}) + + +class JinjaTestCase(unittest.TestCase): + + ### use only these methods for testing. If you need standard + ### unittest method, wrap them! + + def setup(self): + pass + + def teardown(self): + pass + + def setUp(self): + self.setup() + + def tearDown(self): + self.teardown() + + def assert_equal(self, a, b): + return self.assertEqual(a, b) + + def assert_raises(self, *args, **kwargs): + return self.assertRaises(*args, **kwargs) + + def assert_traceback_matches(self, callback, expected_tb): + try: + callback() + except Exception as e: + tb = format_exception(*sys.exc_info()) + if re.search(expected_tb.strip(), ''.join(tb)) is None: + raise self.fail('Traceback did not match:\n\n%s\nexpected:\n%s' + % (''.join(tb), expected_tb)) + else: + self.fail('Expected exception') + + +def find_all_tests(suite): + """Yields all the tests and their names from a given suite.""" + suites = [suite] + while suites: + s = suites.pop() + try: + suites.extend(s) + except TypeError: + yield s, '%s.%s.%s' % ( + s.__class__.__module__, + s.__class__.__name__, + s._testMethodName + ) + + +class BetterLoader(unittest.TestLoader): + """A nicer loader that solves two problems. First of all we are setting + up tests from different sources and we're doing this programmatically + which breaks the default loading logic so this is required anyways. + Secondly this loader has a nicer interpolation for test names than the + default one so you can just do ``run-tests.py ViewTestCase`` and it + will work. + """ + + def getRootSuite(self): + return suite() + + def loadTestsFromName(self, name, module=None): + root = self.getRootSuite() + if name == 'suite': + return root + + all_tests = [] + for testcase, testname in find_all_tests(root): + if testname == name or \ + testname.endswith('.' + name) or \ + ('.' + name + '.') in testname or \ + testname.startswith(name + '.'): + all_tests.append(testcase) + + if not all_tests: + raise LookupError('could not find test case for "%s"' % name) + + if len(all_tests) == 1: + return all_tests[0] + rv = unittest.TestSuite() + for test in all_tests: + rv.addTest(test) + return rv + + +def suite(): + from jinja2.testsuite import ext, filters, tests, core_tags, \ + loader, inheritance, imports, lexnparse, security, api, \ + regression, debug, utils, bytecode_cache, doctests + suite = unittest.TestSuite() + suite.addTest(ext.suite()) + suite.addTest(filters.suite()) + suite.addTest(tests.suite()) + suite.addTest(core_tags.suite()) + suite.addTest(loader.suite()) + suite.addTest(inheritance.suite()) + suite.addTest(imports.suite()) + suite.addTest(lexnparse.suite()) + suite.addTest(security.suite()) + suite.addTest(api.suite()) + suite.addTest(regression.suite()) + suite.addTest(debug.suite()) + suite.addTest(utils.suite()) + suite.addTest(bytecode_cache.suite()) + + # doctests will not run on python 3 currently. Too many issues + # with that, do not test that on that platform. + if PY2: + suite.addTest(doctests.suite()) + + return suite + + +def main(): + """Runs the testsuite as command line application.""" + try: + unittest.main(testLoader=BetterLoader(), defaultTest='suite') + except Exception as e: + print('Error: %s' % e) diff --git a/module/lib/jinja2/testsuite/api.py b/module/lib/jinja2/testsuite/api.py new file mode 100644 index 000000000..1b68bf8b3 --- /dev/null +++ b/module/lib/jinja2/testsuite/api.py @@ -0,0 +1,261 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.api + ~~~~~~~~~~~~~~~~~~~~ + + Tests the public API and related stuff. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest +import os +import tempfile +import shutil + +from jinja2.testsuite import JinjaTestCase +from jinja2._compat import next + +from jinja2 import Environment, Undefined, DebugUndefined, \ + StrictUndefined, UndefinedError, meta, \ + is_undefined, Template, DictLoader +from jinja2.utils import Cycler + +env = Environment() + + +class ExtendedAPITestCase(JinjaTestCase): + + def test_item_and_attribute(self): + from jinja2.sandbox import SandboxedEnvironment + + for env in Environment(), SandboxedEnvironment(): + # the |list is necessary for python3 + tmpl = env.from_string('{{ foo.items()|list }}') + assert tmpl.render(foo={'items': 42}) == "[('items', 42)]" + tmpl = env.from_string('{{ foo|attr("items")()|list }}') + assert tmpl.render(foo={'items': 42}) == "[('items', 42)]" + tmpl = env.from_string('{{ foo["items"] }}') + assert tmpl.render(foo={'items': 42}) == '42' + + def test_finalizer(self): + def finalize_none_empty(value): + if value is None: + value = u'' + return value + env = Environment(finalize=finalize_none_empty) + tmpl = env.from_string('{% for item in seq %}|{{ item }}{% endfor %}') + assert tmpl.render(seq=(None, 1, "foo")) == '||1|foo' + tmpl = env.from_string('<{{ none }}>') + assert tmpl.render() == '<>' + + def test_cycler(self): + items = 1, 2, 3 + c = Cycler(*items) + for item in items + items: + assert c.current == item + assert next(c) == item + next(c) + assert c.current == 2 + c.reset() + assert c.current == 1 + + def test_expressions(self): + expr = env.compile_expression("foo") + assert expr() is None + assert expr(foo=42) == 42 + expr2 = env.compile_expression("foo", undefined_to_none=False) + assert is_undefined(expr2()) + + expr = env.compile_expression("42 + foo") + assert expr(foo=42) == 84 + + def test_template_passthrough(self): + t = Template('Content') + assert env.get_template(t) is t + assert env.select_template([t]) is t + assert env.get_or_select_template([t]) is t + assert env.get_or_select_template(t) is t + + def test_autoescape_autoselect(self): + def select_autoescape(name): + if name is None or '.' not in name: + return False + return name.endswith('.html') + env = Environment(autoescape=select_autoescape, + loader=DictLoader({ + 'test.txt': '{{ foo }}', + 'test.html': '{{ foo }}' + })) + t = env.get_template('test.txt') + assert t.render(foo='<foo>') == '<foo>' + t = env.get_template('test.html') + assert t.render(foo='<foo>') == '<foo>' + t = env.from_string('{{ foo }}') + assert t.render(foo='<foo>') == '<foo>' + + +class MetaTestCase(JinjaTestCase): + + def test_find_undeclared_variables(self): + ast = env.parse('{% set foo = 42 %}{{ bar + foo }}') + x = meta.find_undeclared_variables(ast) + assert x == set(['bar']) + + ast = env.parse('{% set foo = 42 %}{{ bar + foo }}' + '{% macro meh(x) %}{{ x }}{% endmacro %}' + '{% for item in seq %}{{ muh(item) + meh(seq) }}{% endfor %}') + x = meta.find_undeclared_variables(ast) + assert x == set(['bar', 'seq', 'muh']) + + def test_find_refererenced_templates(self): + ast = env.parse('{% extends "layout.html" %}{% include helper %}') + i = meta.find_referenced_templates(ast) + assert next(i) == 'layout.html' + assert next(i) is None + assert list(i) == [] + + ast = env.parse('{% extends "layout.html" %}' + '{% from "test.html" import a, b as c %}' + '{% import "meh.html" as meh %}' + '{% include "muh.html" %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ['layout.html', 'test.html', 'meh.html', 'muh.html'] + + def test_find_included_templates(self): + ast = env.parse('{% include ["foo.html", "bar.html"] %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ['foo.html', 'bar.html'] + + ast = env.parse('{% include ("foo.html", "bar.html") %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ['foo.html', 'bar.html'] + + ast = env.parse('{% include ["foo.html", "bar.html", foo] %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ['foo.html', 'bar.html', None] + + ast = env.parse('{% include ("foo.html", "bar.html", foo) %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ['foo.html', 'bar.html', None] + + +class StreamingTestCase(JinjaTestCase): + + def test_basic_streaming(self): + tmpl = env.from_string("<ul>{% for item in seq %}<li>{{ loop.index " + "}} - {{ item }}</li>{%- endfor %}</ul>") + stream = tmpl.stream(seq=list(range(4))) + self.assert_equal(next(stream), '<ul>') + self.assert_equal(next(stream), '<li>1 - 0</li>') + self.assert_equal(next(stream), '<li>2 - 1</li>') + self.assert_equal(next(stream), '<li>3 - 2</li>') + self.assert_equal(next(stream), '<li>4 - 3</li>') + self.assert_equal(next(stream), '</ul>') + + def test_buffered_streaming(self): + tmpl = env.from_string("<ul>{% for item in seq %}<li>{{ loop.index " + "}} - {{ item }}</li>{%- endfor %}</ul>") + stream = tmpl.stream(seq=list(range(4))) + stream.enable_buffering(size=3) + self.assert_equal(next(stream), u'<ul><li>1 - 0</li><li>2 - 1</li>') + self.assert_equal(next(stream), u'<li>3 - 2</li><li>4 - 3</li></ul>') + + def test_streaming_behavior(self): + tmpl = env.from_string("") + stream = tmpl.stream() + assert not stream.buffered + stream.enable_buffering(20) + assert stream.buffered + stream.disable_buffering() + assert not stream.buffered + + def test_dump_stream(self): + tmp = tempfile.mkdtemp() + try: + tmpl = env.from_string(u"\u2713") + stream = tmpl.stream() + stream.dump(os.path.join(tmp, 'dump.txt'), 'utf-8') + with open(os.path.join(tmp, 'dump.txt'), 'rb') as f: + self.assertEqual(f.read(), b'\xe2\x9c\x93') + finally: + shutil.rmtree(tmp) + + +class UndefinedTestCase(JinjaTestCase): + + def test_stopiteration_is_undefined(self): + def test(): + raise StopIteration() + t = Template('A{{ test() }}B') + assert t.render(test=test) == 'AB' + t = Template('A{{ test().missingattribute }}B') + self.assert_raises(UndefinedError, t.render, test=test) + + def test_undefined_and_special_attributes(self): + try: + Undefined('Foo').__dict__ + except AttributeError: + pass + else: + assert False, "Expected actual attribute error" + + def test_default_undefined(self): + env = Environment(undefined=Undefined) + self.assert_equal(env.from_string('{{ missing }}').render(), u'') + self.assert_raises(UndefinedError, + env.from_string('{{ missing.attribute }}').render) + self.assert_equal(env.from_string('{{ missing|list }}').render(), '[]') + self.assert_equal(env.from_string('{{ missing is not defined }}').render(), 'True') + self.assert_equal(env.from_string('{{ foo.missing }}').render(foo=42), '') + self.assert_equal(env.from_string('{{ not missing }}').render(), 'True') + + def test_debug_undefined(self): + env = Environment(undefined=DebugUndefined) + self.assert_equal(env.from_string('{{ missing }}').render(), '{{ missing }}') + self.assert_raises(UndefinedError, + env.from_string('{{ missing.attribute }}').render) + self.assert_equal(env.from_string('{{ missing|list }}').render(), '[]') + self.assert_equal(env.from_string('{{ missing is not defined }}').render(), 'True') + self.assert_equal(env.from_string('{{ foo.missing }}').render(foo=42), + u"{{ no such element: int object['missing'] }}") + self.assert_equal(env.from_string('{{ not missing }}').render(), 'True') + + def test_strict_undefined(self): + env = Environment(undefined=StrictUndefined) + self.assert_raises(UndefinedError, env.from_string('{{ missing }}').render) + self.assert_raises(UndefinedError, env.from_string('{{ missing.attribute }}').render) + self.assert_raises(UndefinedError, env.from_string('{{ missing|list }}').render) + self.assert_equal(env.from_string('{{ missing is not defined }}').render(), 'True') + self.assert_raises(UndefinedError, env.from_string('{{ foo.missing }}').render, foo=42) + self.assert_raises(UndefinedError, env.from_string('{{ not missing }}').render) + self.assert_equal(env.from_string('{{ missing|default("default", true) }}').render(), 'default') + + def test_indexing_gives_undefined(self): + t = Template("{{ var[42].foo }}") + self.assert_raises(UndefinedError, t.render, var=0) + + def test_none_gives_proper_error(self): + try: + Environment().getattr(None, 'split')() + except UndefinedError as e: + assert e.message == "'None' has no attribute 'split'" + else: + assert False, 'expected exception' + + def test_object_repr(self): + try: + Undefined(obj=42, name='upper')() + except UndefinedError as e: + assert e.message == "'int object' has no attribute 'upper'" + else: + assert False, 'expected exception' + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(ExtendedAPITestCase)) + suite.addTest(unittest.makeSuite(MetaTestCase)) + suite.addTest(unittest.makeSuite(StreamingTestCase)) + suite.addTest(unittest.makeSuite(UndefinedTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/bytecode_cache.py b/module/lib/jinja2/testsuite/bytecode_cache.py new file mode 100644 index 000000000..9f5c635b8 --- /dev/null +++ b/module/lib/jinja2/testsuite/bytecode_cache.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.bytecode_cache + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Test bytecode caching + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from jinja2.testsuite import JinjaTestCase, package_loader + +from jinja2 import Environment +from jinja2.bccache import FileSystemBytecodeCache +from jinja2.exceptions import TemplateNotFound + +bytecode_cache = FileSystemBytecodeCache() +env = Environment( + loader=package_loader, + bytecode_cache=bytecode_cache, +) + + +class ByteCodeCacheTestCase(JinjaTestCase): + + def test_simple(self): + tmpl = env.get_template('test.html') + assert tmpl.render().strip() == 'BAR' + self.assert_raises(TemplateNotFound, env.get_template, 'missing.html') + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(ByteCodeCacheTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/core_tags.py b/module/lib/jinja2/testsuite/core_tags.py new file mode 100644 index 000000000..f1a20fd44 --- /dev/null +++ b/module/lib/jinja2/testsuite/core_tags.py @@ -0,0 +1,305 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.core_tags + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Test the core tags like for and if. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from jinja2.testsuite import JinjaTestCase + +from jinja2 import Environment, TemplateSyntaxError, UndefinedError, \ + DictLoader + +env = Environment() + + +class ForLoopTestCase(JinjaTestCase): + + def test_simple(self): + tmpl = env.from_string('{% for item in seq %}{{ item }}{% endfor %}') + assert tmpl.render(seq=list(range(10))) == '0123456789' + + def test_else(self): + tmpl = env.from_string('{% for item in seq %}XXX{% else %}...{% endfor %}') + assert tmpl.render() == '...' + + def test_empty_blocks(self): + tmpl = env.from_string('<{% for item in seq %}{% else %}{% endfor %}>') + assert tmpl.render() == '<>' + + def test_context_vars(self): + tmpl = env.from_string('''{% for item in seq -%} + {{ loop.index }}|{{ loop.index0 }}|{{ loop.revindex }}|{{ + loop.revindex0 }}|{{ loop.first }}|{{ loop.last }}|{{ + loop.length }}###{% endfor %}''') + one, two, _ = tmpl.render(seq=[0, 1]).split('###') + (one_index, one_index0, one_revindex, one_revindex0, one_first, + one_last, one_length) = one.split('|') + (two_index, two_index0, two_revindex, two_revindex0, two_first, + two_last, two_length) = two.split('|') + + assert int(one_index) == 1 and int(two_index) == 2 + assert int(one_index0) == 0 and int(two_index0) == 1 + assert int(one_revindex) == 2 and int(two_revindex) == 1 + assert int(one_revindex0) == 1 and int(two_revindex0) == 0 + assert one_first == 'True' and two_first == 'False' + assert one_last == 'False' and two_last == 'True' + assert one_length == two_length == '2' + + def test_cycling(self): + tmpl = env.from_string('''{% for item in seq %}{{ + loop.cycle('<1>', '<2>') }}{% endfor %}{% + for item in seq %}{{ loop.cycle(*through) }}{% endfor %}''') + output = tmpl.render(seq=list(range(4)), through=('<1>', '<2>')) + assert output == '<1><2>' * 4 + + def test_scope(self): + tmpl = env.from_string('{% for item in seq %}{% endfor %}{{ item }}') + output = tmpl.render(seq=list(range(10))) + assert not output + + def test_varlen(self): + def inner(): + for item in range(5): + yield item + tmpl = env.from_string('{% for item in iter %}{{ item }}{% endfor %}') + output = tmpl.render(iter=inner()) + assert output == '01234' + + def test_noniter(self): + tmpl = env.from_string('{% for item in none %}...{% endfor %}') + self.assert_raises(TypeError, tmpl.render) + + def test_recursive(self): + tmpl = env.from_string('''{% for item in seq recursive -%} + [{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}''') + assert tmpl.render(seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a='a')]) + ]) == '[1<[1][2]>][2<[1][2]>][3<[a]>]' + + def test_recursive_depth0(self): + tmpl = env.from_string('''{% for item in seq recursive -%} + [{{ loop.depth0 }}:{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}''') + self.assertEqual(tmpl.render(seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a='a')]) + ]), '[0:1<[1:1][1:2]>][0:2<[1:1][1:2]>][0:3<[1:a]>]') + + def test_recursive_depth(self): + tmpl = env.from_string('''{% for item in seq recursive -%} + [{{ loop.depth }}:{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}''') + self.assertEqual(tmpl.render(seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a='a')]) + ]), '[1:1<[2:1][2:2]>][1:2<[2:1][2:2]>][1:3<[2:a]>]') + + def test_looploop(self): + tmpl = env.from_string('''{% for row in table %} + {%- set rowloop = loop -%} + {% for cell in row -%} + [{{ rowloop.index }}|{{ loop.index }}] + {%- endfor %} + {%- endfor %}''') + assert tmpl.render(table=['ab', 'cd']) == '[1|1][1|2][2|1][2|2]' + + def test_reversed_bug(self): + tmpl = env.from_string('{% for i in items %}{{ i }}' + '{% if not loop.last %}' + ',{% endif %}{% endfor %}') + assert tmpl.render(items=reversed([3, 2, 1])) == '1,2,3' + + def test_loop_errors(self): + tmpl = env.from_string('''{% for item in [1] if loop.index + == 0 %}...{% endfor %}''') + self.assert_raises(UndefinedError, tmpl.render) + tmpl = env.from_string('''{% for item in [] %}...{% else + %}{{ loop }}{% endfor %}''') + assert tmpl.render() == '' + + def test_loop_filter(self): + tmpl = env.from_string('{% for item in range(10) if item ' + 'is even %}[{{ item }}]{% endfor %}') + assert tmpl.render() == '[0][2][4][6][8]' + tmpl = env.from_string(''' + {%- for item in range(10) if item is even %}[{{ + loop.index }}:{{ item }}]{% endfor %}''') + assert tmpl.render() == '[1:0][2:2][3:4][4:6][5:8]' + + def test_loop_unassignable(self): + self.assert_raises(TemplateSyntaxError, env.from_string, + '{% for loop in seq %}...{% endfor %}') + + def test_scoped_special_var(self): + t = env.from_string('{% for s in seq %}[{{ loop.first }}{% for c in s %}' + '|{{ loop.first }}{% endfor %}]{% endfor %}') + assert t.render(seq=('ab', 'cd')) == '[True|True|False][False|True|False]' + + def test_scoped_loop_var(self): + t = env.from_string('{% for x in seq %}{{ loop.first }}' + '{% for y in seq %}{% endfor %}{% endfor %}') + assert t.render(seq='ab') == 'TrueFalse' + t = env.from_string('{% for x in seq %}{% for y in seq %}' + '{{ loop.first }}{% endfor %}{% endfor %}') + assert t.render(seq='ab') == 'TrueFalseTrueFalse' + + def test_recursive_empty_loop_iter(self): + t = env.from_string(''' + {%- for item in foo recursive -%}{%- endfor -%} + ''') + assert t.render(dict(foo=[])) == '' + + def test_call_in_loop(self): + t = env.from_string(''' + {%- macro do_something() -%} + [{{ caller() }}] + {%- endmacro %} + + {%- for i in [1, 2, 3] %} + {%- call do_something() -%} + {{ i }} + {%- endcall %} + {%- endfor -%} + ''') + assert t.render() == '[1][2][3]' + + def test_scoping_bug(self): + t = env.from_string(''' + {%- for item in foo %}...{{ item }}...{% endfor %} + {%- macro item(a) %}...{{ a }}...{% endmacro %} + {{- item(2) -}} + ''') + assert t.render(foo=(1,)) == '...1......2...' + + def test_unpacking(self): + tmpl = env.from_string('{% for a, b, c in [[1, 2, 3]] %}' + '{{ a }}|{{ b }}|{{ c }}{% endfor %}') + assert tmpl.render() == '1|2|3' + + +class IfConditionTestCase(JinjaTestCase): + + def test_simple(self): + tmpl = env.from_string('''{% if true %}...{% endif %}''') + assert tmpl.render() == '...' + + def test_elif(self): + tmpl = env.from_string('''{% if false %}XXX{% elif true + %}...{% else %}XXX{% endif %}''') + assert tmpl.render() == '...' + + def test_else(self): + tmpl = env.from_string('{% if false %}XXX{% else %}...{% endif %}') + assert tmpl.render() == '...' + + def test_empty(self): + tmpl = env.from_string('[{% if true %}{% else %}{% endif %}]') + assert tmpl.render() == '[]' + + def test_complete(self): + tmpl = env.from_string('{% if a %}A{% elif b %}B{% elif c == d %}' + 'C{% else %}D{% endif %}') + assert tmpl.render(a=0, b=False, c=42, d=42.0) == 'C' + + def test_no_scope(self): + tmpl = env.from_string('{% if a %}{% set foo = 1 %}{% endif %}{{ foo }}') + assert tmpl.render(a=True) == '1' + tmpl = env.from_string('{% if true %}{% set foo = 1 %}{% endif %}{{ foo }}') + assert tmpl.render() == '1' + + +class MacrosTestCase(JinjaTestCase): + env = Environment(trim_blocks=True) + + def test_simple(self): + tmpl = self.env.from_string('''\ +{% macro say_hello(name) %}Hello {{ name }}!{% endmacro %} +{{ say_hello('Peter') }}''') + assert tmpl.render() == 'Hello Peter!' + + def test_scoping(self): + tmpl = self.env.from_string('''\ +{% macro level1(data1) %} +{% macro level2(data2) %}{{ data1 }}|{{ data2 }}{% endmacro %} +{{ level2('bar') }}{% endmacro %} +{{ level1('foo') }}''') + assert tmpl.render() == 'foo|bar' + + def test_arguments(self): + tmpl = self.env.from_string('''\ +{% macro m(a, b, c='c', d='d') %}{{ a }}|{{ b }}|{{ c }}|{{ d }}{% endmacro %} +{{ m() }}|{{ m('a') }}|{{ m('a', 'b') }}|{{ m(1, 2, 3) }}''') + assert tmpl.render() == '||c|d|a||c|d|a|b|c|d|1|2|3|d' + + def test_varargs(self): + tmpl = self.env.from_string('''\ +{% macro test() %}{{ varargs|join('|') }}{% endmacro %}\ +{{ test(1, 2, 3) }}''') + assert tmpl.render() == '1|2|3' + + def test_simple_call(self): + tmpl = self.env.from_string('''\ +{% macro test() %}[[{{ caller() }}]]{% endmacro %}\ +{% call test() %}data{% endcall %}''') + assert tmpl.render() == '[[data]]' + + def test_complex_call(self): + tmpl = self.env.from_string('''\ +{% macro test() %}[[{{ caller('data') }}]]{% endmacro %}\ +{% call(data) test() %}{{ data }}{% endcall %}''') + assert tmpl.render() == '[[data]]' + + def test_caller_undefined(self): + tmpl = self.env.from_string('''\ +{% set caller = 42 %}\ +{% macro test() %}{{ caller is not defined }}{% endmacro %}\ +{{ test() }}''') + assert tmpl.render() == 'True' + + def test_include(self): + self.env = Environment(loader=DictLoader({'include': + '{% macro test(foo) %}[{{ foo }}]{% endmacro %}'})) + tmpl = self.env.from_string('{% from "include" import test %}{{ test("foo") }}') + assert tmpl.render() == '[foo]' + + def test_macro_api(self): + tmpl = self.env.from_string('{% macro foo(a, b) %}{% endmacro %}' + '{% macro bar() %}{{ varargs }}{{ kwargs }}{% endmacro %}' + '{% macro baz() %}{{ caller() }}{% endmacro %}') + assert tmpl.module.foo.arguments == ('a', 'b') + assert tmpl.module.foo.defaults == () + assert tmpl.module.foo.name == 'foo' + assert not tmpl.module.foo.caller + assert not tmpl.module.foo.catch_kwargs + assert not tmpl.module.foo.catch_varargs + assert tmpl.module.bar.arguments == () + assert tmpl.module.bar.defaults == () + assert not tmpl.module.bar.caller + assert tmpl.module.bar.catch_kwargs + assert tmpl.module.bar.catch_varargs + assert tmpl.module.baz.caller + + def test_callself(self): + tmpl = self.env.from_string('{% macro foo(x) %}{{ x }}{% if x > 1 %}|' + '{{ foo(x - 1) }}{% endif %}{% endmacro %}' + '{{ foo(5) }}') + assert tmpl.render() == '5|4|3|2|1' + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(ForLoopTestCase)) + suite.addTest(unittest.makeSuite(IfConditionTestCase)) + suite.addTest(unittest.makeSuite(MacrosTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/debug.py b/module/lib/jinja2/testsuite/debug.py new file mode 100644 index 000000000..2588a83ea --- /dev/null +++ b/module/lib/jinja2/testsuite/debug.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.debug + ~~~~~~~~~~~~~~~~~~~~~~ + + Tests the debug system. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from jinja2.testsuite import JinjaTestCase, filesystem_loader + +from jinja2 import Environment, TemplateSyntaxError + +env = Environment(loader=filesystem_loader) + + +class DebugTestCase(JinjaTestCase): + + def test_runtime_error(self): + def test(): + tmpl.render(fail=lambda: 1 / 0) + tmpl = env.get_template('broken.html') + self.assert_traceback_matches(test, r''' + File ".*?broken.html", line 2, in (top-level template code|<module>) + \{\{ fail\(\) \}\} + File ".*?debug.pyc?", line \d+, in <lambda> + tmpl\.render\(fail=lambda: 1 / 0\) +ZeroDivisionError: (int(eger)? )?division (or modulo )?by zero +''') + + def test_syntax_error(self): + # XXX: the .*? is necessary for python3 which does not hide + # some of the stack frames we don't want to show. Not sure + # what's up with that, but that is not that critical. Should + # be fixed though. + self.assert_traceback_matches(lambda: env.get_template('syntaxerror.html'), r'''(?sm) + File ".*?syntaxerror.html", line 4, in (template|<module>) + \{% endif %\}.*? +(jinja2\.exceptions\.)?TemplateSyntaxError: Encountered unknown tag 'endif'. Jinja was looking for the following tags: 'endfor' or 'else'. The innermost block that needs to be closed is 'for'. + ''') + + def test_regular_syntax_error(self): + def test(): + raise TemplateSyntaxError('wtf', 42) + self.assert_traceback_matches(test, r''' + File ".*debug.pyc?", line \d+, in test + raise TemplateSyntaxError\('wtf', 42\) +(jinja2\.exceptions\.)?TemplateSyntaxError: wtf + line 42''') + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(DebugTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/doctests.py b/module/lib/jinja2/testsuite/doctests.py new file mode 100644 index 000000000..616d3b6ee --- /dev/null +++ b/module/lib/jinja2/testsuite/doctests.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.doctests + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + The doctests. Collects all tests we want to test from + the Jinja modules. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest +import doctest + + +def suite(): + from jinja2 import utils, sandbox, runtime, meta, loaders, \ + ext, environment, bccache, nodes + suite = unittest.TestSuite() + suite.addTest(doctest.DocTestSuite(utils)) + suite.addTest(doctest.DocTestSuite(sandbox)) + suite.addTest(doctest.DocTestSuite(runtime)) + suite.addTest(doctest.DocTestSuite(meta)) + suite.addTest(doctest.DocTestSuite(loaders)) + suite.addTest(doctest.DocTestSuite(ext)) + suite.addTest(doctest.DocTestSuite(environment)) + suite.addTest(doctest.DocTestSuite(bccache)) + suite.addTest(doctest.DocTestSuite(nodes)) + return suite diff --git a/module/lib/jinja2/testsuite/ext.py b/module/lib/jinja2/testsuite/ext.py new file mode 100644 index 000000000..0f93be945 --- /dev/null +++ b/module/lib/jinja2/testsuite/ext.py @@ -0,0 +1,459 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.ext + ~~~~~~~~~~~~~~~~~~~~ + + Tests for the extensions. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import re +import unittest + +from jinja2.testsuite import JinjaTestCase + +from jinja2 import Environment, DictLoader, contextfunction, nodes +from jinja2.exceptions import TemplateAssertionError +from jinja2.ext import Extension +from jinja2.lexer import Token, count_newlines +from jinja2._compat import next, BytesIO, itervalues, text_type + +importable_object = 23 + +_gettext_re = re.compile(r'_\((.*?)\)(?s)') + + +i18n_templates = { + 'master.html': '<title>{{ page_title|default(_("missing")) }}</title>' + '{% block body %}{% endblock %}', + 'child.html': '{% extends "master.html" %}{% block body %}' + '{% trans %}watch out{% endtrans %}{% endblock %}', + 'plural.html': '{% trans user_count %}One user online{% pluralize %}' + '{{ user_count }} users online{% endtrans %}', + 'plural2.html': '{% trans user_count=get_user_count() %}{{ user_count }}s' + '{% pluralize %}{{ user_count }}p{% endtrans %}', + 'stringformat.html': '{{ _("User: %(num)s")|format(num=user_count) }}' +} + +newstyle_i18n_templates = { + 'master.html': '<title>{{ page_title|default(_("missing")) }}</title>' + '{% block body %}{% endblock %}', + 'child.html': '{% extends "master.html" %}{% block body %}' + '{% trans %}watch out{% endtrans %}{% endblock %}', + 'plural.html': '{% trans user_count %}One user online{% pluralize %}' + '{{ user_count }} users online{% endtrans %}', + 'stringformat.html': '{{ _("User: %(num)s", num=user_count) }}', + 'ngettext.html': '{{ ngettext("%(num)s apple", "%(num)s apples", apples) }}', + 'ngettext_long.html': '{% trans num=apples %}{{ num }} apple{% pluralize %}' + '{{ num }} apples{% endtrans %}', + 'transvars1.html': '{% trans %}User: {{ num }}{% endtrans %}', + 'transvars2.html': '{% trans num=count %}User: {{ num }}{% endtrans %}', + 'transvars3.html': '{% trans count=num %}User: {{ count }}{% endtrans %}', + 'novars.html': '{% trans %}%(hello)s{% endtrans %}', + 'vars.html': '{% trans %}{{ foo }}%(foo)s{% endtrans %}', + 'explicitvars.html': '{% trans foo="42" %}%(foo)s{% endtrans %}' +} + + +languages = { + 'de': { + 'missing': u'fehlend', + 'watch out': u'pass auf', + 'One user online': u'Ein Benutzer online', + '%(user_count)s users online': u'%(user_count)s Benutzer online', + 'User: %(num)s': u'Benutzer: %(num)s', + 'User: %(count)s': u'Benutzer: %(count)s', + '%(num)s apple': u'%(num)s Apfel', + '%(num)s apples': u'%(num)s Äpfel' + } +} + + +@contextfunction +def gettext(context, string): + language = context.get('LANGUAGE', 'en') + return languages.get(language, {}).get(string, string) + + +@contextfunction +def ngettext(context, s, p, n): + language = context.get('LANGUAGE', 'en') + if n != 1: + return languages.get(language, {}).get(p, p) + return languages.get(language, {}).get(s, s) + + +i18n_env = Environment( + loader=DictLoader(i18n_templates), + extensions=['jinja2.ext.i18n'] +) +i18n_env.globals.update({ + '_': gettext, + 'gettext': gettext, + 'ngettext': ngettext +}) + +newstyle_i18n_env = Environment( + loader=DictLoader(newstyle_i18n_templates), + extensions=['jinja2.ext.i18n'] +) +newstyle_i18n_env.install_gettext_callables(gettext, ngettext, newstyle=True) + +class TestExtension(Extension): + tags = set(['test']) + ext_attr = 42 + + def parse(self, parser): + return nodes.Output([self.call_method('_dump', [ + nodes.EnvironmentAttribute('sandboxed'), + self.attr('ext_attr'), + nodes.ImportedName(__name__ + '.importable_object'), + nodes.ContextReference() + ])]).set_lineno(next(parser.stream).lineno) + + def _dump(self, sandboxed, ext_attr, imported_object, context): + return '%s|%s|%s|%s' % ( + sandboxed, + ext_attr, + imported_object, + context.blocks + ) + + +class PreprocessorExtension(Extension): + + def preprocess(self, source, name, filename=None): + return source.replace('[[TEST]]', '({{ foo }})') + + +class StreamFilterExtension(Extension): + + def filter_stream(self, stream): + for token in stream: + if token.type == 'data': + for t in self.interpolate(token): + yield t + else: + yield token + + def interpolate(self, token): + pos = 0 + end = len(token.value) + lineno = token.lineno + while 1: + match = _gettext_re.search(token.value, pos) + if match is None: + break + value = token.value[pos:match.start()] + if value: + yield Token(lineno, 'data', value) + lineno += count_newlines(token.value) + yield Token(lineno, 'variable_begin', None) + yield Token(lineno, 'name', 'gettext') + yield Token(lineno, 'lparen', None) + yield Token(lineno, 'string', match.group(1)) + yield Token(lineno, 'rparen', None) + yield Token(lineno, 'variable_end', None) + pos = match.end() + if pos < end: + yield Token(lineno, 'data', token.value[pos:]) + + +class ExtensionsTestCase(JinjaTestCase): + + def test_extend_late(self): + env = Environment() + env.add_extension('jinja2.ext.autoescape') + t = env.from_string('{% autoescape true %}{{ "<test>" }}{% endautoescape %}') + assert t.render() == '<test>' + + def test_loop_controls(self): + env = Environment(extensions=['jinja2.ext.loopcontrols']) + + tmpl = env.from_string(''' + {%- for item in [1, 2, 3, 4] %} + {%- if item % 2 == 0 %}{% continue %}{% endif -%} + {{ item }} + {%- endfor %}''') + assert tmpl.render() == '13' + + tmpl = env.from_string(''' + {%- for item in [1, 2, 3, 4] %} + {%- if item > 2 %}{% break %}{% endif -%} + {{ item }} + {%- endfor %}''') + assert tmpl.render() == '12' + + def test_do(self): + env = Environment(extensions=['jinja2.ext.do']) + tmpl = env.from_string(''' + {%- set items = [] %} + {%- for char in "foo" %} + {%- do items.append(loop.index0 ~ char) %} + {%- endfor %}{{ items|join(', ') }}''') + assert tmpl.render() == '0f, 1o, 2o' + + def test_with(self): + env = Environment(extensions=['jinja2.ext.with_']) + tmpl = env.from_string('''\ + {% with a=42, b=23 -%} + {{ a }} = {{ b }} + {% endwith -%} + {{ a }} = {{ b }}\ + ''') + assert [x.strip() for x in tmpl.render(a=1, b=2).splitlines()] \ + == ['42 = 23', '1 = 2'] + + def test_extension_nodes(self): + env = Environment(extensions=[TestExtension]) + tmpl = env.from_string('{% test %}') + assert tmpl.render() == 'False|42|23|{}' + + def test_identifier(self): + assert TestExtension.identifier == __name__ + '.TestExtension' + + def test_rebinding(self): + original = Environment(extensions=[TestExtension]) + overlay = original.overlay() + for env in original, overlay: + for ext in itervalues(env.extensions): + assert ext.environment is env + + def test_preprocessor_extension(self): + env = Environment(extensions=[PreprocessorExtension]) + tmpl = env.from_string('{[[TEST]]}') + assert tmpl.render(foo=42) == '{(42)}' + + def test_streamfilter_extension(self): + env = Environment(extensions=[StreamFilterExtension]) + env.globals['gettext'] = lambda x: x.upper() + tmpl = env.from_string('Foo _(bar) Baz') + out = tmpl.render() + assert out == 'Foo BAR Baz' + + def test_extension_ordering(self): + class T1(Extension): + priority = 1 + class T2(Extension): + priority = 2 + env = Environment(extensions=[T1, T2]) + ext = list(env.iter_extensions()) + assert ext[0].__class__ is T1 + assert ext[1].__class__ is T2 + + +class InternationalizationTestCase(JinjaTestCase): + + def test_trans(self): + tmpl = i18n_env.get_template('child.html') + assert tmpl.render(LANGUAGE='de') == '<title>fehlend</title>pass auf' + + def test_trans_plural(self): + tmpl = i18n_env.get_template('plural.html') + assert tmpl.render(LANGUAGE='de', user_count=1) == 'Ein Benutzer online' + assert tmpl.render(LANGUAGE='de', user_count=2) == '2 Benutzer online' + + def test_trans_plural_with_functions(self): + tmpl = i18n_env.get_template('plural2.html') + def get_user_count(): + get_user_count.called += 1 + return 1 + get_user_count.called = 0 + assert tmpl.render(LANGUAGE='de', get_user_count=get_user_count) == '1s' + assert get_user_count.called == 1 + + def test_complex_plural(self): + tmpl = i18n_env.from_string('{% trans foo=42, count=2 %}{{ count }} item{% ' + 'pluralize count %}{{ count }} items{% endtrans %}') + assert tmpl.render() == '2 items' + self.assert_raises(TemplateAssertionError, i18n_env.from_string, + '{% trans foo %}...{% pluralize bar %}...{% endtrans %}') + + def test_trans_stringformatting(self): + tmpl = i18n_env.get_template('stringformat.html') + assert tmpl.render(LANGUAGE='de', user_count=5) == 'Benutzer: 5' + + def test_extract(self): + from jinja2.ext import babel_extract + source = BytesIO(''' + {{ gettext('Hello World') }} + {% trans %}Hello World{% endtrans %} + {% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %} + '''.encode('ascii')) # make python 3 happy + assert list(babel_extract(source, ('gettext', 'ngettext', '_'), [], {})) == [ + (2, 'gettext', u'Hello World', []), + (3, 'gettext', u'Hello World', []), + (4, 'ngettext', (u'%(users)s user', u'%(users)s users', None), []) + ] + + def test_comment_extract(self): + from jinja2.ext import babel_extract + source = BytesIO(''' + {# trans first #} + {{ gettext('Hello World') }} + {% trans %}Hello World{% endtrans %}{# trans second #} + {#: third #} + {% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %} + '''.encode('utf-8')) # make python 3 happy + assert list(babel_extract(source, ('gettext', 'ngettext', '_'), ['trans', ':'], {})) == [ + (3, 'gettext', u'Hello World', ['first']), + (4, 'gettext', u'Hello World', ['second']), + (6, 'ngettext', (u'%(users)s user', u'%(users)s users', None), ['third']) + ] + + +class NewstyleInternationalizationTestCase(JinjaTestCase): + + def test_trans(self): + tmpl = newstyle_i18n_env.get_template('child.html') + assert tmpl.render(LANGUAGE='de') == '<title>fehlend</title>pass auf' + + def test_trans_plural(self): + tmpl = newstyle_i18n_env.get_template('plural.html') + assert tmpl.render(LANGUAGE='de', user_count=1) == 'Ein Benutzer online' + assert tmpl.render(LANGUAGE='de', user_count=2) == '2 Benutzer online' + + def test_complex_plural(self): + tmpl = newstyle_i18n_env.from_string('{% trans foo=42, count=2 %}{{ count }} item{% ' + 'pluralize count %}{{ count }} items{% endtrans %}') + assert tmpl.render() == '2 items' + self.assert_raises(TemplateAssertionError, i18n_env.from_string, + '{% trans foo %}...{% pluralize bar %}...{% endtrans %}') + + def test_trans_stringformatting(self): + tmpl = newstyle_i18n_env.get_template('stringformat.html') + assert tmpl.render(LANGUAGE='de', user_count=5) == 'Benutzer: 5' + + def test_newstyle_plural(self): + tmpl = newstyle_i18n_env.get_template('ngettext.html') + assert tmpl.render(LANGUAGE='de', apples=1) == '1 Apfel' + assert tmpl.render(LANGUAGE='de', apples=5) == u'5 Äpfel' + + def test_autoescape_support(self): + env = Environment(extensions=['jinja2.ext.autoescape', + 'jinja2.ext.i18n']) + env.install_gettext_callables(lambda x: u'<strong>Wert: %(name)s</strong>', + lambda s, p, n: s, newstyle=True) + t = env.from_string('{% autoescape ae %}{{ gettext("foo", name=' + '"<test>") }}{% endautoescape %}') + assert t.render(ae=True) == '<strong>Wert: <test></strong>' + assert t.render(ae=False) == '<strong>Wert: <test></strong>' + + def test_num_used_twice(self): + tmpl = newstyle_i18n_env.get_template('ngettext_long.html') + assert tmpl.render(apples=5, LANGUAGE='de') == u'5 Äpfel' + + def test_num_called_num(self): + source = newstyle_i18n_env.compile(''' + {% trans num=3 %}{{ num }} apple{% pluralize + %}{{ num }} apples{% endtrans %} + ''', raw=True) + # quite hacky, but the only way to properly test that. The idea is + # that the generated code does not pass num twice (although that + # would work) for better performance. This only works on the + # newstyle gettext of course + assert re.search(r"l_ngettext, u?'\%\(num\)s apple', u?'\%\(num\)s " + r"apples', 3", source) is not None + + def test_trans_vars(self): + t1 = newstyle_i18n_env.get_template('transvars1.html') + t2 = newstyle_i18n_env.get_template('transvars2.html') + t3 = newstyle_i18n_env.get_template('transvars3.html') + assert t1.render(num=1, LANGUAGE='de') == 'Benutzer: 1' + assert t2.render(count=23, LANGUAGE='de') == 'Benutzer: 23' + assert t3.render(num=42, LANGUAGE='de') == 'Benutzer: 42' + + def test_novars_vars_escaping(self): + t = newstyle_i18n_env.get_template('novars.html') + assert t.render() == '%(hello)s' + t = newstyle_i18n_env.get_template('vars.html') + assert t.render(foo='42') == '42%(foo)s' + t = newstyle_i18n_env.get_template('explicitvars.html') + assert t.render() == '%(foo)s' + + +class AutoEscapeTestCase(JinjaTestCase): + + def test_scoped_setting(self): + env = Environment(extensions=['jinja2.ext.autoescape'], + autoescape=True) + tmpl = env.from_string(''' + {{ "<HelloWorld>" }} + {% autoescape false %} + {{ "<HelloWorld>" }} + {% endautoescape %} + {{ "<HelloWorld>" }} + ''') + assert tmpl.render().split() == \ + [u'<HelloWorld>', u'<HelloWorld>', u'<HelloWorld>'] + + env = Environment(extensions=['jinja2.ext.autoescape'], + autoescape=False) + tmpl = env.from_string(''' + {{ "<HelloWorld>" }} + {% autoescape true %} + {{ "<HelloWorld>" }} + {% endautoescape %} + {{ "<HelloWorld>" }} + ''') + assert tmpl.render().split() == \ + [u'<HelloWorld>', u'<HelloWorld>', u'<HelloWorld>'] + + def test_nonvolatile(self): + env = Environment(extensions=['jinja2.ext.autoescape'], + autoescape=True) + tmpl = env.from_string('{{ {"foo": "<test>"}|xmlattr|escape }}') + assert tmpl.render() == ' foo="<test>"' + tmpl = env.from_string('{% autoescape false %}{{ {"foo": "<test>"}' + '|xmlattr|escape }}{% endautoescape %}') + assert tmpl.render() == ' foo="&lt;test&gt;"' + + def test_volatile(self): + env = Environment(extensions=['jinja2.ext.autoescape'], + autoescape=True) + tmpl = env.from_string('{% autoescape foo %}{{ {"foo": "<test>"}' + '|xmlattr|escape }}{% endautoescape %}') + assert tmpl.render(foo=False) == ' foo="&lt;test&gt;"' + assert tmpl.render(foo=True) == ' foo="<test>"' + + def test_scoping(self): + env = Environment(extensions=['jinja2.ext.autoescape']) + tmpl = env.from_string('{% autoescape true %}{% set x = "<x>" %}{{ x }}' + '{% endautoescape %}{{ x }}{{ "<y>" }}') + assert tmpl.render(x=1) == '<x>1<y>' + + def test_volatile_scoping(self): + env = Environment(extensions=['jinja2.ext.autoescape']) + tmplsource = ''' + {% autoescape val %} + {% macro foo(x) %} + [{{ x }}] + {% endmacro %} + {{ foo().__class__.__name__ }} + {% endautoescape %} + {{ '<testing>' }} + ''' + tmpl = env.from_string(tmplsource) + assert tmpl.render(val=True).split()[0] == 'Markup' + assert tmpl.render(val=False).split()[0] == text_type.__name__ + + # looking at the source we should see <testing> there in raw + # (and then escaped as well) + env = Environment(extensions=['jinja2.ext.autoescape']) + pysource = env.compile(tmplsource, raw=True) + assert '<testing>\\n' in pysource + + env = Environment(extensions=['jinja2.ext.autoescape'], + autoescape=True) + pysource = env.compile(tmplsource, raw=True) + assert '<testing>\\n' in pysource + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(ExtensionsTestCase)) + suite.addTest(unittest.makeSuite(InternationalizationTestCase)) + suite.addTest(unittest.makeSuite(NewstyleInternationalizationTestCase)) + suite.addTest(unittest.makeSuite(AutoEscapeTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/filters.py b/module/lib/jinja2/testsuite/filters.py new file mode 100644 index 000000000..282dd2d85 --- /dev/null +++ b/module/lib/jinja2/testsuite/filters.py @@ -0,0 +1,515 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.filters + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Tests for the jinja filters. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest +from jinja2.testsuite import JinjaTestCase + +from jinja2 import Markup, Environment +from jinja2._compat import text_type, implements_to_string + +env = Environment() + + +class FilterTestCase(JinjaTestCase): + + def test_filter_calling(self): + rv = env.call_filter('sum', [1, 2, 3]) + self.assert_equal(rv, 6) + + def test_capitalize(self): + tmpl = env.from_string('{{ "foo bar"|capitalize }}') + assert tmpl.render() == 'Foo bar' + + def test_center(self): + tmpl = env.from_string('{{ "foo"|center(9) }}') + assert tmpl.render() == ' foo ' + + def test_default(self): + tmpl = env.from_string( + "{{ missing|default('no') }}|{{ false|default('no') }}|" + "{{ false|default('no', true) }}|{{ given|default('no') }}" + ) + assert tmpl.render(given='yes') == 'no|False|no|yes' + + def test_dictsort(self): + tmpl = env.from_string( + '{{ foo|dictsort }}|' + '{{ foo|dictsort(true) }}|' + '{{ foo|dictsort(false, "value") }}' + ) + out = tmpl.render(foo={"aa": 0, "b": 1, "c": 2, "AB": 3}) + assert out == ("[('aa', 0), ('AB', 3), ('b', 1), ('c', 2)]|" + "[('AB', 3), ('aa', 0), ('b', 1), ('c', 2)]|" + "[('aa', 0), ('b', 1), ('c', 2), ('AB', 3)]") + + def test_batch(self): + tmpl = env.from_string("{{ foo|batch(3)|list }}|" + "{{ foo|batch(3, 'X')|list }}") + out = tmpl.render(foo=list(range(10))) + assert out == ("[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]|" + "[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 'X', 'X']]") + + def test_slice(self): + tmpl = env.from_string('{{ foo|slice(3)|list }}|' + '{{ foo|slice(3, "X")|list }}') + out = tmpl.render(foo=list(range(10))) + assert out == ("[[0, 1, 2, 3], [4, 5, 6], [7, 8, 9]]|" + "[[0, 1, 2, 3], [4, 5, 6, 'X'], [7, 8, 9, 'X']]") + + def test_escape(self): + tmpl = env.from_string('''{{ '<">&'|escape }}''') + out = tmpl.render() + assert out == '<">&' + + def test_striptags(self): + tmpl = env.from_string('''{{ foo|striptags }}''') + out = tmpl.render(foo=' <p>just a small \n <a href="#">' + 'example</a> link</p>\n<p>to a webpage</p> ' + '<!-- <p>and some commented stuff</p> -->') + assert out == 'just a small example link to a webpage' + + def test_filesizeformat(self): + tmpl = env.from_string( + '{{ 100|filesizeformat }}|' + '{{ 1000|filesizeformat }}|' + '{{ 1000000|filesizeformat }}|' + '{{ 1000000000|filesizeformat }}|' + '{{ 1000000000000|filesizeformat }}|' + '{{ 100|filesizeformat(true) }}|' + '{{ 1000|filesizeformat(true) }}|' + '{{ 1000000|filesizeformat(true) }}|' + '{{ 1000000000|filesizeformat(true) }}|' + '{{ 1000000000000|filesizeformat(true) }}' + ) + out = tmpl.render() + self.assert_equal(out, ( + '100 Bytes|1.0 kB|1.0 MB|1.0 GB|1.0 TB|100 Bytes|' + '1000 Bytes|976.6 KiB|953.7 MiB|931.3 GiB' + )) + + def test_filesizeformat_issue59(self): + tmpl = env.from_string( + '{{ 300|filesizeformat }}|' + '{{ 3000|filesizeformat }}|' + '{{ 3000000|filesizeformat }}|' + '{{ 3000000000|filesizeformat }}|' + '{{ 3000000000000|filesizeformat }}|' + '{{ 300|filesizeformat(true) }}|' + '{{ 3000|filesizeformat(true) }}|' + '{{ 3000000|filesizeformat(true) }}' + ) + out = tmpl.render() + self.assert_equal(out, ( + '300 Bytes|3.0 kB|3.0 MB|3.0 GB|3.0 TB|300 Bytes|' + '2.9 KiB|2.9 MiB' + )) + + + def test_first(self): + tmpl = env.from_string('{{ foo|first }}') + out = tmpl.render(foo=list(range(10))) + assert out == '0' + + def test_float(self): + tmpl = env.from_string('{{ "42"|float }}|' + '{{ "ajsghasjgd"|float }}|' + '{{ "32.32"|float }}') + out = tmpl.render() + assert out == '42.0|0.0|32.32' + + def test_format(self): + tmpl = env.from_string('''{{ "%s|%s"|format("a", "b") }}''') + out = tmpl.render() + assert out == 'a|b' + + def test_indent(self): + tmpl = env.from_string('{{ foo|indent(2) }}|{{ foo|indent(2, true) }}') + text = '\n'.join([' '.join(['foo', 'bar'] * 2)] * 2) + out = tmpl.render(foo=text) + assert out == ('foo bar foo bar\n foo bar foo bar| ' + 'foo bar foo bar\n foo bar foo bar') + + def test_int(self): + tmpl = env.from_string('{{ "42"|int }}|{{ "ajsghasjgd"|int }}|' + '{{ "32.32"|int }}') + out = tmpl.render() + assert out == '42|0|32' + + def test_join(self): + tmpl = env.from_string('{{ [1, 2, 3]|join("|") }}') + out = tmpl.render() + assert out == '1|2|3' + + env2 = Environment(autoescape=True) + tmpl = env2.from_string('{{ ["<foo>", "<span>foo</span>"|safe]|join }}') + assert tmpl.render() == '<foo><span>foo</span>' + + def test_join_attribute(self): + class User(object): + def __init__(self, username): + self.username = username + tmpl = env.from_string('''{{ users|join(', ', 'username') }}''') + assert tmpl.render(users=map(User, ['foo', 'bar'])) == 'foo, bar' + + def test_last(self): + tmpl = env.from_string('''{{ foo|last }}''') + out = tmpl.render(foo=list(range(10))) + assert out == '9' + + def test_length(self): + tmpl = env.from_string('''{{ "hello world"|length }}''') + out = tmpl.render() + assert out == '11' + + def test_lower(self): + tmpl = env.from_string('''{{ "FOO"|lower }}''') + out = tmpl.render() + assert out == 'foo' + + def test_pprint(self): + from pprint import pformat + tmpl = env.from_string('''{{ data|pprint }}''') + data = list(range(1000)) + assert tmpl.render(data=data) == pformat(data) + + def test_random(self): + tmpl = env.from_string('''{{ seq|random }}''') + seq = list(range(100)) + for _ in range(10): + assert int(tmpl.render(seq=seq)) in seq + + def test_reverse(self): + tmpl = env.from_string('{{ "foobar"|reverse|join }}|' + '{{ [1, 2, 3]|reverse|list }}') + assert tmpl.render() == 'raboof|[3, 2, 1]' + + def test_string(self): + x = [1, 2, 3, 4, 5] + tmpl = env.from_string('''{{ obj|string }}''') + assert tmpl.render(obj=x) == text_type(x) + + def test_title(self): + tmpl = env.from_string('''{{ "foo bar"|title }}''') + assert tmpl.render() == "Foo Bar" + tmpl = env.from_string('''{{ "foo's bar"|title }}''') + assert tmpl.render() == "Foo's Bar" + tmpl = env.from_string('''{{ "foo bar"|title }}''') + assert tmpl.render() == "Foo Bar" + tmpl = env.from_string('''{{ "f bar f"|title }}''') + assert tmpl.render() == "F Bar F" + tmpl = env.from_string('''{{ "foo-bar"|title }}''') + assert tmpl.render() == "Foo-Bar" + tmpl = env.from_string('''{{ "foo\tbar"|title }}''') + assert tmpl.render() == "Foo\tBar" + tmpl = env.from_string('''{{ "FOO\tBAR"|title }}''') + assert tmpl.render() == "Foo\tBar" + + def test_truncate(self): + tmpl = env.from_string( + '{{ data|truncate(15, true, ">>>") }}|' + '{{ data|truncate(15, false, ">>>") }}|' + '{{ smalldata|truncate(15) }}' + ) + out = tmpl.render(data='foobar baz bar' * 1000, + smalldata='foobar baz bar') + assert out == 'foobar baz barf>>>|foobar baz >>>|foobar baz bar' + + def test_upper(self): + tmpl = env.from_string('{{ "foo"|upper }}') + assert tmpl.render() == 'FOO' + + def test_urlize(self): + tmpl = env.from_string('{{ "foo http://www.example.com/ bar"|urlize }}') + assert tmpl.render() == 'foo <a href="http://www.example.com/">'\ + 'http://www.example.com/</a> bar' + + def test_wordcount(self): + tmpl = env.from_string('{{ "foo bar baz"|wordcount }}') + assert tmpl.render() == '3' + + def test_block(self): + tmpl = env.from_string('{% filter lower|escape %}<HEHE>{% endfilter %}') + assert tmpl.render() == '<hehe>' + + def test_chaining(self): + tmpl = env.from_string('''{{ ['<foo>', '<bar>']|first|upper|escape }}''') + assert tmpl.render() == '<FOO>' + + def test_sum(self): + tmpl = env.from_string('''{{ [1, 2, 3, 4, 5, 6]|sum }}''') + assert tmpl.render() == '21' + + def test_sum_attributes(self): + tmpl = env.from_string('''{{ values|sum('value') }}''') + assert tmpl.render(values=[ + {'value': 23}, + {'value': 1}, + {'value': 18}, + ]) == '42' + + def test_sum_attributes_nested(self): + tmpl = env.from_string('''{{ values|sum('real.value') }}''') + assert tmpl.render(values=[ + {'real': {'value': 23}}, + {'real': {'value': 1}}, + {'real': {'value': 18}}, + ]) == '42' + + def test_sum_attributes_tuple(self): + tmpl = env.from_string('''{{ values.items()|sum('1') }}''') + assert tmpl.render(values={ + 'foo': 23, + 'bar': 1, + 'baz': 18, + }) == '42' + + def test_abs(self): + tmpl = env.from_string('''{{ -1|abs }}|{{ 1|abs }}''') + assert tmpl.render() == '1|1', tmpl.render() + + def test_round_positive(self): + tmpl = env.from_string('{{ 2.7|round }}|{{ 2.1|round }}|' + "{{ 2.1234|round(3, 'floor') }}|" + "{{ 2.1|round(0, 'ceil') }}") + assert tmpl.render() == '3.0|2.0|2.123|3.0', tmpl.render() + + def test_round_negative(self): + tmpl = env.from_string('{{ 21.3|round(-1)}}|' + "{{ 21.3|round(-1, 'ceil')}}|" + "{{ 21.3|round(-1, 'floor')}}") + assert tmpl.render() == '20.0|30.0|20.0',tmpl.render() + + def test_xmlattr(self): + tmpl = env.from_string("{{ {'foo': 42, 'bar': 23, 'fish': none, " + "'spam': missing, 'blub:blub': '<?>'}|xmlattr }}") + out = tmpl.render().split() + assert len(out) == 3 + assert 'foo="42"' in out + assert 'bar="23"' in out + assert 'blub:blub="<?>"' in out + + def test_sort1(self): + tmpl = env.from_string('{{ [2, 3, 1]|sort }}|{{ [2, 3, 1]|sort(true) }}') + assert tmpl.render() == '[1, 2, 3]|[3, 2, 1]' + + def test_sort2(self): + tmpl = env.from_string('{{ "".join(["c", "A", "b", "D"]|sort) }}') + assert tmpl.render() == 'AbcD' + + def test_sort3(self): + tmpl = env.from_string('''{{ ['foo', 'Bar', 'blah']|sort }}''') + assert tmpl.render() == "['Bar', 'blah', 'foo']" + + def test_sort4(self): + @implements_to_string + class Magic(object): + def __init__(self, value): + self.value = value + def __str__(self): + return text_type(self.value) + tmpl = env.from_string('''{{ items|sort(attribute='value')|join }}''') + assert tmpl.render(items=map(Magic, [3, 2, 4, 1])) == '1234' + + def test_groupby(self): + tmpl = env.from_string(''' + {%- for grouper, list in [{'foo': 1, 'bar': 2}, + {'foo': 2, 'bar': 3}, + {'foo': 1, 'bar': 1}, + {'foo': 3, 'bar': 4}]|groupby('foo') -%} + {{ grouper }}{% for x in list %}: {{ x.foo }}, {{ x.bar }}{% endfor %}| + {%- endfor %}''') + assert tmpl.render().split('|') == [ + "1: 1, 2: 1, 1", + "2: 2, 3", + "3: 3, 4", + "" + ] + + def test_groupby_tuple_index(self): + tmpl = env.from_string(''' + {%- for grouper, list in [('a', 1), ('a', 2), ('b', 1)]|groupby(0) -%} + {{ grouper }}{% for x in list %}:{{ x.1 }}{% endfor %}| + {%- endfor %}''') + assert tmpl.render() == 'a:1:2|b:1|' + + def test_groupby_multidot(self): + class Date(object): + def __init__(self, day, month, year): + self.day = day + self.month = month + self.year = year + class Article(object): + def __init__(self, title, *date): + self.date = Date(*date) + self.title = title + articles = [ + Article('aha', 1, 1, 1970), + Article('interesting', 2, 1, 1970), + Article('really?', 3, 1, 1970), + Article('totally not', 1, 1, 1971) + ] + tmpl = env.from_string(''' + {%- for year, list in articles|groupby('date.year') -%} + {{ year }}{% for x in list %}[{{ x.title }}]{% endfor %}| + {%- endfor %}''') + assert tmpl.render(articles=articles).split('|') == [ + '1970[aha][interesting][really?]', + '1971[totally not]', + '' + ] + + def test_filtertag(self): + tmpl = env.from_string("{% filter upper|replace('FOO', 'foo') %}" + "foobar{% endfilter %}") + assert tmpl.render() == 'fooBAR' + + def test_replace(self): + env = Environment() + tmpl = env.from_string('{{ string|replace("o", 42) }}') + assert tmpl.render(string='<foo>') == '<f4242>' + env = Environment(autoescape=True) + tmpl = env.from_string('{{ string|replace("o", 42) }}') + assert tmpl.render(string='<foo>') == '<f4242>' + tmpl = env.from_string('{{ string|replace("<", 42) }}') + assert tmpl.render(string='<foo>') == '42foo>' + tmpl = env.from_string('{{ string|replace("o", ">x<") }}') + assert tmpl.render(string=Markup('foo')) == 'f>x<>x<' + + def test_forceescape(self): + tmpl = env.from_string('{{ x|forceescape }}') + assert tmpl.render(x=Markup('<div />')) == u'<div />' + + def test_safe(self): + env = Environment(autoescape=True) + tmpl = env.from_string('{{ "<div>foo</div>"|safe }}') + assert tmpl.render() == '<div>foo</div>' + tmpl = env.from_string('{{ "<div>foo</div>" }}') + assert tmpl.render() == '<div>foo</div>' + + def test_urlencode(self): + env = Environment(autoescape=True) + tmpl = env.from_string('{{ "Hello, world!"|urlencode }}') + assert tmpl.render() == 'Hello%2C%20world%21' + tmpl = env.from_string('{{ o|urlencode }}') + assert tmpl.render(o=u"Hello, world\u203d") == "Hello%2C%20world%E2%80%BD" + assert tmpl.render(o=(("f", 1),)) == "f=1" + assert tmpl.render(o=(('f', 1), ("z", 2))) == "f=1&z=2" + assert tmpl.render(o=((u"\u203d", 1),)) == "%E2%80%BD=1" + assert tmpl.render(o={u"\u203d": 1}) == "%E2%80%BD=1" + assert tmpl.render(o={0: 1}) == "0=1" + + def test_simple_map(self): + env = Environment() + tmpl = env.from_string('{{ ["1", "2", "3"]|map("int")|sum }}') + self.assertEqual(tmpl.render(), '6') + + def test_attribute_map(self): + class User(object): + def __init__(self, name): + self.name = name + env = Environment() + users = [ + User('john'), + User('jane'), + User('mike'), + ] + tmpl = env.from_string('{{ users|map(attribute="name")|join("|") }}') + self.assertEqual(tmpl.render(users=users), 'john|jane|mike') + + def test_empty_map(self): + env = Environment() + tmpl = env.from_string('{{ none|map("upper")|list }}') + self.assertEqual(tmpl.render(), '[]') + + def test_simple_select(self): + env = Environment() + tmpl = env.from_string('{{ [1, 2, 3, 4, 5]|select("odd")|join("|") }}') + self.assertEqual(tmpl.render(), '1|3|5') + + def test_bool_select(self): + env = Environment() + tmpl = env.from_string('{{ [none, false, 0, 1, 2, 3, 4, 5]|select|join("|") }}') + self.assertEqual(tmpl.render(), '1|2|3|4|5') + + def test_simple_reject(self): + env = Environment() + tmpl = env.from_string('{{ [1, 2, 3, 4, 5]|reject("odd")|join("|") }}') + self.assertEqual(tmpl.render(), '2|4') + + def test_bool_reject(self): + env = Environment() + tmpl = env.from_string('{{ [none, false, 0, 1, 2, 3, 4, 5]|reject|join("|") }}') + self.assertEqual(tmpl.render(), 'None|False|0') + + def test_simple_select_attr(self): + class User(object): + def __init__(self, name, is_active): + self.name = name + self.is_active = is_active + env = Environment() + users = [ + User('john', True), + User('jane', True), + User('mike', False), + ] + tmpl = env.from_string('{{ users|selectattr("is_active")|' + 'map(attribute="name")|join("|") }}') + self.assertEqual(tmpl.render(users=users), 'john|jane') + + def test_simple_reject_attr(self): + class User(object): + def __init__(self, name, is_active): + self.name = name + self.is_active = is_active + env = Environment() + users = [ + User('john', True), + User('jane', True), + User('mike', False), + ] + tmpl = env.from_string('{{ users|rejectattr("is_active")|' + 'map(attribute="name")|join("|") }}') + self.assertEqual(tmpl.render(users=users), 'mike') + + def test_func_select_attr(self): + class User(object): + def __init__(self, id, name): + self.id = id + self.name = name + env = Environment() + users = [ + User(1, 'john'), + User(2, 'jane'), + User(3, 'mike'), + ] + tmpl = env.from_string('{{ users|selectattr("id", "odd")|' + 'map(attribute="name")|join("|") }}') + self.assertEqual(tmpl.render(users=users), 'john|mike') + + def test_func_reject_attr(self): + class User(object): + def __init__(self, id, name): + self.id = id + self.name = name + env = Environment() + users = [ + User(1, 'john'), + User(2, 'jane'), + User(3, 'mike'), + ] + tmpl = env.from_string('{{ users|rejectattr("id", "odd")|' + 'map(attribute="name")|join("|") }}') + self.assertEqual(tmpl.render(users=users), 'jane') + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(FilterTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/imports.py b/module/lib/jinja2/testsuite/imports.py new file mode 100644 index 000000000..3db9008de --- /dev/null +++ b/module/lib/jinja2/testsuite/imports.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.imports + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Tests the import features (with includes). + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from jinja2.testsuite import JinjaTestCase + +from jinja2 import Environment, DictLoader +from jinja2.exceptions import TemplateNotFound, TemplatesNotFound + + +test_env = Environment(loader=DictLoader(dict( + module='{% macro test() %}[{{ foo }}|{{ bar }}]{% endmacro %}', + header='[{{ foo }}|{{ 23 }}]', + o_printer='({{ o }})' +))) +test_env.globals['bar'] = 23 + + +class ImportsTestCase(JinjaTestCase): + + def test_context_imports(self): + t = test_env.from_string('{% import "module" as m %}{{ m.test() }}') + assert t.render(foo=42) == '[|23]' + t = test_env.from_string('{% import "module" as m without context %}{{ m.test() }}') + assert t.render(foo=42) == '[|23]' + t = test_env.from_string('{% import "module" as m with context %}{{ m.test() }}') + assert t.render(foo=42) == '[42|23]' + t = test_env.from_string('{% from "module" import test %}{{ test() }}') + assert t.render(foo=42) == '[|23]' + t = test_env.from_string('{% from "module" import test without context %}{{ test() }}') + assert t.render(foo=42) == '[|23]' + t = test_env.from_string('{% from "module" import test with context %}{{ test() }}') + assert t.render(foo=42) == '[42|23]' + + def test_trailing_comma(self): + test_env.from_string('{% from "foo" import bar, baz with context %}') + test_env.from_string('{% from "foo" import bar, baz, with context %}') + test_env.from_string('{% from "foo" import bar, with context %}') + test_env.from_string('{% from "foo" import bar, with, context %}') + test_env.from_string('{% from "foo" import bar, with with context %}') + + def test_exports(self): + m = test_env.from_string(''' + {% macro toplevel() %}...{% endmacro %} + {% macro __private() %}...{% endmacro %} + {% set variable = 42 %} + {% for item in [1] %} + {% macro notthere() %}{% endmacro %} + {% endfor %} + ''').module + assert m.toplevel() == '...' + assert not hasattr(m, '__missing') + assert m.variable == 42 + assert not hasattr(m, 'notthere') + + +class IncludesTestCase(JinjaTestCase): + + def test_context_include(self): + t = test_env.from_string('{% include "header" %}') + assert t.render(foo=42) == '[42|23]' + t = test_env.from_string('{% include "header" with context %}') + assert t.render(foo=42) == '[42|23]' + t = test_env.from_string('{% include "header" without context %}') + assert t.render(foo=42) == '[|23]' + + def test_choice_includes(self): + t = test_env.from_string('{% include ["missing", "header"] %}') + assert t.render(foo=42) == '[42|23]' + + t = test_env.from_string('{% include ["missing", "missing2"] ignore missing %}') + assert t.render(foo=42) == '' + + t = test_env.from_string('{% include ["missing", "missing2"] %}') + self.assert_raises(TemplateNotFound, t.render) + try: + t.render() + except TemplatesNotFound as e: + assert e.templates == ['missing', 'missing2'] + assert e.name == 'missing2' + else: + assert False, 'thou shalt raise' + + def test_includes(t, **ctx): + ctx['foo'] = 42 + assert t.render(ctx) == '[42|23]' + + t = test_env.from_string('{% include ["missing", "header"] %}') + test_includes(t) + t = test_env.from_string('{% include x %}') + test_includes(t, x=['missing', 'header']) + t = test_env.from_string('{% include [x, "header"] %}') + test_includes(t, x='missing') + t = test_env.from_string('{% include x %}') + test_includes(t, x='header') + t = test_env.from_string('{% include x %}') + test_includes(t, x='header') + t = test_env.from_string('{% include [x] %}') + test_includes(t, x='header') + + def test_include_ignoring_missing(self): + t = test_env.from_string('{% include "missing" %}') + self.assert_raises(TemplateNotFound, t.render) + for extra in '', 'with context', 'without context': + t = test_env.from_string('{% include "missing" ignore missing ' + + extra + ' %}') + assert t.render() == '' + + def test_context_include_with_overrides(self): + env = Environment(loader=DictLoader(dict( + main="{% for item in [1, 2, 3] %}{% include 'item' %}{% endfor %}", + item="{{ item }}" + ))) + assert env.get_template("main").render() == "123" + + def test_unoptimized_scopes(self): + t = test_env.from_string(""" + {% macro outer(o) %} + {% macro inner() %} + {% include "o_printer" %} + {% endmacro %} + {{ inner() }} + {% endmacro %} + {{ outer("FOO") }} + """) + assert t.render().strip() == '(FOO)' + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(ImportsTestCase)) + suite.addTest(unittest.makeSuite(IncludesTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/inheritance.py b/module/lib/jinja2/testsuite/inheritance.py new file mode 100644 index 000000000..e0f51cda9 --- /dev/null +++ b/module/lib/jinja2/testsuite/inheritance.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.inheritance + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Tests the template inheritance feature. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from jinja2.testsuite import JinjaTestCase + +from jinja2 import Environment, DictLoader, TemplateError + + +LAYOUTTEMPLATE = '''\ +|{% block block1 %}block 1 from layout{% endblock %} +|{% block block2 %}block 2 from layout{% endblock %} +|{% block block3 %} +{% block block4 %}nested block 4 from layout{% endblock %} +{% endblock %}|''' + +LEVEL1TEMPLATE = '''\ +{% extends "layout" %} +{% block block1 %}block 1 from level1{% endblock %}''' + +LEVEL2TEMPLATE = '''\ +{% extends "level1" %} +{% block block2 %}{% block block5 %}nested block 5 from level2{% +endblock %}{% endblock %}''' + +LEVEL3TEMPLATE = '''\ +{% extends "level2" %} +{% block block5 %}block 5 from level3{% endblock %} +{% block block4 %}block 4 from level3{% endblock %} +''' + +LEVEL4TEMPLATE = '''\ +{% extends "level3" %} +{% block block3 %}block 3 from level4{% endblock %} +''' + +WORKINGTEMPLATE = '''\ +{% extends "layout" %} +{% block block1 %} + {% if false %} + {% block block2 %} + this should workd + {% endblock %} + {% endif %} +{% endblock %} +''' + +DOUBLEEXTENDS = '''\ +{% extends "layout" %} +{% extends "layout" %} +{% block block1 %} + {% if false %} + {% block block2 %} + this should workd + {% endblock %} + {% endif %} +{% endblock %} +''' + + +env = Environment(loader=DictLoader({ + 'layout': LAYOUTTEMPLATE, + 'level1': LEVEL1TEMPLATE, + 'level2': LEVEL2TEMPLATE, + 'level3': LEVEL3TEMPLATE, + 'level4': LEVEL4TEMPLATE, + 'working': WORKINGTEMPLATE, + 'doublee': DOUBLEEXTENDS, +}), trim_blocks=True) + + +class InheritanceTestCase(JinjaTestCase): + + def test_layout(self): + tmpl = env.get_template('layout') + assert tmpl.render() == ('|block 1 from layout|block 2 from ' + 'layout|nested block 4 from layout|') + + def test_level1(self): + tmpl = env.get_template('level1') + assert tmpl.render() == ('|block 1 from level1|block 2 from ' + 'layout|nested block 4 from layout|') + + def test_level2(self): + tmpl = env.get_template('level2') + assert tmpl.render() == ('|block 1 from level1|nested block 5 from ' + 'level2|nested block 4 from layout|') + + def test_level3(self): + tmpl = env.get_template('level3') + assert tmpl.render() == ('|block 1 from level1|block 5 from level3|' + 'block 4 from level3|') + + def test_level4(sel): + tmpl = env.get_template('level4') + assert tmpl.render() == ('|block 1 from level1|block 5 from ' + 'level3|block 3 from level4|') + + def test_super(self): + env = Environment(loader=DictLoader({ + 'a': '{% block intro %}INTRO{% endblock %}|' + 'BEFORE|{% block data %}INNER{% endblock %}|AFTER', + 'b': '{% extends "a" %}{% block data %}({{ ' + 'super() }}){% endblock %}', + 'c': '{% extends "b" %}{% block intro %}--{{ ' + 'super() }}--{% endblock %}\n{% block data ' + '%}[{{ super() }}]{% endblock %}' + })) + tmpl = env.get_template('c') + assert tmpl.render() == '--INTRO--|BEFORE|[(INNER)]|AFTER' + + def test_working(self): + tmpl = env.get_template('working') + + def test_reuse_blocks(self): + tmpl = env.from_string('{{ self.foo() }}|{% block foo %}42' + '{% endblock %}|{{ self.foo() }}') + assert tmpl.render() == '42|42|42' + + def test_preserve_blocks(self): + env = Environment(loader=DictLoader({ + 'a': '{% if false %}{% block x %}A{% endblock %}{% endif %}{{ self.x() }}', + 'b': '{% extends "a" %}{% block x %}B{{ super() }}{% endblock %}' + })) + tmpl = env.get_template('b') + assert tmpl.render() == 'BA' + + def test_dynamic_inheritance(self): + env = Environment(loader=DictLoader({ + 'master1': 'MASTER1{% block x %}{% endblock %}', + 'master2': 'MASTER2{% block x %}{% endblock %}', + 'child': '{% extends master %}{% block x %}CHILD{% endblock %}' + })) + tmpl = env.get_template('child') + for m in range(1, 3): + assert tmpl.render(master='master%d' % m) == 'MASTER%dCHILD' % m + + def test_multi_inheritance(self): + env = Environment(loader=DictLoader({ + 'master1': 'MASTER1{% block x %}{% endblock %}', + 'master2': 'MASTER2{% block x %}{% endblock %}', + 'child': '''{% if master %}{% extends master %}{% else %}{% extends + 'master1' %}{% endif %}{% block x %}CHILD{% endblock %}''' + })) + tmpl = env.get_template('child') + assert tmpl.render(master='master2') == 'MASTER2CHILD' + assert tmpl.render(master='master1') == 'MASTER1CHILD' + assert tmpl.render() == 'MASTER1CHILD' + + def test_scoped_block(self): + env = Environment(loader=DictLoader({ + 'master.html': '{% for item in seq %}[{% block item scoped %}' + '{% endblock %}]{% endfor %}' + })) + t = env.from_string('{% extends "master.html" %}{% block item %}' + '{{ item }}{% endblock %}') + assert t.render(seq=list(range(5))) == '[0][1][2][3][4]' + + def test_super_in_scoped_block(self): + env = Environment(loader=DictLoader({ + 'master.html': '{% for item in seq %}[{% block item scoped %}' + '{{ item }}{% endblock %}]{% endfor %}' + })) + t = env.from_string('{% extends "master.html" %}{% block item %}' + '{{ super() }}|{{ item * 2 }}{% endblock %}') + assert t.render(seq=list(range(5))) == '[0|0][1|2][2|4][3|6][4|8]' + + def test_scoped_block_after_inheritance(self): + env = Environment(loader=DictLoader({ + 'layout.html': ''' + {% block useless %}{% endblock %} + ''', + 'index.html': ''' + {%- extends 'layout.html' %} + {% from 'helpers.html' import foo with context %} + {% block useless %} + {% for x in [1, 2, 3] %} + {% block testing scoped %} + {{ foo(x) }} + {% endblock %} + {% endfor %} + {% endblock %} + ''', + 'helpers.html': ''' + {% macro foo(x) %}{{ the_foo + x }}{% endmacro %} + ''' + })) + rv = env.get_template('index.html').render(the_foo=42).split() + assert rv == ['43', '44', '45'] + + +class BugFixTestCase(JinjaTestCase): + + def test_fixed_macro_scoping_bug(self): + assert Environment(loader=DictLoader({ + 'test.html': '''\ + {% extends 'details.html' %} + + {% macro my_macro() %} + my_macro + {% endmacro %} + + {% block inner_box %} + {{ my_macro() }} + {% endblock %} + ''', + 'details.html': '''\ + {% extends 'standard.html' %} + + {% macro my_macro() %} + my_macro + {% endmacro %} + + {% block content %} + {% block outer_box %} + outer_box + {% block inner_box %} + inner_box + {% endblock %} + {% endblock %} + {% endblock %} + ''', + 'standard.html': ''' + {% block content %} {% endblock %} + ''' + })).get_template("test.html").render().split() == [u'outer_box', u'my_macro'] + + def test_double_extends(self): + """Ensures that a template with more than 1 {% extends ... %} usage + raises a ``TemplateError``. + """ + try: + tmpl = env.get_template('doublee') + except Exception as e: + assert isinstance(e, TemplateError) + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(InheritanceTestCase)) + suite.addTest(unittest.makeSuite(BugFixTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/lexnparse.py b/module/lib/jinja2/testsuite/lexnparse.py new file mode 100644 index 000000000..bd1c94cd3 --- /dev/null +++ b/module/lib/jinja2/testsuite/lexnparse.py @@ -0,0 +1,593 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.lexnparse + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + All the unittests regarding lexing, parsing and syntax. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from jinja2.testsuite import JinjaTestCase + +from jinja2 import Environment, Template, TemplateSyntaxError, \ + UndefinedError, nodes +from jinja2._compat import next, iteritems, text_type, PY2 +from jinja2.lexer import Token, TokenStream, TOKEN_EOF, \ + TOKEN_BLOCK_BEGIN, TOKEN_BLOCK_END + +env = Environment() + + +# how does a string look like in jinja syntax? +if PY2: + def jinja_string_repr(string): + return repr(string)[1:] +else: + jinja_string_repr = repr + + +class TokenStreamTestCase(JinjaTestCase): + test_tokens = [Token(1, TOKEN_BLOCK_BEGIN, ''), + Token(2, TOKEN_BLOCK_END, ''), + ] + + def test_simple(self): + ts = TokenStream(self.test_tokens, "foo", "bar") + assert ts.current.type is TOKEN_BLOCK_BEGIN + assert bool(ts) + assert not bool(ts.eos) + next(ts) + assert ts.current.type is TOKEN_BLOCK_END + assert bool(ts) + assert not bool(ts.eos) + next(ts) + assert ts.current.type is TOKEN_EOF + assert not bool(ts) + assert bool(ts.eos) + + def test_iter(self): + token_types = [t.type for t in TokenStream(self.test_tokens, "foo", "bar")] + assert token_types == ['block_begin', 'block_end', ] + + +class LexerTestCase(JinjaTestCase): + + def test_raw1(self): + tmpl = env.from_string('{% raw %}foo{% endraw %}|' + '{%raw%}{{ bar }}|{% baz %}{% endraw %}') + assert tmpl.render() == 'foo|{{ bar }}|{% baz %}' + + def test_raw2(self): + tmpl = env.from_string('1 {%- raw -%} 2 {%- endraw -%} 3') + assert tmpl.render() == '123' + + def test_balancing(self): + env = Environment('{%', '%}', '${', '}') + tmpl = env.from_string('''{% for item in seq + %}${{'foo': item}|upper}{% endfor %}''') + assert tmpl.render(seq=list(range(3))) == "{'FOO': 0}{'FOO': 1}{'FOO': 2}" + + def test_comments(self): + env = Environment('<!--', '-->', '{', '}') + tmpl = env.from_string('''\ +<ul> +<!--- for item in seq --> + <li>{item}</li> +<!--- endfor --> +</ul>''') + assert tmpl.render(seq=list(range(3))) == ("<ul>\n <li>0</li>\n " + "<li>1</li>\n <li>2</li>\n</ul>") + + def test_string_escapes(self): + for char in u'\0', u'\u2668', u'\xe4', u'\t', u'\r', u'\n': + tmpl = env.from_string('{{ %s }}' % jinja_string_repr(char)) + assert tmpl.render() == char + assert env.from_string('{{ "\N{HOT SPRINGS}" }}').render() == u'\u2668' + + def test_bytefallback(self): + from pprint import pformat + tmpl = env.from_string(u'''{{ 'foo'|pprint }}|{{ 'bär'|pprint }}''') + assert tmpl.render() == pformat('foo') + '|' + pformat(u'bär') + + def test_operators(self): + from jinja2.lexer import operators + for test, expect in iteritems(operators): + if test in '([{}])': + continue + stream = env.lexer.tokenize('{{ %s }}' % test) + next(stream) + assert stream.current.type == expect + + def test_normalizing(self): + for seq in '\r', '\r\n', '\n': + env = Environment(newline_sequence=seq) + tmpl = env.from_string('1\n2\r\n3\n4\n') + result = tmpl.render() + assert result.replace(seq, 'X') == '1X2X3X4' + + def test_trailing_newline(self): + for keep in [True, False]: + env = Environment(keep_trailing_newline=keep) + for template,expected in [ + ('', {}), + ('no\nnewline', {}), + ('with\nnewline\n', {False: 'with\nnewline'}), + ('with\nseveral\n\n\n', {False: 'with\nseveral\n\n'}), + ]: + tmpl = env.from_string(template) + expect = expected.get(keep, template) + result = tmpl.render() + assert result == expect, (keep, template, result, expect) + +class ParserTestCase(JinjaTestCase): + + def test_php_syntax(self): + env = Environment('<?', '?>', '<?=', '?>', '<!--', '-->') + tmpl = env.from_string('''\ +<!-- I'm a comment, I'm not interesting -->\ +<? for item in seq -?> + <?= item ?> +<?- endfor ?>''') + assert tmpl.render(seq=list(range(5))) == '01234' + + def test_erb_syntax(self): + env = Environment('<%', '%>', '<%=', '%>', '<%#', '%>') + tmpl = env.from_string('''\ +<%# I'm a comment, I'm not interesting %>\ +<% for item in seq -%> + <%= item %> +<%- endfor %>''') + assert tmpl.render(seq=list(range(5))) == '01234' + + def test_comment_syntax(self): + env = Environment('<!--', '-->', '${', '}', '<!--#', '-->') + tmpl = env.from_string('''\ +<!--# I'm a comment, I'm not interesting -->\ +<!-- for item in seq ---> + ${item} +<!--- endfor -->''') + assert tmpl.render(seq=list(range(5))) == '01234' + + def test_balancing(self): + tmpl = env.from_string('''{{{'foo':'bar'}.foo}}''') + assert tmpl.render() == 'bar' + + def test_start_comment(self): + tmpl = env.from_string('''{# foo comment +and bar comment #} +{% macro blub() %}foo{% endmacro %} +{{ blub() }}''') + assert tmpl.render().strip() == 'foo' + + def test_line_syntax(self): + env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%') + tmpl = env.from_string('''\ +<%# regular comment %> +% for item in seq: + ${item} +% endfor''') + assert [int(x.strip()) for x in tmpl.render(seq=list(range(5))).split()] == \ + list(range(5)) + + env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##') + tmpl = env.from_string('''\ +<%# regular comment %> +% for item in seq: + ${item} ## the rest of the stuff +% endfor''') + assert [int(x.strip()) for x in tmpl.render(seq=list(range(5))).split()] == \ + list(range(5)) + + def test_line_syntax_priority(self): + # XXX: why is the whitespace there in front of the newline? + env = Environment('{%', '%}', '${', '}', '/*', '*/', '##', '#') + tmpl = env.from_string('''\ +/* ignore me. + I'm a multiline comment */ +## for item in seq: +* ${item} # this is just extra stuff +## endfor''') + assert tmpl.render(seq=[1, 2]).strip() == '* 1\n* 2' + env = Environment('{%', '%}', '${', '}', '/*', '*/', '#', '##') + tmpl = env.from_string('''\ +/* ignore me. + I'm a multiline comment */ +# for item in seq: +* ${item} ## this is just extra stuff + ## extra stuff i just want to ignore +# endfor''') + assert tmpl.render(seq=[1, 2]).strip() == '* 1\n\n* 2' + + def test_error_messages(self): + def assert_error(code, expected): + try: + Template(code) + except TemplateSyntaxError as e: + assert str(e) == expected, 'unexpected error message' + else: + assert False, 'that was supposed to be an error' + + assert_error('{% for item in seq %}...{% endif %}', + "Encountered unknown tag 'endif'. Jinja was looking " + "for the following tags: 'endfor' or 'else'. The " + "innermost block that needs to be closed is 'for'.") + assert_error('{% if foo %}{% for item in seq %}...{% endfor %}{% endfor %}', + "Encountered unknown tag 'endfor'. Jinja was looking for " + "the following tags: 'elif' or 'else' or 'endif'. The " + "innermost block that needs to be closed is 'if'.") + assert_error('{% if foo %}', + "Unexpected end of template. Jinja was looking for the " + "following tags: 'elif' or 'else' or 'endif'. The " + "innermost block that needs to be closed is 'if'.") + assert_error('{% for item in seq %}', + "Unexpected end of template. Jinja was looking for the " + "following tags: 'endfor' or 'else'. The innermost block " + "that needs to be closed is 'for'.") + assert_error('{% block foo-bar-baz %}', + "Block names in Jinja have to be valid Python identifiers " + "and may not contain hyphens, use an underscore instead.") + assert_error('{% unknown_tag %}', + "Encountered unknown tag 'unknown_tag'.") + + +class SyntaxTestCase(JinjaTestCase): + + def test_call(self): + env = Environment() + env.globals['foo'] = lambda a, b, c, e, g: a + b + c + e + g + tmpl = env.from_string("{{ foo('a', c='d', e='f', *['b'], **{'g': 'h'}) }}") + assert tmpl.render() == 'abdfh' + + def test_slicing(self): + tmpl = env.from_string('{{ [1, 2, 3][:] }}|{{ [1, 2, 3][::-1] }}') + assert tmpl.render() == '[1, 2, 3]|[3, 2, 1]' + + def test_attr(self): + tmpl = env.from_string("{{ foo.bar }}|{{ foo['bar'] }}") + assert tmpl.render(foo={'bar': 42}) == '42|42' + + def test_subscript(self): + tmpl = env.from_string("{{ foo[0] }}|{{ foo[-1] }}") + assert tmpl.render(foo=[0, 1, 2]) == '0|2' + + def test_tuple(self): + tmpl = env.from_string('{{ () }}|{{ (1,) }}|{{ (1, 2) }}') + assert tmpl.render() == '()|(1,)|(1, 2)' + + def test_math(self): + tmpl = env.from_string('{{ (1 + 1 * 2) - 3 / 2 }}|{{ 2**3 }}') + assert tmpl.render() == '1.5|8' + + def test_div(self): + tmpl = env.from_string('{{ 3 // 2 }}|{{ 3 / 2 }}|{{ 3 % 2 }}') + assert tmpl.render() == '1|1.5|1' + + def test_unary(self): + tmpl = env.from_string('{{ +3 }}|{{ -3 }}') + assert tmpl.render() == '3|-3' + + def test_concat(self): + tmpl = env.from_string("{{ [1, 2] ~ 'foo' }}") + assert tmpl.render() == '[1, 2]foo' + + def test_compare(self): + tmpl = env.from_string('{{ 1 > 0 }}|{{ 1 >= 1 }}|{{ 2 < 3 }}|' + '{{ 2 == 2 }}|{{ 1 <= 1 }}') + assert tmpl.render() == 'True|True|True|True|True' + + def test_inop(self): + tmpl = env.from_string('{{ 1 in [1, 2, 3] }}|{{ 1 not in [1, 2, 3] }}') + assert tmpl.render() == 'True|False' + + def test_literals(self): + tmpl = env.from_string('{{ [] }}|{{ {} }}|{{ () }}') + assert tmpl.render().lower() == '[]|{}|()' + + def test_bool(self): + tmpl = env.from_string('{{ true and false }}|{{ false ' + 'or true }}|{{ not false }}') + assert tmpl.render() == 'False|True|True' + + def test_grouping(self): + tmpl = env.from_string('{{ (true and false) or (false and true) and not false }}') + assert tmpl.render() == 'False' + + def test_django_attr(self): + tmpl = env.from_string('{{ [1, 2, 3].0 }}|{{ [[1]].0.0 }}') + assert tmpl.render() == '1|1' + + def test_conditional_expression(self): + tmpl = env.from_string('''{{ 0 if true else 1 }}''') + assert tmpl.render() == '0' + + def test_short_conditional_expression(self): + tmpl = env.from_string('<{{ 1 if false }}>') + assert tmpl.render() == '<>' + + tmpl = env.from_string('<{{ (1 if false).bar }}>') + self.assert_raises(UndefinedError, tmpl.render) + + def test_filter_priority(self): + tmpl = env.from_string('{{ "foo"|upper + "bar"|upper }}') + assert tmpl.render() == 'FOOBAR' + + def test_function_calls(self): + tests = [ + (True, '*foo, bar'), + (True, '*foo, *bar'), + (True, '*foo, bar=42'), + (True, '**foo, *bar'), + (True, '**foo, bar'), + (False, 'foo, bar'), + (False, 'foo, bar=42'), + (False, 'foo, bar=23, *args'), + (False, 'a, b=c, *d, **e'), + (False, '*foo, **bar') + ] + for should_fail, sig in tests: + if should_fail: + self.assert_raises(TemplateSyntaxError, + env.from_string, '{{ foo(%s) }}' % sig) + else: + env.from_string('foo(%s)' % sig) + + def test_tuple_expr(self): + for tmpl in [ + '{{ () }}', + '{{ (1, 2) }}', + '{{ (1, 2,) }}', + '{{ 1, }}', + '{{ 1, 2 }}', + '{% for foo, bar in seq %}...{% endfor %}', + '{% for x in foo, bar %}...{% endfor %}', + '{% for x in foo, %}...{% endfor %}' + ]: + assert env.from_string(tmpl) + + def test_trailing_comma(self): + tmpl = env.from_string('{{ (1, 2,) }}|{{ [1, 2,] }}|{{ {1: 2,} }}') + assert tmpl.render().lower() == '(1, 2)|[1, 2]|{1: 2}' + + def test_block_end_name(self): + env.from_string('{% block foo %}...{% endblock foo %}') + self.assert_raises(TemplateSyntaxError, env.from_string, + '{% block x %}{% endblock y %}') + + def test_constant_casing(self): + for const in True, False, None: + tmpl = env.from_string('{{ %s }}|{{ %s }}|{{ %s }}' % ( + str(const), str(const).lower(), str(const).upper() + )) + assert tmpl.render() == '%s|%s|' % (const, const) + + def test_test_chaining(self): + self.assert_raises(TemplateSyntaxError, env.from_string, + '{{ foo is string is sequence }}') + assert env.from_string('{{ 42 is string or 42 is number }}' + ).render() == 'True' + + def test_string_concatenation(self): + tmpl = env.from_string('{{ "foo" "bar" "baz" }}') + assert tmpl.render() == 'foobarbaz' + + def test_notin(self): + bar = range(100) + tmpl = env.from_string('''{{ not 42 in bar }}''') + assert tmpl.render(bar=bar) == text_type(not 42 in bar) + + def test_implicit_subscribed_tuple(self): + class Foo(object): + def __getitem__(self, x): + return x + t = env.from_string('{{ foo[1, 2] }}') + assert t.render(foo=Foo()) == u'(1, 2)' + + def test_raw2(self): + tmpl = env.from_string('{% raw %}{{ FOO }} and {% BAR %}{% endraw %}') + assert tmpl.render() == '{{ FOO }} and {% BAR %}' + + def test_const(self): + tmpl = env.from_string('{{ true }}|{{ false }}|{{ none }}|' + '{{ none is defined }}|{{ missing is defined }}') + assert tmpl.render() == 'True|False|None|True|False' + + def test_neg_filter_priority(self): + node = env.parse('{{ -1|foo }}') + assert isinstance(node.body[0].nodes[0], nodes.Filter) + assert isinstance(node.body[0].nodes[0].node, nodes.Neg) + + def test_const_assign(self): + constass1 = '''{% set true = 42 %}''' + constass2 = '''{% for none in seq %}{% endfor %}''' + for tmpl in constass1, constass2: + self.assert_raises(TemplateSyntaxError, env.from_string, tmpl) + + def test_localset(self): + tmpl = env.from_string('''{% set foo = 0 %}\ +{% for item in [1, 2] %}{% set foo = 1 %}{% endfor %}\ +{{ foo }}''') + assert tmpl.render() == '0' + + def test_parse_unary(self): + tmpl = env.from_string('{{ -foo["bar"] }}') + assert tmpl.render(foo={'bar': 42}) == '-42' + tmpl = env.from_string('{{ -foo["bar"]|abs }}') + assert tmpl.render(foo={'bar': 42}) == '42' + + +class LstripBlocksTestCase(JinjaTestCase): + + def test_lstrip(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {% if True %}\n {% endif %}''') + assert tmpl.render() == "\n" + + def test_lstrip_trim(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string(''' {% if True %}\n {% endif %}''') + assert tmpl.render() == "" + + def test_no_lstrip(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {%+ if True %}\n {%+ endif %}''') + assert tmpl.render() == " \n " + + def test_lstrip_endline(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' hello{% if True %}\n goodbye{% endif %}''') + assert tmpl.render() == " hello\n goodbye" + + def test_lstrip_inline(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {% if True %}hello {% endif %}''') + assert tmpl.render() == 'hello ' + + def test_lstrip_nested(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {% if True %}a {% if True %}b {% endif %}c {% endif %}''') + assert tmpl.render() == 'a b c ' + + def test_lstrip_left_chars(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' abc {% if True %} + hello{% endif %}''') + assert tmpl.render() == ' abc \n hello' + + def test_lstrip_embeded_strings(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {% set x = " {% str %} " %}{{ x }}''') + assert tmpl.render() == ' {% str %} ' + + def test_lstrip_preserve_leading_newlines(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string('''\n\n\n{% set hello = 1 %}''') + assert tmpl.render() == '\n\n\n' + + def test_lstrip_comment(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {# if True #} +hello + {#endif#}''') + assert tmpl.render() == '\nhello\n' + + def test_lstrip_angle_bracket_simple(self): + env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string(''' <% if True %>hello <% endif %>''') + assert tmpl.render() == 'hello ' + + def test_lstrip_angle_bracket_comment(self): + env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string(''' <%# if True %>hello <%# endif %>''') + assert tmpl.render() == 'hello ' + + def test_lstrip_angle_bracket(self): + env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ + <%# regular comment %> + <% for item in seq %> +${item} ## the rest of the stuff + <% endfor %>''') + assert tmpl.render(seq=range(5)) == \ + ''.join('%s\n' % x for x in range(5)) + + def test_lstrip_angle_bracket_compact(self): + env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ + <%#regular comment%> + <%for item in seq%> +${item} ## the rest of the stuff + <%endfor%>''') + assert tmpl.render(seq=range(5)) == \ + ''.join('%s\n' % x for x in range(5)) + + def test_php_syntax_with_manual(self): + env = Environment('<?', '?>', '<?=', '?>', '<!--', '-->', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ + <!-- I'm a comment, I'm not interesting --> + <? for item in seq -?> + <?= item ?> + <?- endfor ?>''') + assert tmpl.render(seq=range(5)) == '01234' + + def test_php_syntax(self): + env = Environment('<?', '?>', '<?=', '?>', '<!--', '-->', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ + <!-- I'm a comment, I'm not interesting --> + <? for item in seq ?> + <?= item ?> + <? endfor ?>''') + assert tmpl.render(seq=range(5)) == ''.join(' %s\n' % x for x in range(5)) + + def test_php_syntax_compact(self): + env = Environment('<?', '?>', '<?=', '?>', '<!--', '-->', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ + <!-- I'm a comment, I'm not interesting --> + <?for item in seq?> + <?=item?> + <?endfor?>''') + assert tmpl.render(seq=range(5)) == ''.join(' %s\n' % x for x in range(5)) + + def test_erb_syntax(self): + env = Environment('<%', '%>', '<%=', '%>', '<%#', '%>', + lstrip_blocks=True, trim_blocks=True) + #env.from_string('') + #for n,r in env.lexer.rules.iteritems(): + # print n + #print env.lexer.rules['root'][0][0].pattern + #print "'%s'" % tmpl.render(seq=range(5)) + tmpl = env.from_string('''\ +<%# I'm a comment, I'm not interesting %> + <% for item in seq %> + <%= item %> + <% endfor %> +''') + assert tmpl.render(seq=range(5)) == ''.join(' %s\n' % x for x in range(5)) + + def test_erb_syntax_with_manual(self): + env = Environment('<%', '%>', '<%=', '%>', '<%#', '%>', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ +<%# I'm a comment, I'm not interesting %> + <% for item in seq -%> + <%= item %> + <%- endfor %>''') + assert tmpl.render(seq=range(5)) == '01234' + + def test_erb_syntax_no_lstrip(self): + env = Environment('<%', '%>', '<%=', '%>', '<%#', '%>', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ +<%# I'm a comment, I'm not interesting %> + <%+ for item in seq -%> + <%= item %> + <%- endfor %>''') + assert tmpl.render(seq=range(5)) == ' 01234' + + def test_comment_syntax(self): + env = Environment('<!--', '-->', '${', '}', '<!--#', '-->', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ +<!--# I'm a comment, I'm not interesting -->\ +<!-- for item in seq ---> + ${item} +<!--- endfor -->''') + assert tmpl.render(seq=range(5)) == '01234' + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TokenStreamTestCase)) + suite.addTest(unittest.makeSuite(LexerTestCase)) + suite.addTest(unittest.makeSuite(ParserTestCase)) + suite.addTest(unittest.makeSuite(SyntaxTestCase)) + suite.addTest(unittest.makeSuite(LstripBlocksTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/loader.py b/module/lib/jinja2/testsuite/loader.py new file mode 100644 index 000000000..a7350aab9 --- /dev/null +++ b/module/lib/jinja2/testsuite/loader.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.loader + ~~~~~~~~~~~~~~~~~~~~~~~ + + Test the loaders. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import os +import sys +import tempfile +import shutil +import unittest + +from jinja2.testsuite import JinjaTestCase, dict_loader, \ + package_loader, filesystem_loader, function_loader, \ + choice_loader, prefix_loader + +from jinja2 import Environment, loaders +from jinja2._compat import PYPY, PY2 +from jinja2.loaders import split_template_path +from jinja2.exceptions import TemplateNotFound + + +class LoaderTestCase(JinjaTestCase): + + def test_dict_loader(self): + env = Environment(loader=dict_loader) + tmpl = env.get_template('justdict.html') + assert tmpl.render().strip() == 'FOO' + self.assert_raises(TemplateNotFound, env.get_template, 'missing.html') + + def test_package_loader(self): + env = Environment(loader=package_loader) + tmpl = env.get_template('test.html') + assert tmpl.render().strip() == 'BAR' + self.assert_raises(TemplateNotFound, env.get_template, 'missing.html') + + def test_filesystem_loader(self): + env = Environment(loader=filesystem_loader) + tmpl = env.get_template('test.html') + assert tmpl.render().strip() == 'BAR' + tmpl = env.get_template('foo/test.html') + assert tmpl.render().strip() == 'FOO' + self.assert_raises(TemplateNotFound, env.get_template, 'missing.html') + + def test_choice_loader(self): + env = Environment(loader=choice_loader) + tmpl = env.get_template('justdict.html') + assert tmpl.render().strip() == 'FOO' + tmpl = env.get_template('test.html') + assert tmpl.render().strip() == 'BAR' + self.assert_raises(TemplateNotFound, env.get_template, 'missing.html') + + def test_function_loader(self): + env = Environment(loader=function_loader) + tmpl = env.get_template('justfunction.html') + assert tmpl.render().strip() == 'FOO' + self.assert_raises(TemplateNotFound, env.get_template, 'missing.html') + + def test_prefix_loader(self): + env = Environment(loader=prefix_loader) + tmpl = env.get_template('a/test.html') + assert tmpl.render().strip() == 'BAR' + tmpl = env.get_template('b/justdict.html') + assert tmpl.render().strip() == 'FOO' + self.assert_raises(TemplateNotFound, env.get_template, 'missing') + + def test_caching(self): + changed = False + class TestLoader(loaders.BaseLoader): + def get_source(self, environment, template): + return u'foo', None, lambda: not changed + env = Environment(loader=TestLoader(), cache_size=-1) + tmpl = env.get_template('template') + assert tmpl is env.get_template('template') + changed = True + assert tmpl is not env.get_template('template') + changed = False + + env = Environment(loader=TestLoader(), cache_size=0) + assert env.get_template('template') \ + is not env.get_template('template') + + env = Environment(loader=TestLoader(), cache_size=2) + t1 = env.get_template('one') + t2 = env.get_template('two') + assert t2 is env.get_template('two') + assert t1 is env.get_template('one') + t3 = env.get_template('three') + assert 'one' in env.cache + assert 'two' not in env.cache + assert 'three' in env.cache + + def test_dict_loader_cache_invalidates(self): + mapping = {'foo': "one"} + env = Environment(loader=loaders.DictLoader(mapping)) + assert env.get_template('foo').render() == "one" + mapping['foo'] = "two" + assert env.get_template('foo').render() == "two" + + def test_split_template_path(self): + assert split_template_path('foo/bar') == ['foo', 'bar'] + assert split_template_path('./foo/bar') == ['foo', 'bar'] + self.assert_raises(TemplateNotFound, split_template_path, '../foo') + + +class ModuleLoaderTestCase(JinjaTestCase): + archive = None + + def compile_down(self, zip='deflated', py_compile=False): + super(ModuleLoaderTestCase, self).setup() + log = [] + self.reg_env = Environment(loader=prefix_loader) + if zip is not None: + self.archive = tempfile.mkstemp(suffix='.zip')[1] + else: + self.archive = tempfile.mkdtemp() + self.reg_env.compile_templates(self.archive, zip=zip, + log_function=log.append, + py_compile=py_compile) + self.mod_env = Environment(loader=loaders.ModuleLoader(self.archive)) + return ''.join(log) + + def teardown(self): + super(ModuleLoaderTestCase, self).teardown() + if hasattr(self, 'mod_env'): + if os.path.isfile(self.archive): + os.remove(self.archive) + else: + shutil.rmtree(self.archive) + self.archive = None + + def test_log(self): + log = self.compile_down() + assert 'Compiled "a/foo/test.html" as ' \ + 'tmpl_a790caf9d669e39ea4d280d597ec891c4ef0404a' in log + assert 'Finished compiling templates' in log + assert 'Could not compile "a/syntaxerror.html": ' \ + 'Encountered unknown tag \'endif\'' in log + + def _test_common(self): + tmpl1 = self.reg_env.get_template('a/test.html') + tmpl2 = self.mod_env.get_template('a/test.html') + assert tmpl1.render() == tmpl2.render() + + tmpl1 = self.reg_env.get_template('b/justdict.html') + tmpl2 = self.mod_env.get_template('b/justdict.html') + assert tmpl1.render() == tmpl2.render() + + def test_deflated_zip_compile(self): + self.compile_down(zip='deflated') + self._test_common() + + def test_stored_zip_compile(self): + self.compile_down(zip='stored') + self._test_common() + + def test_filesystem_compile(self): + self.compile_down(zip=None) + self._test_common() + + def test_weak_references(self): + self.compile_down() + tmpl = self.mod_env.get_template('a/test.html') + key = loaders.ModuleLoader.get_template_key('a/test.html') + name = self.mod_env.loader.module.__name__ + + assert hasattr(self.mod_env.loader.module, key) + assert name in sys.modules + + # unset all, ensure the module is gone from sys.modules + self.mod_env = tmpl = None + + try: + import gc + gc.collect() + except: + pass + + assert name not in sys.modules + + # This test only makes sense on non-pypy python 2 + if PY2 and not PYPY: + def test_byte_compilation(self): + log = self.compile_down(py_compile=True) + assert 'Byte-compiled "a/test.html"' in log + tmpl1 = self.mod_env.get_template('a/test.html') + mod = self.mod_env.loader.module. \ + tmpl_3c4ddf650c1a73df961a6d3d2ce2752f1b8fd490 + assert mod.__file__.endswith('.pyc') + + def test_choice_loader(self): + log = self.compile_down() + + self.mod_env.loader = loaders.ChoiceLoader([ + self.mod_env.loader, + loaders.DictLoader({'DICT_SOURCE': 'DICT_TEMPLATE'}) + ]) + + tmpl1 = self.mod_env.get_template('a/test.html') + self.assert_equal(tmpl1.render(), 'BAR') + tmpl2 = self.mod_env.get_template('DICT_SOURCE') + self.assert_equal(tmpl2.render(), 'DICT_TEMPLATE') + + def test_prefix_loader(self): + log = self.compile_down() + + self.mod_env.loader = loaders.PrefixLoader({ + 'MOD': self.mod_env.loader, + 'DICT': loaders.DictLoader({'test.html': 'DICT_TEMPLATE'}) + }) + + tmpl1 = self.mod_env.get_template('MOD/a/test.html') + self.assert_equal(tmpl1.render(), 'BAR') + tmpl2 = self.mod_env.get_template('DICT/test.html') + self.assert_equal(tmpl2.render(), 'DICT_TEMPLATE') + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(LoaderTestCase)) + suite.addTest(unittest.makeSuite(ModuleLoaderTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/regression.py b/module/lib/jinja2/testsuite/regression.py new file mode 100644 index 000000000..c5f7d5c65 --- /dev/null +++ b/module/lib/jinja2/testsuite/regression.py @@ -0,0 +1,279 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.regression + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Tests corner cases and bugs. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from jinja2.testsuite import JinjaTestCase + +from jinja2 import Template, Environment, DictLoader, TemplateSyntaxError, \ + TemplateNotFound, PrefixLoader +from jinja2._compat import text_type + +env = Environment() + + +class CornerTestCase(JinjaTestCase): + + def test_assigned_scoping(self): + t = env.from_string(''' + {%- for item in (1, 2, 3, 4) -%} + [{{ item }}] + {%- endfor %} + {{- item -}} + ''') + assert t.render(item=42) == '[1][2][3][4]42' + + t = env.from_string(''' + {%- for item in (1, 2, 3, 4) -%} + [{{ item }}] + {%- endfor %} + {%- set item = 42 %} + {{- item -}} + ''') + assert t.render() == '[1][2][3][4]42' + + t = env.from_string(''' + {%- set item = 42 %} + {%- for item in (1, 2, 3, 4) -%} + [{{ item }}] + {%- endfor %} + {{- item -}} + ''') + assert t.render() == '[1][2][3][4]42' + + def test_closure_scoping(self): + t = env.from_string(''' + {%- set wrapper = "<FOO>" %} + {%- for item in (1, 2, 3, 4) %} + {%- macro wrapper() %}[{{ item }}]{% endmacro %} + {{- wrapper() }} + {%- endfor %} + {{- wrapper -}} + ''') + assert t.render() == '[1][2][3][4]<FOO>' + + t = env.from_string(''' + {%- for item in (1, 2, 3, 4) %} + {%- macro wrapper() %}[{{ item }}]{% endmacro %} + {{- wrapper() }} + {%- endfor %} + {%- set wrapper = "<FOO>" %} + {{- wrapper -}} + ''') + assert t.render() == '[1][2][3][4]<FOO>' + + t = env.from_string(''' + {%- for item in (1, 2, 3, 4) %} + {%- macro wrapper() %}[{{ item }}]{% endmacro %} + {{- wrapper() }} + {%- endfor %} + {{- wrapper -}} + ''') + assert t.render(wrapper=23) == '[1][2][3][4]23' + + +class BugTestCase(JinjaTestCase): + + def test_keyword_folding(self): + env = Environment() + env.filters['testing'] = lambda value, some: value + some + assert env.from_string("{{ 'test'|testing(some='stuff') }}") \ + .render() == 'teststuff' + + def test_extends_output_bugs(self): + env = Environment(loader=DictLoader({ + 'parent.html': '(({% block title %}{% endblock %}))' + })) + + t = env.from_string('{% if expr %}{% extends "parent.html" %}{% endif %}' + '[[{% block title %}title{% endblock %}]]' + '{% for item in [1, 2, 3] %}({{ item }}){% endfor %}') + assert t.render(expr=False) == '[[title]](1)(2)(3)' + assert t.render(expr=True) == '((title))' + + def test_urlize_filter_escaping(self): + tmpl = env.from_string('{{ "http://www.example.org/<foo"|urlize }}') + assert tmpl.render() == '<a href="http://www.example.org/<foo">http://www.example.org/<foo</a>' + + def test_loop_call_loop(self): + tmpl = env.from_string(''' + + {% macro test() %} + {{ caller() }} + {% endmacro %} + + {% for num1 in range(5) %} + {% call test() %} + {% for num2 in range(10) %} + {{ loop.index }} + {% endfor %} + {% endcall %} + {% endfor %} + + ''') + + assert tmpl.render().split() == [text_type(x) for x in range(1, 11)] * 5 + + def test_weird_inline_comment(self): + env = Environment(line_statement_prefix='%') + self.assert_raises(TemplateSyntaxError, env.from_string, + '% for item in seq {# missing #}\n...% endfor') + + def test_old_macro_loop_scoping_bug(self): + tmpl = env.from_string('{% for i in (1, 2) %}{{ i }}{% endfor %}' + '{% macro i() %}3{% endmacro %}{{ i() }}') + assert tmpl.render() == '123' + + def test_partial_conditional_assignments(self): + tmpl = env.from_string('{% if b %}{% set a = 42 %}{% endif %}{{ a }}') + assert tmpl.render(a=23) == '23' + assert tmpl.render(b=True) == '42' + + def test_stacked_locals_scoping_bug(self): + env = Environment(line_statement_prefix='#') + t = env.from_string('''\ +# for j in [1, 2]: +# set x = 1 +# for i in [1, 2]: +# print x +# if i % 2 == 0: +# set x = x + 1 +# endif +# endfor +# endfor +# if a +# print 'A' +# elif b +# print 'B' +# elif c == d +# print 'C' +# else +# print 'D' +# endif + ''') + assert t.render(a=0, b=False, c=42, d=42.0) == '1111C' + + def test_stacked_locals_scoping_bug_twoframe(self): + t = Template(''' + {% set x = 1 %} + {% for item in foo %} + {% if item == 1 %} + {% set x = 2 %} + {% endif %} + {% endfor %} + {{ x }} + ''') + rv = t.render(foo=[1]).strip() + assert rv == u'1' + + def test_call_with_args(self): + t = Template("""{% macro dump_users(users) -%} + <ul> + {%- for user in users -%} + <li><p>{{ user.username|e }}</p>{{ caller(user) }}</li> + {%- endfor -%} + </ul> + {%- endmacro -%} + + {% call(user) dump_users(list_of_user) -%} + <dl> + <dl>Realname</dl> + <dd>{{ user.realname|e }}</dd> + <dl>Description</dl> + <dd>{{ user.description }}</dd> + </dl> + {% endcall %}""") + + assert [x.strip() for x in t.render(list_of_user=[{ + 'username':'apo', + 'realname':'something else', + 'description':'test' + }]).splitlines()] == [ + u'<ul><li><p>apo</p><dl>', + u'<dl>Realname</dl>', + u'<dd>something else</dd>', + u'<dl>Description</dl>', + u'<dd>test</dd>', + u'</dl>', + u'</li></ul>' + ] + + def test_empty_if_condition_fails(self): + self.assert_raises(TemplateSyntaxError, Template, '{% if %}....{% endif %}') + self.assert_raises(TemplateSyntaxError, Template, '{% if foo %}...{% elif %}...{% endif %}') + self.assert_raises(TemplateSyntaxError, Template, '{% for x in %}..{% endfor %}') + + def test_recursive_loop_bug(self): + tpl1 = Template(""" + {% for p in foo recursive%} + {{p.bar}} + {% for f in p.fields recursive%} + {{f.baz}} + {{p.bar}} + {% if f.rec %} + {{ loop(f.sub) }} + {% endif %} + {% endfor %} + {% endfor %} + """) + + tpl2 = Template(""" + {% for p in foo%} + {{p.bar}} + {% for f in p.fields recursive%} + {{f.baz}} + {{p.bar}} + {% if f.rec %} + {{ loop(f.sub) }} + {% endif %} + {% endfor %} + {% endfor %} + """) + + def test_else_loop_bug(self): + t = Template(''' + {% for x in y %} + {{ loop.index0 }} + {% else %} + {% for i in range(3) %}{{ i }}{% endfor %} + {% endfor %} + ''') + self.assertEqual(t.render(y=[]).strip(), '012') + + def test_correct_prefix_loader_name(self): + env = Environment(loader=PrefixLoader({ + 'foo': DictLoader({}) + })) + try: + env.get_template('foo/bar.html') + except TemplateNotFound as e: + assert e.name == 'foo/bar.html' + else: + assert False, 'expected error here' + + def test_contextfunction_callable_classes(self): + from jinja2.utils import contextfunction + class CallableClass(object): + @contextfunction + def __call__(self, ctx): + return ctx.resolve('hello') + + tpl = Template("""{{ callableclass() }}""") + output = tpl.render(callableclass = CallableClass(), hello = 'TEST') + expected = 'TEST' + + self.assert_equal(output, expected) + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(CornerTestCase)) + suite.addTest(unittest.makeSuite(BugTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/res/__init__.py b/module/lib/jinja2/testsuite/res/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/module/lib/jinja2/testsuite/res/__init__.py diff --git a/module/lib/jinja2/testsuite/res/templates/broken.html b/module/lib/jinja2/testsuite/res/templates/broken.html new file mode 100644 index 000000000..77669fae5 --- /dev/null +++ b/module/lib/jinja2/testsuite/res/templates/broken.html @@ -0,0 +1,3 @@ +Before +{{ fail() }} +After diff --git a/module/lib/jinja2/testsuite/res/templates/foo/test.html b/module/lib/jinja2/testsuite/res/templates/foo/test.html new file mode 100644 index 000000000..b7d6715e2 --- /dev/null +++ b/module/lib/jinja2/testsuite/res/templates/foo/test.html @@ -0,0 +1 @@ +FOO diff --git a/module/lib/jinja2/testsuite/res/templates/syntaxerror.html b/module/lib/jinja2/testsuite/res/templates/syntaxerror.html new file mode 100644 index 000000000..f21b81793 --- /dev/null +++ b/module/lib/jinja2/testsuite/res/templates/syntaxerror.html @@ -0,0 +1,4 @@ +Foo +{% for item in broken %} + ... +{% endif %} diff --git a/module/lib/jinja2/testsuite/res/templates/test.html b/module/lib/jinja2/testsuite/res/templates/test.html new file mode 100644 index 000000000..ba578e48b --- /dev/null +++ b/module/lib/jinja2/testsuite/res/templates/test.html @@ -0,0 +1 @@ +BAR diff --git a/module/lib/jinja2/testsuite/security.py b/module/lib/jinja2/testsuite/security.py new file mode 100644 index 000000000..246d0f073 --- /dev/null +++ b/module/lib/jinja2/testsuite/security.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.security + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Checks the sandbox and other security features. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest + +from jinja2.testsuite import JinjaTestCase + +from jinja2 import Environment +from jinja2.sandbox import SandboxedEnvironment, \ + ImmutableSandboxedEnvironment, unsafe +from jinja2 import Markup, escape +from jinja2.exceptions import SecurityError, TemplateSyntaxError, \ + TemplateRuntimeError +from jinja2._compat import text_type + + +class PrivateStuff(object): + + def bar(self): + return 23 + + @unsafe + def foo(self): + return 42 + + def __repr__(self): + return 'PrivateStuff' + + +class PublicStuff(object): + bar = lambda self: 23 + _foo = lambda self: 42 + + def __repr__(self): + return 'PublicStuff' + + +class SandboxTestCase(JinjaTestCase): + + def test_unsafe(self): + env = SandboxedEnvironment() + self.assert_raises(SecurityError, env.from_string("{{ foo.foo() }}").render, + foo=PrivateStuff()) + self.assert_equal(env.from_string("{{ foo.bar() }}").render(foo=PrivateStuff()), '23') + + self.assert_raises(SecurityError, env.from_string("{{ foo._foo() }}").render, + foo=PublicStuff()) + self.assert_equal(env.from_string("{{ foo.bar() }}").render(foo=PublicStuff()), '23') + self.assert_equal(env.from_string("{{ foo.__class__ }}").render(foo=42), '') + self.assert_equal(env.from_string("{{ foo.func_code }}").render(foo=lambda:None), '') + # security error comes from __class__ already. + self.assert_raises(SecurityError, env.from_string( + "{{ foo.__class__.__subclasses__() }}").render, foo=42) + + def test_immutable_environment(self): + env = ImmutableSandboxedEnvironment() + self.assert_raises(SecurityError, env.from_string( + '{{ [].append(23) }}').render) + self.assert_raises(SecurityError, env.from_string( + '{{ {1:2}.clear() }}').render) + + def test_restricted(self): + env = SandboxedEnvironment() + self.assert_raises(TemplateSyntaxError, env.from_string, + "{% for item.attribute in seq %}...{% endfor %}") + self.assert_raises(TemplateSyntaxError, env.from_string, + "{% for foo, bar.baz in seq %}...{% endfor %}") + + def test_markup_operations(self): + # adding two strings should escape the unsafe one + unsafe = '<script type="application/x-some-script">alert("foo");</script>' + safe = Markup('<em>username</em>') + assert unsafe + safe == text_type(escape(unsafe)) + text_type(safe) + + # string interpolations are safe to use too + assert Markup('<em>%s</em>') % '<bad user>' == \ + '<em><bad user></em>' + assert Markup('<em>%(username)s</em>') % { + 'username': '<bad user>' + } == '<em><bad user></em>' + + # an escaped object is markup too + assert type(Markup('foo') + 'bar') is Markup + + # and it implements __html__ by returning itself + x = Markup("foo") + assert x.__html__() is x + + # it also knows how to treat __html__ objects + class Foo(object): + def __html__(self): + return '<em>awesome</em>' + def __unicode__(self): + return 'awesome' + assert Markup(Foo()) == '<em>awesome</em>' + assert Markup('<strong>%s</strong>') % Foo() == \ + '<strong><em>awesome</em></strong>' + + # escaping and unescaping + assert escape('"<>&\'') == '"<>&'' + assert Markup("<em>Foo & Bar</em>").striptags() == "Foo & Bar" + assert Markup("<test>").unescape() == "<test>" + + def test_template_data(self): + env = Environment(autoescape=True) + t = env.from_string('{% macro say_hello(name) %}' + '<p>Hello {{ name }}!</p>{% endmacro %}' + '{{ say_hello("<blink>foo</blink>") }}') + escaped_out = '<p>Hello <blink>foo</blink>!</p>' + assert t.render() == escaped_out + assert text_type(t.module) == escaped_out + assert escape(t.module) == escaped_out + assert t.module.say_hello('<blink>foo</blink>') == escaped_out + assert escape(t.module.say_hello('<blink>foo</blink>')) == escaped_out + + def test_attr_filter(self): + env = SandboxedEnvironment() + tmpl = env.from_string('{{ cls|attr("__subclasses__")() }}') + self.assert_raises(SecurityError, tmpl.render, cls=int) + + def test_binary_operator_intercepting(self): + def disable_op(left, right): + raise TemplateRuntimeError('that operator so does not work') + for expr, ctx, rv in ('1 + 2', {}, '3'), ('a + 2', {'a': 2}, '4'): + env = SandboxedEnvironment() + env.binop_table['+'] = disable_op + t = env.from_string('{{ %s }}' % expr) + assert t.render(ctx) == rv + env.intercepted_binops = frozenset(['+']) + t = env.from_string('{{ %s }}' % expr) + try: + t.render(ctx) + except TemplateRuntimeError as e: + pass + else: + self.fail('expected runtime error') + + def test_unary_operator_intercepting(self): + def disable_op(arg): + raise TemplateRuntimeError('that operator so does not work') + for expr, ctx, rv in ('-1', {}, '-1'), ('-a', {'a': 2}, '-2'): + env = SandboxedEnvironment() + env.unop_table['-'] = disable_op + t = env.from_string('{{ %s }}' % expr) + assert t.render(ctx) == rv + env.intercepted_unops = frozenset(['-']) + t = env.from_string('{{ %s }}' % expr) + try: + t.render(ctx) + except TemplateRuntimeError as e: + pass + else: + self.fail('expected runtime error') + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(SandboxTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/tests.py b/module/lib/jinja2/testsuite/tests.py new file mode 100644 index 000000000..3ece7a8ff --- /dev/null +++ b/module/lib/jinja2/testsuite/tests.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.tests + ~~~~~~~~~~~~~~~~~~~~~~ + + Who tests the tests? + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import unittest +from jinja2.testsuite import JinjaTestCase + +from jinja2 import Markup, Environment + +env = Environment() + + +class TestsTestCase(JinjaTestCase): + + def test_defined(self): + tmpl = env.from_string('{{ missing is defined }}|{{ true is defined }}') + assert tmpl.render() == 'False|True' + + def test_even(self): + tmpl = env.from_string('''{{ 1 is even }}|{{ 2 is even }}''') + assert tmpl.render() == 'False|True' + + def test_odd(self): + tmpl = env.from_string('''{{ 1 is odd }}|{{ 2 is odd }}''') + assert tmpl.render() == 'True|False' + + def test_lower(self): + tmpl = env.from_string('''{{ "foo" is lower }}|{{ "FOO" is lower }}''') + assert tmpl.render() == 'True|False' + + def test_typechecks(self): + tmpl = env.from_string(''' + {{ 42 is undefined }} + {{ 42 is defined }} + {{ 42 is none }} + {{ none is none }} + {{ 42 is number }} + {{ 42 is string }} + {{ "foo" is string }} + {{ "foo" is sequence }} + {{ [1] is sequence }} + {{ range is callable }} + {{ 42 is callable }} + {{ range(5) is iterable }} + {{ {} is mapping }} + {{ mydict is mapping }} + {{ [] is mapping }} + ''') + class MyDict(dict): + pass + assert tmpl.render(mydict=MyDict()).split() == [ + 'False', 'True', 'False', 'True', 'True', 'False', + 'True', 'True', 'True', 'True', 'False', 'True', + 'True', 'True', 'False' + ] + + def test_sequence(self): + tmpl = env.from_string( + '{{ [1, 2, 3] is sequence }}|' + '{{ "foo" is sequence }}|' + '{{ 42 is sequence }}' + ) + assert tmpl.render() == 'True|True|False' + + def test_upper(self): + tmpl = env.from_string('{{ "FOO" is upper }}|{{ "foo" is upper }}') + assert tmpl.render() == 'True|False' + + def test_sameas(self): + tmpl = env.from_string('{{ foo is sameas false }}|' + '{{ 0 is sameas false }}') + assert tmpl.render(foo=False) == 'True|False' + + def test_no_paren_for_arg1(self): + tmpl = env.from_string('{{ foo is sameas none }}') + assert tmpl.render(foo=None) == 'True' + + def test_escaped(self): + env = Environment(autoescape=True) + tmpl = env.from_string('{{ x is escaped }}|{{ y is escaped }}') + assert tmpl.render(x='foo', y=Markup('foo')) == 'False|True' + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestsTestCase)) + return suite diff --git a/module/lib/jinja2/testsuite/utils.py b/module/lib/jinja2/testsuite/utils.py new file mode 100644 index 000000000..cab9b09a9 --- /dev/null +++ b/module/lib/jinja2/testsuite/utils.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +""" + jinja2.testsuite.utils + ~~~~~~~~~~~~~~~~~~~~~~ + + Tests utilities jinja uses. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import gc +import unittest + +import pickle + +from jinja2.testsuite import JinjaTestCase + +from jinja2.utils import LRUCache, escape, object_type_repr + + +class LRUCacheTestCase(JinjaTestCase): + + def test_simple(self): + d = LRUCache(3) + d["a"] = 1 + d["b"] = 2 + d["c"] = 3 + d["a"] + d["d"] = 4 + assert len(d) == 3 + assert 'a' in d and 'c' in d and 'd' in d and 'b' not in d + + def test_pickleable(self): + cache = LRUCache(2) + cache["foo"] = 42 + cache["bar"] = 23 + cache["foo"] + + for protocol in range(3): + copy = pickle.loads(pickle.dumps(cache, protocol)) + assert copy.capacity == cache.capacity + assert copy._mapping == cache._mapping + assert copy._queue == cache._queue + + +class HelpersTestCase(JinjaTestCase): + + def test_object_type_repr(self): + class X(object): + pass + self.assert_equal(object_type_repr(42), 'int object') + self.assert_equal(object_type_repr([]), 'list object') + self.assert_equal(object_type_repr(X()), + 'jinja2.testsuite.utils.X object') + self.assert_equal(object_type_repr(None), 'None') + self.assert_equal(object_type_repr(Ellipsis), 'Ellipsis') + + +class MarkupLeakTestCase(JinjaTestCase): + + def test_markup_leaks(self): + counts = set() + for count in range(20): + for item in range(1000): + escape("foo") + escape("<foo>") + escape(u"foo") + escape(u"<foo>") + counts.add(len(gc.get_objects())) + assert len(counts) == 1, 'ouch, c extension seems to leak objects' + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(LRUCacheTestCase)) + suite.addTest(unittest.makeSuite(HelpersTestCase)) + + # this test only tests the c extension + if not hasattr(escape, 'func_code'): + suite.addTest(unittest.makeSuite(MarkupLeakTestCase)) + + return suite diff --git a/module/lib/jinja2/utils.py b/module/lib/jinja2/utils.py index 7b77b8eb7..ddc47da0a 100644 --- a/module/lib/jinja2/utils.py +++ b/module/lib/jinja2/utils.py @@ -9,21 +9,17 @@ :license: BSD, see LICENSE for more details. """ import re -import sys import errno -try: - from thread import allocate_lock -except ImportError: - from dummy_thread import allocate_lock from collections import deque -from itertools import imap +from jinja2._compat import text_type, string_types, implements_iterator, \ + allocate_lock, url_quote _word_split_re = re.compile(r'(\s+)') _punctuation_re = re.compile( '^(?P<lead>(?:%s)*)(?P<middle>.*?)(?P<trail>(?:%s)*)$' % ( - '|'.join(imap(re.escape, ('(', '<', '<'))), - '|'.join(imap(re.escape, ('.', ',', ')', '>', '\n', '>'))) + '|'.join(map(re.escape, ('(', '<', '<'))), + '|'.join(map(re.escape, ('.', ',', ')', '>', '\n', '>'))) ) ) _simple_email_re = re.compile(r'^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$') @@ -38,77 +34,7 @@ missing = type('MissingType', (), {'__repr__': lambda x: 'missing'})() # internal code internal_code = set() - -# concatenate a list of strings and convert them to unicode. -# unfortunately there is a bug in python 2.4 and lower that causes -# unicode.join trash the traceback. -_concat = u''.join -try: - def _test_gen_bug(): - raise TypeError(_test_gen_bug) - yield None - _concat(_test_gen_bug()) -except TypeError, _error: - if not _error.args or _error.args[0] is not _test_gen_bug: - def concat(gen): - try: - return _concat(list(gen)) - except: - # this hack is needed so that the current frame - # does not show up in the traceback. - exc_type, exc_value, tb = sys.exc_info() - raise exc_type, exc_value, tb.tb_next - else: - concat = _concat - del _test_gen_bug, _error - - -# for python 2.x we create outselves a next() function that does the -# basics without exception catching. -try: - next = next -except NameError: - def next(x): - return x.next() - - -# if this python version is unable to deal with unicode filenames -# when passed to encode we let this function encode it properly. -# This is used in a couple of places. As far as Jinja is concerned -# filenames are unicode *or* bytestrings in 2.x and unicode only in -# 3.x because compile cannot handle bytes -if sys.version_info < (3, 0): - def _encode_filename(filename): - if isinstance(filename, unicode): - return filename.encode('utf-8') - return filename -else: - def _encode_filename(filename): - assert filename is None or isinstance(filename, str), \ - 'filenames must be strings' - return filename - -from keyword import iskeyword as is_python_keyword - - -# common types. These do exist in the special types module too which however -# does not exist in IronPython out of the box. Also that way we don't have -# to deal with implementation specific stuff here -class _C(object): - def method(self): pass -def _func(): - yield None -FunctionType = type(_func) -GeneratorType = type(_func()) -MethodType = type(_C.method) -CodeType = type(_C.method.func_code) -try: - raise TypeError() -except TypeError: - _tb = sys.exc_info()[2] - TracebackType = type(_tb) - FrameType = type(_tb.tb_frame) -del _C, _tb, _func +concat = u''.join def contextfunction(f): @@ -128,7 +54,7 @@ def contextfunction(f): def evalcontextfunction(f): - """This decoraotr can be used to mark a function or method as an eval + """This decorator can be used to mark a function or method as an eval context callable. This is similar to the :func:`contextfunction` but instead of passing the context, an evaluation context object is passed. For more information about the eval context, see @@ -152,7 +78,7 @@ def environmentfunction(f): def internalcode(f): """Marks the function as internally used""" - internal_code.add(f.func_code) + internal_code.add(f.__code__) return f @@ -191,7 +117,7 @@ def clear_caches(): def import_string(import_name, silent=False): - """Imports an object based on a string. This use useful if you want to + """Imports an object based on a string. This is useful if you want to use import paths as endpoints or something similar. An import path can be specified either in dotted notation (``xml.sax.saxutils.escape``) or with a colon as object delimiter (``xml.sax.saxutils:escape``). @@ -222,7 +148,7 @@ def open_if_exists(filename, mode='rb'): """ try: return open(filename, mode) - except IOError, e: + except IOError as e: if e.errno not in (errno.ENOENT, errno.EISDIR): raise @@ -271,7 +197,7 @@ def urlize(text, trim_url_limit=None, nofollow=False): trim_url = lambda x, limit=trim_url_limit: limit is not None \ and (x[:limit] + (len(x) >=limit and '...' or '')) or x - words = _word_split_re.split(unicode(escape(text))) + words = _word_split_re.split(text_type(escape(text))) nofollow_attr = nofollow and ' rel="nofollow"' or '' for i, word in enumerate(words): match = _punctuation_re.match(word) @@ -280,6 +206,7 @@ def urlize(text, trim_url_limit=None, nofollow=False): if middle.startswith('www.') or ( '@' not in middle and not middle.startswith('http://') and + not middle.startswith('https://') and len(middle) > 0 and middle[0] in _letters + _digits and ( middle.endswith('.org') or @@ -307,7 +234,7 @@ def generate_lorem_ipsum(n=5, html=True, min=20, max=100): words = LOREM_IPSUM_WORDS.split() result = [] - for _ in xrange(n): + for _ in range(n): next_capitalized = True last_comma = last_fullstop = 0 word = None @@ -315,7 +242,7 @@ def generate_lorem_ipsum(n=5, html=True, min=20, max=100): p = [] # each paragraph contains out of 20 to 100 words. - for idx, _ in enumerate(xrange(randrange(min, max))): + for idx, _ in enumerate(range(randrange(min, max))): while True: word = choice(words) if word != last: @@ -349,6 +276,21 @@ def generate_lorem_ipsum(n=5, html=True, min=20, max=100): return Markup(u'\n'.join(u'<p>%s</p>' % escape(x) for x in result)) +def unicode_urlencode(obj, charset='utf-8'): + """URL escapes a single bytestring or unicode string with the + given charset if applicable to URL safe quoting under all rules + that need to be considered under all supported Python versions. + + If non strings are provided they are converted to their unicode + representation first. + """ + if not isinstance(obj, string_types): + obj = text_type(obj) + if isinstance(obj, text_type): + obj = obj.encode(charset) + return text_type(url_quote(obj)) + + class LRUCache(object): """A simple LRU Cache implementation.""" @@ -366,18 +308,10 @@ class LRUCache(object): # alias all queue methods for faster lookup self._popleft = self._queue.popleft self._pop = self._queue.pop - if hasattr(self._queue, 'remove'): - self._remove = self._queue.remove + self._remove = self._queue.remove self._wlock = allocate_lock() self._append = self._queue.append - def _remove(self, obj): - """Python 2.4 compatibility.""" - for idx, item in enumerate(self._queue): - if item == obj: - del self._queue[idx] - break - def __getstate__(self): return { 'capacity': self.capacity, @@ -393,7 +327,7 @@ class LRUCache(object): return (self.capacity,) def copy(self): - """Return an shallow copy of the instance.""" + """Return a shallow copy of the instance.""" rv = self.__class__(self.capacity) rv._mapping.update(self._mapping) rv._queue = deque(self._queue) @@ -410,11 +344,15 @@ class LRUCache(object): """Set `default` if the key is not in the cache otherwise leave unchanged. Return the value of this key. """ + self._wlock.acquire() try: - return self[key] - except KeyError: - self[key] = default - return default + try: + return self[key] + except KeyError: + self[key] = default + return default + finally: + self._wlock.release() def clear(self): """Clear the cache.""" @@ -443,19 +381,23 @@ class LRUCache(object): """Get an item from the cache. Moves the item up so that it has the highest priority then. - Raise an `KeyError` if it does not exist. + Raise a `KeyError` if it does not exist. """ - rv = self._mapping[key] - if self._queue[-1] != key: - try: - self._remove(key) - except ValueError: - # if something removed the key from the container - # when we read, ignore the ValueError that we would - # get otherwise. - pass - self._append(key) - return rv + self._wlock.acquire() + try: + rv = self._mapping[key] + if self._queue[-1] != key: + try: + self._remove(key) + except ValueError: + # if something removed the key from the container + # when we read, ignore the ValueError that we would + # get otherwise. + pass + self._append(key) + return rv + finally: + self._wlock.release() def __setitem__(self, key, value): """Sets the value for an item. Moves the item up so that it @@ -464,11 +406,7 @@ class LRUCache(object): self._wlock.acquire() try: if key in self._mapping: - try: - self._remove(key) - except ValueError: - # __getitem__ is not locked, it might happen - pass + self._remove(key) elif len(self._mapping) == self.capacity: del self._mapping[self._popleft()] self._append(key) @@ -478,7 +416,7 @@ class LRUCache(object): def __delitem__(self, key): """Remove an item from the cache dict. - Raise an `KeyError` if it does not exist. + Raise a `KeyError` if it does not exist. """ self._wlock.acquire() try: @@ -538,6 +476,7 @@ except ImportError: pass +@implements_iterator class Cycler(object): """A cycle helper for templates.""" @@ -556,7 +495,7 @@ class Cycler(object): """Returns the current item.""" return self.items[self.pos] - def next(self): + def __next__(self): """Goes one item ahead and returns it.""" rv = self.current self.pos = (self.pos + 1) % len(self.items) @@ -577,25 +516,5 @@ class Joiner(object): return self.sep -# try markupsafe first, if that fails go with Jinja2's bundled version -# of markupsafe. Markupsafe was previously Jinja2's implementation of -# the Markup object but was moved into a separate package in a patchleve -# release -try: - from markupsafe import Markup, escape, soft_unicode -except ImportError: - from jinja2._markupsafe import Markup, escape, soft_unicode - - -# partials -try: - from functools import partial -except ImportError: - class partial(object): - def __init__(self, _func, *args, **kwargs): - self._func = _func - self._args = args - self._kwargs = kwargs - def __call__(self, *args, **kwargs): - kwargs.update(self._kwargs) - return self._func(*(self._args + args), **kwargs) +# Imported here because that's where it was in the past +from markupsafe import Markup, escape, soft_unicode diff --git a/module/lib/jinja2/_markupsafe/__init__.py b/module/lib/markupsafe/__init__.py index ec7bd572d..275540154 100644 --- a/module/lib/jinja2/_markupsafe/__init__.py +++ b/module/lib/markupsafe/__init__.py @@ -9,7 +9,10 @@ :license: BSD, see LICENSE for more details. """ import re -from itertools import imap +import string +from collections import Mapping +from markupsafe._compat import text_type, string_types, int_types, \ + unichr, iteritems, PY2 __all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent'] @@ -19,7 +22,7 @@ _striptags_re = re.compile(r'(<!--.*?-->|<[^>]*>)') _entity_re = re.compile(r'&([^;]+);') -class Markup(unicode): +class Markup(text_type): r"""Marks a string as being safe for inclusion in HTML/XML output without needing to be escaped. This implements the `__html__` interface a couple of frameworks and web applications use. :class:`Markup` is a direct @@ -40,7 +43,7 @@ class Markup(unicode): >>> class Foo(object): ... def __html__(self): ... return '<a href="#">foo</a>' - ... + ... >>> Markup(Foo()) Markup(u'<a href="#">foo</a>') @@ -68,65 +71,66 @@ class Markup(unicode): if hasattr(base, '__html__'): base = base.__html__() if encoding is None: - return unicode.__new__(cls, base) - return unicode.__new__(cls, base, encoding, errors) + return text_type.__new__(cls, base) + return text_type.__new__(cls, base, encoding, errors) def __html__(self): return self def __add__(self, other): - if hasattr(other, '__html__') or isinstance(other, basestring): - return self.__class__(unicode(self) + unicode(escape(other))) + if isinstance(other, string_types) or hasattr(other, '__html__'): + return self.__class__(super(Markup, self).__add__(self.escape(other))) return NotImplemented def __radd__(self, other): - if hasattr(other, '__html__') or isinstance(other, basestring): - return self.__class__(unicode(escape(other)) + unicode(self)) + if hasattr(other, '__html__') or isinstance(other, string_types): + return self.escape(other).__add__(self) return NotImplemented def __mul__(self, num): - if isinstance(num, (int, long)): - return self.__class__(unicode.__mul__(self, num)) + if isinstance(num, int_types): + return self.__class__(text_type.__mul__(self, num)) return NotImplemented __rmul__ = __mul__ def __mod__(self, arg): if isinstance(arg, tuple): - arg = tuple(imap(_MarkupEscapeHelper, arg)) + arg = tuple(_MarkupEscapeHelper(x, self.escape) for x in arg) else: - arg = _MarkupEscapeHelper(arg) - return self.__class__(unicode.__mod__(self, arg)) + arg = _MarkupEscapeHelper(arg, self.escape) + return self.__class__(text_type.__mod__(self, arg)) def __repr__(self): return '%s(%s)' % ( self.__class__.__name__, - unicode.__repr__(self) + text_type.__repr__(self) ) def join(self, seq): - return self.__class__(unicode.join(self, imap(escape, seq))) - join.__doc__ = unicode.join.__doc__ + return self.__class__(text_type.join(self, map(self.escape, seq))) + join.__doc__ = text_type.join.__doc__ def split(self, *args, **kwargs): - return map(self.__class__, unicode.split(self, *args, **kwargs)) - split.__doc__ = unicode.split.__doc__ + return list(map(self.__class__, text_type.split(self, *args, **kwargs))) + split.__doc__ = text_type.split.__doc__ def rsplit(self, *args, **kwargs): - return map(self.__class__, unicode.rsplit(self, *args, **kwargs)) - rsplit.__doc__ = unicode.rsplit.__doc__ + return list(map(self.__class__, text_type.rsplit(self, *args, **kwargs))) + rsplit.__doc__ = text_type.rsplit.__doc__ def splitlines(self, *args, **kwargs): - return map(self.__class__, unicode.splitlines(self, *args, **kwargs)) - splitlines.__doc__ = unicode.splitlines.__doc__ + return list(map(self.__class__, text_type.splitlines( + self, *args, **kwargs))) + splitlines.__doc__ = text_type.splitlines.__doc__ def unescape(self): - r"""Unescape markup again into an unicode string. This also resolves + r"""Unescape markup again into an text_type string. This also resolves known HTML4 and XHTML entities: >>> Markup("Main » <em>About</em>").unescape() u'Main \xbb <em>About</em>' """ - from jinja2._markupsafe._constants import HTML_ENTITIES + from markupsafe._constants import HTML_ENTITIES def handle_match(m): name = m.group(1) if name in HTML_ENTITIES: @@ -139,10 +143,10 @@ class Markup(unicode): except ValueError: pass return u'' - return _entity_re.sub(handle_match, unicode(self)) + return _entity_re.sub(handle_match, text_type(self)) def striptags(self): - r"""Unescape markup into an unicode string and strip all tags. This + r"""Unescape markup into an text_type string and strip all tags. This also resolves known HTML4 and XHTML entities. Whitespace is normalized to one: @@ -163,11 +167,11 @@ class Markup(unicode): return cls(rv) return rv - def make_wrapper(name): - orig = getattr(unicode, name) + def make_simple_escaping_wrapper(name): + orig = getattr(text_type, name) def func(self, *args, **kwargs): - args = _escape_argspec(list(args), enumerate(args)) - _escape_argspec(kwargs, kwargs.iteritems()) + args = _escape_argspec(list(args), enumerate(args), self.escape) + _escape_argspec(kwargs, iteritems(kwargs), self.escape) return self.__class__(orig(self, *args, **kwargs)) func.__name__ = orig.__name__ func.__doc__ = orig.__doc__ @@ -177,28 +181,93 @@ class Markup(unicode): 'title', 'lower', 'upper', 'replace', 'ljust', \ 'rjust', 'lstrip', 'rstrip', 'center', 'strip', \ 'translate', 'expandtabs', 'swapcase', 'zfill': - locals()[method] = make_wrapper(method) + locals()[method] = make_simple_escaping_wrapper(method) # new in python 2.5 - if hasattr(unicode, 'partition'): - partition = make_wrapper('partition'), - rpartition = make_wrapper('rpartition') + if hasattr(text_type, 'partition'): + def partition(self, sep): + return tuple(map(self.__class__, + text_type.partition(self, self.escape(sep)))) + def rpartition(self, sep): + return tuple(map(self.__class__, + text_type.rpartition(self, self.escape(sep)))) # new in python 2.6 - if hasattr(unicode, 'format'): - format = make_wrapper('format') + if hasattr(text_type, 'format'): + def format(*args, **kwargs): + self, args = args[0], args[1:] + formatter = EscapeFormatter(self.escape) + kwargs = _MagicFormatMapping(args, kwargs) + return self.__class__(formatter.vformat(self, args, kwargs)) + + def __html_format__(self, format_spec): + if format_spec: + raise ValueError('Unsupported format specification ' + 'for Markup.') + return self # not in python 3 - if hasattr(unicode, '__getslice__'): - __getslice__ = make_wrapper('__getslice__') + if hasattr(text_type, '__getslice__'): + __getslice__ = make_simple_escaping_wrapper('__getslice__') - del method, make_wrapper + del method, make_simple_escaping_wrapper -def _escape_argspec(obj, iterable): +class _MagicFormatMapping(Mapping): + """This class implements a dummy wrapper to fix a bug in the Python + standard library for string formatting. + + See http://bugs.python.org/issue13598 for information about why + this is necessary. + """ + + def __init__(self, args, kwargs): + self._args = args + self._kwargs = kwargs + self._last_index = 0 + + def __getitem__(self, key): + if key == '': + idx = self._last_index + self._last_index += 1 + try: + return self._args[idx] + except LookupError: + pass + key = str(idx) + return self._kwargs[key] + + def __iter__(self): + return iter(self._kwargs) + + def __len__(self): + return len(self._kwargs) + + +if hasattr(text_type, 'format'): + class EscapeFormatter(string.Formatter): + + def __init__(self, escape): + self.escape = escape + + def format_field(self, value, format_spec): + if hasattr(value, '__html_format__'): + rv = value.__html_format__(format_spec) + elif hasattr(value, '__html__'): + if format_spec: + raise ValueError('No format specification allowed ' + 'when formatting an object with ' + 'its __html__ method.') + rv = value.__html__() + else: + rv = string.Formatter.format_field(self, value, format_spec) + return text_type(self.escape(rv)) + + +def _escape_argspec(obj, iterable, escape): """Helper for various string-wrapped functions.""" for key, value in iterable: - if hasattr(value, '__html__') or isinstance(value, basestring): + if hasattr(value, '__html__') or isinstance(value, string_types): obj[key] = escape(value) return obj @@ -206,13 +275,13 @@ def _escape_argspec(obj, iterable): class _MarkupEscapeHelper(object): """Helper for Markup.__mod__""" - def __init__(self, obj): + def __init__(self, obj, escape): self.obj = obj + self.escape = escape - __getitem__ = lambda s, x: _MarkupEscapeHelper(s.obj[x]) - __str__ = lambda s: str(escape(s.obj)) - __unicode__ = lambda s: unicode(escape(s.obj)) - __repr__ = lambda s: str(escape(repr(s.obj))) + __getitem__ = lambda s, x: _MarkupEscapeHelper(s.obj[x], s.escape) + __unicode__ = __str__ = lambda s: text_type(s.escape(s.obj)) + __repr__ = lambda s: str(s.escape(repr(s.obj))) __int__ = lambda s: int(s.obj) __float__ = lambda s: float(s.obj) @@ -220,6 +289,10 @@ class _MarkupEscapeHelper(object): # we have to import it down here as the speedups and native # modules imports the markup type which is define above. try: - from jinja2._markupsafe._speedups import escape, escape_silent, soft_unicode + from markupsafe._speedups import escape, escape_silent, soft_unicode except ImportError: - from jinja2._markupsafe._native import escape, escape_silent, soft_unicode + from markupsafe._native import escape, escape_silent, soft_unicode + +if not PY2: + soft_str = soft_unicode + __all__.append('soft_str') diff --git a/module/lib/markupsafe/_compat.py b/module/lib/markupsafe/_compat.py new file mode 100644 index 000000000..62e5632ad --- /dev/null +++ b/module/lib/markupsafe/_compat.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +""" + markupsafe._compat + ~~~~~~~~~~~~~~~~~~ + + Compatibility module for different Python versions. + + :copyright: (c) 2013 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +import sys + +PY2 = sys.version_info[0] == 2 + +if not PY2: + text_type = str + string_types = (str,) + unichr = chr + int_types = (int,) + iteritems = lambda x: iter(x.items()) +else: + text_type = unicode + string_types = (str, unicode) + unichr = unichr + int_types = (int, long) + iteritems = lambda x: x.iteritems() diff --git a/module/lib/jinja2/_markupsafe/_constants.py b/module/lib/markupsafe/_constants.py index 919bf03c5..919bf03c5 100644 --- a/module/lib/jinja2/_markupsafe/_constants.py +++ b/module/lib/markupsafe/_constants.py diff --git a/module/lib/jinja2/_markupsafe/_native.py b/module/lib/markupsafe/_native.py index 7b95828ec..5e83f10a1 100644 --- a/module/lib/jinja2/_markupsafe/_native.py +++ b/module/lib/markupsafe/_native.py @@ -8,7 +8,8 @@ :copyright: (c) 2010 by Armin Ronacher. :license: BSD, see LICENSE for more details. """ -from jinja2._markupsafe import Markup +from markupsafe import Markup +from markupsafe._compat import text_type def escape(s): @@ -18,7 +19,7 @@ def escape(s): """ if hasattr(s, '__html__'): return s.__html__() - return Markup(unicode(s) + return Markup(text_type(s) .replace('&', '&') .replace('>', '>') .replace('<', '<') @@ -40,6 +41,6 @@ def soft_unicode(s): """Make a string unicode if it isn't already. That way a markup string is not converted back to unicode. """ - if not isinstance(s, unicode): - s = unicode(s) + if not isinstance(s, text_type): + s = text_type(s) return s diff --git a/module/lib/markupsafe/_speedups.c b/module/lib/markupsafe/_speedups.c new file mode 100644 index 000000000..f349febf2 --- /dev/null +++ b/module/lib/markupsafe/_speedups.c @@ -0,0 +1,239 @@ +/** + * markupsafe._speedups + * ~~~~~~~~~~~~~~~~~~~~ + * + * This module implements functions for automatic escaping in C for better + * performance. + * + * :copyright: (c) 2010 by Armin Ronacher. + * :license: BSD. + */ + +#include <Python.h> + +#define ESCAPED_CHARS_TABLE_SIZE 63 +#define UNICHR(x) (PyUnicode_AS_UNICODE((PyUnicodeObject*)PyUnicode_DecodeASCII(x, strlen(x), NULL))); + +#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) +typedef int Py_ssize_t; +#define PY_SSIZE_T_MAX INT_MAX +#define PY_SSIZE_T_MIN INT_MIN +#endif + + +static PyObject* markup; +static Py_ssize_t escaped_chars_delta_len[ESCAPED_CHARS_TABLE_SIZE]; +static Py_UNICODE *escaped_chars_repl[ESCAPED_CHARS_TABLE_SIZE]; + +static int +init_constants(void) +{ + PyObject *module; + /* happing of characters to replace */ + escaped_chars_repl['"'] = UNICHR("""); + escaped_chars_repl['\''] = UNICHR("'"); + escaped_chars_repl['&'] = UNICHR("&"); + escaped_chars_repl['<'] = UNICHR("<"); + escaped_chars_repl['>'] = UNICHR(">"); + + /* lengths of those characters when replaced - 1 */ + memset(escaped_chars_delta_len, 0, sizeof (escaped_chars_delta_len)); + escaped_chars_delta_len['"'] = escaped_chars_delta_len['\''] = \ + escaped_chars_delta_len['&'] = 4; + escaped_chars_delta_len['<'] = escaped_chars_delta_len['>'] = 3; + + /* import markup type so that we can mark the return value */ + module = PyImport_ImportModule("markupsafe"); + if (!module) + return 0; + markup = PyObject_GetAttrString(module, "Markup"); + Py_DECREF(module); + + return 1; +} + +static PyObject* +escape_unicode(PyUnicodeObject *in) +{ + PyUnicodeObject *out; + Py_UNICODE *inp = PyUnicode_AS_UNICODE(in); + const Py_UNICODE *inp_end = PyUnicode_AS_UNICODE(in) + PyUnicode_GET_SIZE(in); + Py_UNICODE *next_escp; + Py_UNICODE *outp; + Py_ssize_t delta=0, erepl=0, delta_len=0; + + /* First we need to figure out how long the escaped string will be */ + while (*(inp) || inp < inp_end) { + if (*inp < ESCAPED_CHARS_TABLE_SIZE) { + delta += escaped_chars_delta_len[*inp]; + erepl += !!escaped_chars_delta_len[*inp]; + } + ++inp; + } + + /* Do we need to escape anything at all? */ + if (!erepl) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = (PyUnicodeObject*)PyUnicode_FromUnicode(NULL, PyUnicode_GET_SIZE(in) + delta); + if (!out) + return NULL; + + outp = PyUnicode_AS_UNICODE(out); + inp = PyUnicode_AS_UNICODE(in); + while (erepl-- > 0) { + /* look for the next substitution */ + next_escp = inp; + while (next_escp < inp_end) { + if (*next_escp < ESCAPED_CHARS_TABLE_SIZE && + (delta_len = escaped_chars_delta_len[*next_escp])) { + ++delta_len; + break; + } + ++next_escp; + } + + if (next_escp > inp) { + /* copy unescaped chars between inp and next_escp */ + Py_UNICODE_COPY(outp, inp, next_escp-inp); + outp += next_escp - inp; + } + + /* escape 'next_escp' */ + Py_UNICODE_COPY(outp, escaped_chars_repl[*next_escp], delta_len); + outp += delta_len; + + inp = next_escp + 1; + } + if (inp < inp_end) + Py_UNICODE_COPY(outp, inp, PyUnicode_GET_SIZE(in) - (inp - PyUnicode_AS_UNICODE(in))); + + return (PyObject*)out; +} + + +static PyObject* +escape(PyObject *self, PyObject *text) +{ + PyObject *s = NULL, *rv = NULL, *html; + + /* we don't have to escape integers, bools or floats */ + if (PyLong_CheckExact(text) || +#if PY_MAJOR_VERSION < 3 + PyInt_CheckExact(text) || +#endif + PyFloat_CheckExact(text) || PyBool_Check(text) || + text == Py_None) + return PyObject_CallFunctionObjArgs(markup, text, NULL); + + /* if the object has an __html__ method that performs the escaping */ + html = PyObject_GetAttrString(text, "__html__"); + if (html) { + rv = PyObject_CallObject(html, NULL); + Py_DECREF(html); + return rv; + } + + /* otherwise make the object unicode if it isn't, then escape */ + PyErr_Clear(); + if (!PyUnicode_Check(text)) { +#if PY_MAJOR_VERSION < 3 + PyObject *unicode = PyObject_Unicode(text); +#else + PyObject *unicode = PyObject_Str(text); +#endif + if (!unicode) + return NULL; + s = escape_unicode((PyUnicodeObject*)unicode); + Py_DECREF(unicode); + } + else + s = escape_unicode((PyUnicodeObject*)text); + + /* convert the unicode string into a markup object. */ + rv = PyObject_CallFunctionObjArgs(markup, (PyObject*)s, NULL); + Py_DECREF(s); + return rv; +} + + +static PyObject* +escape_silent(PyObject *self, PyObject *text) +{ + if (text != Py_None) + return escape(self, text); + return PyObject_CallFunctionObjArgs(markup, NULL); +} + + +static PyObject* +soft_unicode(PyObject *self, PyObject *s) +{ + if (!PyUnicode_Check(s)) +#if PY_MAJOR_VERSION < 3 + return PyObject_Unicode(s); +#else + return PyObject_Str(s); +#endif + Py_INCREF(s); + return s; +} + + +static PyMethodDef module_methods[] = { + {"escape", (PyCFunction)escape, METH_O, + "escape(s) -> markup\n\n" + "Convert the characters &, <, >, ', and \" in string s to HTML-safe\n" + "sequences. Use this if you need to display text that might contain\n" + "such characters in HTML. Marks return value as markup string."}, + {"escape_silent", (PyCFunction)escape_silent, METH_O, + "escape_silent(s) -> markup\n\n" + "Like escape but converts None to an empty string."}, + {"soft_unicode", (PyCFunction)soft_unicode, METH_O, + "soft_unicode(object) -> string\n\n" + "Make a string unicode if it isn't already. That way a markup\n" + "string is not converted back to unicode."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + +#if PY_MAJOR_VERSION < 3 + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC +init_speedups(void) +{ + if (!init_constants()) + return; + + Py_InitModule3("markupsafe._speedups", module_methods, ""); +} + +#else /* Python 3.x module initialization */ + +static struct PyModuleDef module_definition = { + PyModuleDef_HEAD_INIT, + "markupsafe._speedups", + NULL, + -1, + module_methods, + NULL, + NULL, + NULL, + NULL +}; + +PyMODINIT_FUNC +PyInit__speedups(void) +{ + if (!init_constants()) + return NULL; + + return PyModule_Create(&module_definition); +} + +#endif diff --git a/module/lib/markupsafe/tests.py b/module/lib/markupsafe/tests.py new file mode 100644 index 000000000..636993629 --- /dev/null +++ b/module/lib/markupsafe/tests.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +import gc +import sys +import unittest +from markupsafe import Markup, escape, escape_silent +from markupsafe._compat import text_type + + +class MarkupTestCase(unittest.TestCase): + + def test_adding(self): + # adding two strings should escape the unsafe one + unsafe = '<script type="application/x-some-script">alert("foo");</script>' + safe = Markup('<em>username</em>') + assert unsafe + safe == text_type(escape(unsafe)) + text_type(safe) + + def test_string_interpolation(self): + # string interpolations are safe to use too + assert Markup('<em>%s</em>') % '<bad user>' == \ + '<em><bad user></em>' + assert Markup('<em>%(username)s</em>') % { + 'username': '<bad user>' + } == '<em><bad user></em>' + + assert Markup('%i') % 3.14 == '3' + assert Markup('%.2f') % 3.14 == '3.14' + + def test_type_behavior(self): + # an escaped object is markup too + assert type(Markup('foo') + 'bar') is Markup + + # and it implements __html__ by returning itself + x = Markup("foo") + assert x.__html__() is x + + def test_html_interop(self): + # it also knows how to treat __html__ objects + class Foo(object): + def __html__(self): + return '<em>awesome</em>' + def __unicode__(self): + return 'awesome' + __str__ = __unicode__ + assert Markup(Foo()) == '<em>awesome</em>' + assert Markup('<strong>%s</strong>') % Foo() == \ + '<strong><em>awesome</em></strong>' + + def test_tuple_interpol(self): + self.assertEqual(Markup('<em>%s:%s</em>') % ( + '<foo>', + '<bar>', + ), Markup(u'<em><foo>:<bar></em>')) + + def test_dict_interpol(self): + self.assertEqual(Markup('<em>%(foo)s</em>') % { + 'foo': '<foo>', + }, Markup(u'<em><foo></em>')) + self.assertEqual(Markup('<em>%(foo)s:%(bar)s</em>') % { + 'foo': '<foo>', + 'bar': '<bar>', + }, Markup(u'<em><foo>:<bar></em>')) + + def test_escaping(self): + # escaping and unescaping + assert escape('"<>&\'') == '"<>&'' + assert Markup("<em>Foo & Bar</em>").striptags() == "Foo & Bar" + assert Markup("<test>").unescape() == "<test>" + + def test_formatting(self): + for actual, expected in ( + (Markup('%i') % 3.14, '3'), + (Markup('%.2f') % 3.14159, '3.14'), + (Markup('%s %s %s') % ('<', 123, '>'), '< 123 >'), + (Markup('<em>{awesome}</em>').format(awesome='<awesome>'), + '<em><awesome></em>'), + (Markup('{0[1][bar]}').format([0, {'bar': '<bar/>'}]), + '<bar/>'), + (Markup('{0[1][bar]}').format([0, {'bar': Markup('<bar/>')}]), + '<bar/>')): + assert actual == expected, "%r should be %r!" % (actual, expected) + + # This is new in 2.7 + if sys.version_info >= (2, 7): + def test_formatting_empty(self): + formatted = Markup('{}').format(0) + assert formatted == Markup('0') + + def test_custom_formatting(self): + class HasHTMLOnly(object): + def __html__(self): + return Markup('<foo>') + + class HasHTMLAndFormat(object): + def __html__(self): + return Markup('<foo>') + def __html_format__(self, spec): + return Markup('<FORMAT>') + + assert Markup('{0}').format(HasHTMLOnly()) == Markup('<foo>') + assert Markup('{0}').format(HasHTMLAndFormat()) == Markup('<FORMAT>') + + def test_complex_custom_formatting(self): + class User(object): + def __init__(self, id, username): + self.id = id + self.username = username + def __html_format__(self, format_spec): + if format_spec == 'link': + return Markup('<a href="/user/{0}">{1}</a>').format( + self.id, + self.__html__(), + ) + elif format_spec: + raise ValueError('Invalid format spec') + return self.__html__() + def __html__(self): + return Markup('<span class=user>{0}</span>').format(self.username) + + user = User(1, 'foo') + assert Markup('<p>User: {0:link}').format(user) == \ + Markup('<p>User: <a href="/user/1"><span class=user>foo</span></a>') + + def test_all_set(self): + import markupsafe as markup + for item in markup.__all__: + getattr(markup, item) + + def test_escape_silent(self): + assert escape_silent(None) == Markup() + assert escape(None) == Markup(None) + assert escape_silent('<foo>') == Markup(u'<foo>') + + def test_splitting(self): + self.assertEqual(Markup('a b').split(), [ + Markup('a'), + Markup('b') + ]) + self.assertEqual(Markup('a b').rsplit(), [ + Markup('a'), + Markup('b') + ]) + self.assertEqual(Markup('a\nb').splitlines(), [ + Markup('a'), + Markup('b') + ]) + + def test_mul(self): + self.assertEqual(Markup('a') * 3, Markup('aaa')) + + +class MarkupLeakTestCase(unittest.TestCase): + + def test_markup_leaks(self): + counts = set() + for count in range(20): + for item in range(1000): + escape("foo") + escape("<foo>") + escape(u"foo") + escape(u"<foo>") + counts.add(len(gc.get_objects())) + assert len(counts) == 1, 'ouch, c extension seems to leak objects' + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(MarkupTestCase)) + + # this test only tests the c extension + if not hasattr(escape, 'func_code'): + suite.addTest(unittest.makeSuite(MarkupLeakTestCase)) + + return suite + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') + +# vim:sts=4:sw=4:et: |