diff options
author | nkozlovskiy <nmk@ydb.tech> | 2023-09-29 12:24:06 +0300 |
---|---|---|
committer | nkozlovskiy <nmk@ydb.tech> | 2023-09-29 12:41:34 +0300 |
commit | e0e3e1717e3d33762ce61950504f9637a6e669ed (patch) | |
tree | bca3ff6939b10ed60c3d5c12439963a1146b9711 /contrib/python/Jinja2/py2 | |
parent | 38f2c5852db84c7b4d83adfcb009eb61541d1ccd (diff) | |
download | ydb-e0e3e1717e3d33762ce61950504f9637a6e669ed.tar.gz |
add ydb deps
Diffstat (limited to 'contrib/python/Jinja2/py2')
60 files changed, 20157 insertions, 0 deletions
diff --git a/contrib/python/Jinja2/py2/.dist-info/METADATA b/contrib/python/Jinja2/py2/.dist-info/METADATA new file mode 100644 index 0000000000..1af8df0f71 --- /dev/null +++ b/contrib/python/Jinja2/py2/.dist-info/METADATA @@ -0,0 +1,106 @@ +Metadata-Version: 2.1 +Name: Jinja2 +Version: 2.11.3 +Summary: A very fast and expressive template engine. +Home-page: https://palletsprojects.com/p/jinja/ +Author: Armin Ronacher +Author-email: armin.ronacher@active-4.com +Maintainer: Pallets +Maintainer-email: contact@palletsprojects.com +License: BSD-3-Clause +Project-URL: Documentation, https://jinja.palletsprojects.com/ +Project-URL: Code, https://github.com/pallets/jinja +Project-URL: Issue tracker, https://github.com/pallets/jinja/issues +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Web Environment +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Processing :: Markup :: HTML +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.* +Description-Content-Type: text/x-rst +Requires-Dist: MarkupSafe (>=0.23) +Provides-Extra: i18n +Requires-Dist: Babel (>=0.8) ; extra == 'i18n' + +Jinja +===== + +Jinja is a fast, expressive, extensible templating engine. Special +placeholders in the template allow writing code similar to Python +syntax. Then the template is passed data to render the final document. + +It includes: + +- Template inheritance and inclusion. +- Define and import macros within templates. +- HTML templates can use autoescaping to prevent XSS from untrusted + user input. +- A sandboxed environment can safely render untrusted templates. +- AsyncIO support for generating templates and calling async + functions. +- I18N support with Babel. +- Templates are compiled to optimized Python code just-in-time and + cached, or can be compiled ahead-of-time. +- Exceptions point to the correct line in templates to make debugging + easier. +- Extensible filters, tests, functions, and even syntax. + +Jinja's philosophy is that while application logic belongs in Python if +possible, it shouldn't make the template designer's job difficult by +restricting functionality too much. + + +Installing +---------- + +Install and update using `pip`_: + +.. code-block:: text + + $ pip install -U Jinja2 + +.. _pip: https://pip.pypa.io/en/stable/quickstart/ + + +In A Nutshell +------------- + +.. code-block:: jinja + + {% extends "base.html" %} + {% block title %}Members{% endblock %} + {% block content %} + <ul> + {% for user in users %} + <li><a href="{{ user.url }}">{{ user.username }}</a></li> + {% endfor %} + </ul> + {% endblock %} + + +Links +----- + +- Website: https://palletsprojects.com/p/jinja/ +- Documentation: https://jinja.palletsprojects.com/ +- Releases: https://pypi.org/project/Jinja2/ +- Code: https://github.com/pallets/jinja +- Issue tracker: https://github.com/pallets/jinja/issues +- Test status: https://dev.azure.com/pallets/jinja/_build +- Official chat: https://discord.gg/t6rrQZH + + diff --git a/contrib/python/Jinja2/py2/.dist-info/entry_points.txt b/contrib/python/Jinja2/py2/.dist-info/entry_points.txt new file mode 100644 index 0000000000..3619483fd4 --- /dev/null +++ b/contrib/python/Jinja2/py2/.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[babel.extractors] +jinja2 = jinja2.ext:babel_extract [i18n] + diff --git a/contrib/python/Jinja2/py2/.dist-info/top_level.txt b/contrib/python/Jinja2/py2/.dist-info/top_level.txt new file mode 100644 index 0000000000..7f7afbf3bf --- /dev/null +++ b/contrib/python/Jinja2/py2/.dist-info/top_level.txt @@ -0,0 +1 @@ +jinja2 diff --git a/contrib/python/Jinja2/py2/LICENSE.rst b/contrib/python/Jinja2/py2/LICENSE.rst new file mode 100644 index 0000000000..c37cae49ec --- /dev/null +++ b/contrib/python/Jinja2/py2/LICENSE.rst @@ -0,0 +1,28 @@ +Copyright 2007 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/python/Jinja2/py2/README.rst b/contrib/python/Jinja2/py2/README.rst new file mode 100644 index 0000000000..060b19efee --- /dev/null +++ b/contrib/python/Jinja2/py2/README.rst @@ -0,0 +1,66 @@ +Jinja +===== + +Jinja is a fast, expressive, extensible templating engine. Special +placeholders in the template allow writing code similar to Python +syntax. Then the template is passed data to render the final document. + +It includes: + +- Template inheritance and inclusion. +- Define and import macros within templates. +- HTML templates can use autoescaping to prevent XSS from untrusted + user input. +- A sandboxed environment can safely render untrusted templates. +- AsyncIO support for generating templates and calling async + functions. +- I18N support with Babel. +- Templates are compiled to optimized Python code just-in-time and + cached, or can be compiled ahead-of-time. +- Exceptions point to the correct line in templates to make debugging + easier. +- Extensible filters, tests, functions, and even syntax. + +Jinja's philosophy is that while application logic belongs in Python if +possible, it shouldn't make the template designer's job difficult by +restricting functionality too much. + + +Installing +---------- + +Install and update using `pip`_: + +.. code-block:: text + + $ pip install -U Jinja2 + +.. _pip: https://pip.pypa.io/en/stable/quickstart/ + + +In A Nutshell +------------- + +.. code-block:: jinja + + {% extends "base.html" %} + {% block title %}Members{% endblock %} + {% block content %} + <ul> + {% for user in users %} + <li><a href="{{ user.url }}">{{ user.username }}</a></li> + {% endfor %} + </ul> + {% endblock %} + + +Links +----- + +- Website: https://palletsprojects.com/p/jinja/ +- Documentation: https://jinja.palletsprojects.com/ +- Releases: https://pypi.org/project/Jinja2/ +- Code: https://github.com/pallets/jinja +- Issue tracker: https://github.com/pallets/jinja/issues +- Test status: https://dev.azure.com/pallets/jinja/_build +- Official chat: https://discord.gg/t6rrQZH diff --git a/contrib/python/Jinja2/py2/jinja2/__init__.py b/contrib/python/Jinja2/py2/jinja2/__init__.py new file mode 100644 index 0000000000..b444e9a841 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/__init__.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +"""Jinja is a template engine written in pure Python. It provides a +non-XML syntax that supports inline expressions and an optional +sandboxed environment. +""" +from markupsafe import escape +from markupsafe import Markup + +from .bccache import BytecodeCache +from .bccache import FileSystemBytecodeCache +from .bccache import MemcachedBytecodeCache +from .environment import Environment +from .environment import Template +from .exceptions import TemplateAssertionError +from .exceptions import TemplateError +from .exceptions import TemplateNotFound +from .exceptions import TemplateRuntimeError +from .exceptions import TemplatesNotFound +from .exceptions import TemplateSyntaxError +from .exceptions import UndefinedError +from .filters import contextfilter +from .filters import environmentfilter +from .filters import evalcontextfilter +from .loaders import BaseLoader +from .loaders import ChoiceLoader +from .loaders import DictLoader +from .loaders import FileSystemLoader +from .loaders import FunctionLoader +from .loaders import ModuleLoader +from .loaders import PackageLoader +from .loaders import PrefixLoader +from .loaders import ResourceLoader +from .runtime import ChainableUndefined +from .runtime import DebugUndefined +from .runtime import make_logging_undefined +from .runtime import StrictUndefined +from .runtime import Undefined +from .utils import clear_caches +from .utils import contextfunction +from .utils import environmentfunction +from .utils import evalcontextfunction +from .utils import is_undefined +from .utils import select_autoescape + +__version__ = "2.11.3" diff --git a/contrib/python/Jinja2/py2/jinja2/_compat.py b/contrib/python/Jinja2/py2/jinja2/_compat.py new file mode 100644 index 0000000000..1f044954a0 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/_compat.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import marshal +import sys + +PY2 = sys.version_info[0] == 2 +PYPY = hasattr(sys, "pypy_translation_info") +_identity = lambda x: x + +if not PY2: + unichr = chr + range_type = range + text_type = str + string_types = (str,) + integer_types = (int,) + + iterkeys = lambda d: iter(d.keys()) + itervalues = lambda d: iter(d.values()) + iteritems = lambda d: iter(d.items()) + + import pickle + from io import BytesIO, StringIO + + NativeStringIO = StringIO + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + + ifilter = filter + imap = map + izip = zip + intern = sys.intern + + implements_iterator = _identity + implements_to_string = _identity + encode_filename = _identity + + marshal_dump = marshal.dump + marshal_load = marshal.load + +else: + unichr = unichr + text_type = unicode + range_type = xrange + string_types = (str, unicode) + integer_types = (int, long) + + iterkeys = lambda d: d.iterkeys() + itervalues = lambda d: d.itervalues() + iteritems = lambda d: d.iteritems() + + import cPickle as pickle + from cStringIO import StringIO as BytesIO, StringIO + + NativeStringIO = BytesIO + + exec("def reraise(tp, value, tb=None):\n raise tp, value, tb") + + from itertools import imap, izip, ifilter + + intern = intern + + def implements_iterator(cls): + cls.next = cls.__next__ + del cls.__next__ + return cls + + def implements_to_string(cls): + cls.__unicode__ = cls.__str__ + cls.__str__ = lambda x: x.__unicode__().encode("utf-8") + return cls + + def encode_filename(filename): + if isinstance(filename, unicode): + return filename.encode("utf-8") + return filename + + def marshal_dump(code, f): + if isinstance(f, file): + marshal.dump(code, f) + else: + f.write(marshal.dumps(code)) + + def marshal_load(f): + if isinstance(f, file): + return marshal.load(f) + return marshal.loads(f.read()) + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a + # dummy metaclass for one level of class instantiation that replaces + # itself with the actual metaclass. + class metaclass(type): + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + + return type.__new__(metaclass, "temporary_class", (), {}) + + +try: + from urllib.parse import quote_from_bytes as url_quote +except ImportError: + from urllib import quote as url_quote + + +try: + from collections import abc +except ImportError: + import collections as abc + + +try: + from os import fspath +except ImportError: + try: + from pathlib import PurePath + except ImportError: + PurePath = None + + def fspath(path): + if hasattr(path, "__fspath__"): + return path.__fspath__() + + # Python 3.5 doesn't have __fspath__ yet, use str. + if PurePath is not None and isinstance(path, PurePath): + return str(path) + + return path diff --git a/contrib/python/Jinja2/py2/jinja2/_identifier.py b/contrib/python/Jinja2/py2/jinja2/_identifier.py new file mode 100644 index 0000000000..224d5449d1 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/_identifier.py @@ -0,0 +1,6 @@ +import re + +# generated by scripts/generate_identifier_pattern.py +pattern = re.compile( + r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏_𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+" # noqa: B950 +) diff --git a/contrib/python/Jinja2/py2/jinja2/bccache.py b/contrib/python/Jinja2/py2/jinja2/bccache.py new file mode 100644 index 0000000000..9c0661030f --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/bccache.py @@ -0,0 +1,350 @@ +# -*- coding: utf-8 -*- +"""The optional bytecode cache system. This is useful if you have very +complex template situations and the compilation of all those templates +slows down your application too much. + +Situations where this is useful are often forking web applications that +are initialized on the first request. +""" +import errno +import fnmatch +import os +import stat +import sys +import tempfile +from hashlib import sha1 +from os import listdir +from os import path + +from ._compat import BytesIO +from ._compat import marshal_dump +from ._compat import marshal_load +from ._compat import pickle +from ._compat import text_type +from .utils import open_if_exists + +bc_version = 4 +# Magic bytes to identify Jinja bytecode cache files. Contains the +# Python major and minor version to avoid loading incompatible bytecode +# if a project upgrades its Python version. +bc_magic = ( + b"j2" + + pickle.dumps(bc_version, 2) + + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2) +) + + +class Bucket(object): + """Buckets are used to store the bytecode for one template. It's created + and initialized by the bytecode cache and passed to the loading functions. + + The buckets get an internal checksum from the cache assigned and use this + to automatically reject outdated cache material. Individual bytecode + cache subclasses don't have to care about cache invalidation. + """ + + def __init__(self, environment, key, checksum): + self.environment = environment + self.key = key + self.checksum = checksum + self.reset() + + def reset(self): + """Resets the bucket (unloads the bytecode).""" + self.code = None + + def load_bytecode(self, f): + """Loads bytecode from a file or file like object.""" + # make sure the magic header is correct + magic = f.read(len(bc_magic)) + if magic != bc_magic: + self.reset() + return + # the source code of the file changed, we need to reload + checksum = pickle.load(f) + if self.checksum != checksum: + self.reset() + return + # if marshal_load fails then we need to reload + try: + self.code = marshal_load(f) + except (EOFError, ValueError, TypeError): + self.reset() + return + + def write_bytecode(self, f): + """Dump the bytecode into the file or file like object passed.""" + if self.code is None: + raise TypeError("can't write empty bucket") + f.write(bc_magic) + pickle.dump(self.checksum, f, 2) + marshal_dump(self.code, f) + + def bytecode_from_string(self, string): + """Load bytecode from a string.""" + self.load_bytecode(BytesIO(string)) + + def bytecode_to_string(self): + """Return the bytecode as string.""" + out = BytesIO() + self.write_bytecode(out) + return out.getvalue() + + +class BytecodeCache(object): + """To implement your own bytecode cache you have to subclass this class + and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of + these methods are passed a :class:`~jinja2.bccache.Bucket`. + + A very basic bytecode cache that saves the bytecode on the file system:: + + from os import path + + class MyCache(BytecodeCache): + + def __init__(self, directory): + self.directory = directory + + def load_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + if path.exists(filename): + with open(filename, 'rb') as f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + with open(filename, 'wb') as f: + bucket.write_bytecode(f) + + A more advanced version of a filesystem based bytecode cache is part of + Jinja. + """ + + def load_bytecode(self, bucket): + """Subclasses have to override this method to load bytecode into a + bucket. If they are not able to find code in the cache for the + bucket, it must not do anything. + """ + raise NotImplementedError() + + def dump_bytecode(self, bucket): + """Subclasses have to override this method to write the bytecode + from a bucket back to the cache. If it unable to do so it must not + fail silently but raise an exception. + """ + raise NotImplementedError() + + def clear(self): + """Clears the cache. This method is not used by Jinja but should be + implemented to allow applications to clear the bytecode cache used + by a particular environment. + """ + + def get_cache_key(self, name, filename=None): + """Returns the unique hash key for this template name.""" + hash = sha1(name.encode("utf-8")) + if filename is not None: + filename = "|" + filename + if isinstance(filename, text_type): + filename = filename.encode("utf-8") + hash.update(filename) + return hash.hexdigest() + + def get_source_checksum(self, source): + """Returns a checksum for the source.""" + return sha1(source.encode("utf-8")).hexdigest() + + def get_bucket(self, environment, name, filename, source): + """Return a cache bucket for the given template. All arguments are + mandatory but filename may be `None`. + """ + key = self.get_cache_key(name, filename) + checksum = self.get_source_checksum(source) + bucket = Bucket(environment, key, checksum) + self.load_bytecode(bucket) + return bucket + + def set_bucket(self, bucket): + """Put the bucket into the cache.""" + self.dump_bytecode(bucket) + + +class FileSystemBytecodeCache(BytecodeCache): + """A bytecode cache that stores bytecode on the filesystem. It accepts + two arguments: The directory where the cache items are stored and a + pattern string that is used to build the filename. + + If no directory is specified a default cache directory is selected. On + Windows the user's temp directory is used, on UNIX systems a directory + is created for the user in the system temp directory. + + The pattern can be used to have multiple separate caches operate on the + same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` + is replaced with the cache key. + + >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') + + This bytecode cache supports clearing of the cache using the clear method. + """ + + def __init__(self, directory=None, pattern="__jinja2_%s.cache"): + if directory is None: + directory = self._get_default_cache_dir() + self.directory = directory + self.pattern = pattern + + def _get_default_cache_dir(self): + def _unsafe_dir(): + raise RuntimeError( + "Cannot determine safe temp directory. You " + "need to explicitly provide one." + ) + + tmpdir = tempfile.gettempdir() + + # On windows the temporary directory is used specific unless + # explicitly forced otherwise. We can just use that. + if os.name == "nt": + return tmpdir + if not hasattr(os, "getuid"): + _unsafe_dir() + + dirname = "_jinja2-cache-%d" % os.getuid() + actual_dir = os.path.join(tmpdir, dirname) + + try: + os.mkdir(actual_dir, stat.S_IRWXU) + except OSError as e: + if e.errno != errno.EEXIST: + raise + try: + os.chmod(actual_dir, stat.S_IRWXU) + actual_dir_stat = os.lstat(actual_dir) + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): + _unsafe_dir() + except OSError as e: + if e.errno != errno.EEXIST: + raise + + actual_dir_stat = os.lstat(actual_dir) + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): + _unsafe_dir() + + return actual_dir + + def _get_cache_filename(self, bucket): + return path.join(self.directory, self.pattern % bucket.key) + + def load_bytecode(self, bucket): + f = open_if_exists(self._get_cache_filename(bucket), "rb") + if f is not None: + try: + bucket.load_bytecode(f) + finally: + f.close() + + def dump_bytecode(self, bucket): + f = open(self._get_cache_filename(bucket), "wb") + try: + bucket.write_bytecode(f) + finally: + f.close() + + def clear(self): + # imported lazily here because google app-engine doesn't support + # write access on the file system and the function does not exist + # normally. + from os import remove + + files = fnmatch.filter(listdir(self.directory), self.pattern % "*") + for filename in files: + try: + remove(path.join(self.directory, filename)) + except OSError: + pass + + +class MemcachedBytecodeCache(BytecodeCache): + """This class implements a bytecode cache that uses a memcache cache for + storing the information. It does not enforce a specific memcache library + (tummy's memcache or cmemcache) but will accept any class that provides + the minimal interface required. + + Libraries compatible with this class: + + - `cachelib <https://github.com/pallets/cachelib>`_ + - `python-memcached <https://pypi.org/project/python-memcached/>`_ + + (Unfortunately the django cache interface is not compatible because it + does not support storing binary data, only unicode. You can however pass + the underlying cache client to the bytecode cache which is available + as `django.core.cache.cache._client`.) + + The minimal interface for the client passed to the constructor is this: + + .. class:: MinimalClientInterface + + .. method:: set(key, value[, timeout]) + + Stores the bytecode in the cache. `value` is a string and + `timeout` the timeout of the key. If timeout is not provided + a default timeout or no timeout should be assumed, if it's + provided it's an integer with the number of seconds the cache + item should exist. + + .. method:: get(key) + + Returns the value for the cache key. If the item does not + exist in the cache the return value must be `None`. + + The other arguments to the constructor are the prefix for all keys that + is added before the actual cache key and the timeout for the bytecode in + the cache system. We recommend a high (or no) timeout. + + This bytecode cache does not support clearing of used items in the cache. + The clear method is a no-operation function. + + .. versionadded:: 2.7 + Added support for ignoring memcache errors through the + `ignore_memcache_errors` parameter. + """ + + def __init__( + self, + client, + prefix="jinja2/bytecode/", + timeout=None, + ignore_memcache_errors=True, + ): + self.client = client + self.prefix = prefix + self.timeout = timeout + self.ignore_memcache_errors = ignore_memcache_errors + + def load_bytecode(self, bucket): + try: + code = self.client.get(self.prefix + bucket.key) + except Exception: + if not self.ignore_memcache_errors: + raise + code = None + if code is not None: + bucket.bytecode_from_string(code) + + def dump_bytecode(self, bucket): + args = (self.prefix + bucket.key, bucket.bytecode_to_string()) + if self.timeout is not None: + args += (self.timeout,) + try: + self.client.set(*args) + except Exception: + if not self.ignore_memcache_errors: + raise diff --git a/contrib/python/Jinja2/py2/jinja2/compiler.py b/contrib/python/Jinja2/py2/jinja2/compiler.py new file mode 100644 index 0000000000..63297b42c3 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/compiler.py @@ -0,0 +1,1843 @@ +# -*- coding: utf-8 -*- +"""Compiles nodes from the parser into Python code.""" +from collections import namedtuple +from functools import update_wrapper +from itertools import chain +from keyword import iskeyword as is_python_keyword + +from markupsafe import escape +from markupsafe import Markup + +from . import nodes +from ._compat import imap +from ._compat import iteritems +from ._compat import izip +from ._compat import NativeStringIO +from ._compat import range_type +from ._compat import string_types +from ._compat import text_type +from .exceptions import TemplateAssertionError +from .idtracking import Symbols +from .idtracking import VAR_LOAD_ALIAS +from .idtracking import VAR_LOAD_PARAMETER +from .idtracking import VAR_LOAD_RESOLVE +from .idtracking import VAR_LOAD_UNDEFINED +from .nodes import EvalContext +from .optimizer import Optimizer +from .utils import concat +from .visitor import NodeVisitor + +operators = { + "eq": "==", + "ne": "!=", + "gt": ">", + "gteq": ">=", + "lt": "<", + "lteq": "<=", + "in": "in", + "notin": "not in", +} + +# what method to iterate over items do we want to use for dict iteration +# in generated code? on 2.x let's go with iteritems, on 3.x with items +if hasattr(dict, "iteritems"): + dict_item_iter = "iteritems" +else: + dict_item_iter = "items" + +code_features = ["division"] + +# does this python version support generator stops? (PEP 0479) +try: + exec("from __future__ import generator_stop") + code_features.append("generator_stop") +except SyntaxError: + pass + +# does this python version support yield from? +try: + exec("def f(): yield from x()") +except SyntaxError: + supports_yield_from = False +else: + supports_yield_from = True + + +def optimizeconst(f): + def new_func(self, node, frame, **kwargs): + # Only optimize if the frame is not volatile + if self.optimized and not frame.eval_ctx.volatile: + new_node = self.optimizer.visit(node, frame.eval_ctx) + if new_node != node: + return self.visit(new_node, frame) + return f(self, node, frame, **kwargs) + + return update_wrapper(new_func, f) + + +def generate( + node, environment, name, filename, stream=None, defer_init=False, optimized=True +): + """Generate the python source for a node tree.""" + if not isinstance(node, nodes.Template): + raise TypeError("Can't compile non template nodes") + generator = environment.code_generator_class( + environment, name, filename, stream, defer_init, optimized + ) + generator.visit(node) + if stream is None: + return generator.stream.getvalue() + + +def has_safe_repr(value): + """Does the node have a safe representation?""" + if value is None or value is NotImplemented or value is Ellipsis: + return True + if type(value) in (bool, int, float, complex, range_type, Markup) + string_types: + return True + if type(value) in (tuple, list, set, frozenset): + for item in value: + if not has_safe_repr(item): + return False + return True + elif type(value) is dict: + for key, value in iteritems(value): + if not has_safe_repr(key): + return False + if not has_safe_repr(value): + return False + return True + return False + + +def find_undeclared(nodes, names): + """Check if the names passed are accessed undeclared. The return value + is a set of all the undeclared names from the sequence of names found. + """ + visitor = UndeclaredNameVisitor(names) + try: + for node in nodes: + visitor.visit(node) + except VisitorExit: + pass + return visitor.undeclared + + +class MacroRef(object): + def __init__(self, node): + self.node = node + self.accesses_caller = False + self.accesses_kwargs = False + self.accesses_varargs = False + + +class Frame(object): + """Holds compile time information for us.""" + + def __init__(self, eval_ctx, parent=None, level=None): + self.eval_ctx = eval_ctx + self.symbols = Symbols(parent and parent.symbols or None, level=level) + + # a toplevel frame is the root + soft frames such as if conditions. + self.toplevel = False + + # the root frame is basically just the outermost frame, so no if + # conditions. This information is used to optimize inheritance + # situations. + self.rootlevel = False + + # in some dynamic inheritance situations the compiler needs to add + # write tests around output statements. + self.require_output_check = parent and parent.require_output_check + + # inside some tags we are using a buffer rather than yield statements. + # this for example affects {% filter %} or {% macro %}. If a frame + # is buffered this variable points to the name of the list used as + # buffer. + self.buffer = None + + # the name of the block we're in, otherwise None. + self.block = parent and parent.block or None + + # the parent of this frame + self.parent = parent + + if parent is not None: + self.buffer = parent.buffer + + def copy(self): + """Create a copy of the current one.""" + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.symbols = self.symbols.copy() + return rv + + def inner(self, isolated=False): + """Return an inner frame.""" + if isolated: + return Frame(self.eval_ctx, level=self.symbols.level + 1) + return Frame(self.eval_ctx, self) + + def soft(self): + """Return a soft frame. A soft frame may not be modified as + standalone thing as it shares the resources with the frame it + was created of, but it's not a rootlevel frame any longer. + + This is only used to implement if-statements. + """ + rv = self.copy() + rv.rootlevel = False + return rv + + __copy__ = copy + + +class VisitorExit(RuntimeError): + """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" + + +class DependencyFinderVisitor(NodeVisitor): + """A visitor that collects filter and test calls.""" + + def __init__(self): + self.filters = set() + self.tests = set() + + def visit_Filter(self, node): + self.generic_visit(node) + self.filters.add(node.name) + + def visit_Test(self, node): + self.generic_visit(node) + self.tests.add(node.name) + + def visit_Block(self, node): + """Stop visiting at blocks.""" + + +class UndeclaredNameVisitor(NodeVisitor): + """A visitor that checks if a name is accessed without being + declared. This is different from the frame visitor as it will + not stop at closure frames. + """ + + def __init__(self, names): + self.names = set(names) + self.undeclared = set() + + def visit_Name(self, node): + if node.ctx == "load" and node.name in self.names: + self.undeclared.add(node.name) + if self.undeclared == self.names: + raise VisitorExit() + else: + self.names.discard(node.name) + + def visit_Block(self, node): + """Stop visiting a blocks.""" + + +class CompilerExit(Exception): + """Raised if the compiler encountered a situation where it just + doesn't make sense to further process the code. Any block that + raises such an exception is not further processed. + """ + + +class CodeGenerator(NodeVisitor): + def __init__( + self, environment, name, filename, stream=None, defer_init=False, optimized=True + ): + if stream is None: + stream = NativeStringIO() + self.environment = environment + self.name = name + self.filename = filename + self.stream = stream + self.created_block_context = False + self.defer_init = defer_init + self.optimized = optimized + if optimized: + self.optimizer = Optimizer(environment) + + # aliases for imports + self.import_aliases = {} + + # a registry for all blocks. Because blocks are moved out + # into the global python scope they are registered here + self.blocks = {} + + # the number of extends statements so far + self.extends_so_far = 0 + + # some templates have a rootlevel extends. In this case we + # can safely assume that we're a child template and do some + # more optimizations. + self.has_known_extends = False + + # the current line number + self.code_lineno = 1 + + # registry of all filters and tests (global, not block local) + self.tests = {} + self.filters = {} + + # the debug information + self.debug_info = [] + self._write_debug_info = None + + # the number of new lines before the next write() + self._new_lines = 0 + + # the line number of the last written statement + self._last_line = 0 + + # true if nothing was written so far. + self._first_write = True + + # used by the `temporary_identifier` method to get new + # unique, temporary identifier + self._last_identifier = 0 + + # the current indentation + self._indentation = 0 + + # Tracks toplevel assignments + self._assign_stack = [] + + # Tracks parameter definition blocks + self._param_def_block = [] + + # Tracks the current context. + self._context_reference_stack = ["context"] + + # -- Various compilation helpers + + def fail(self, msg, lineno): + """Fail with a :exc:`TemplateAssertionError`.""" + raise TemplateAssertionError(msg, lineno, self.name, self.filename) + + def temporary_identifier(self): + """Get a new unique identifier.""" + self._last_identifier += 1 + return "t_%d" % self._last_identifier + + def buffer(self, frame): + """Enable buffering for the frame from that point onwards.""" + frame.buffer = self.temporary_identifier() + self.writeline("%s = []" % frame.buffer) + + def return_buffer_contents(self, frame, force_unescaped=False): + """Return the buffer contents of the frame.""" + if not force_unescaped: + if frame.eval_ctx.volatile: + self.writeline("if context.eval_ctx.autoescape:") + self.indent() + self.writeline("return Markup(concat(%s))" % frame.buffer) + self.outdent() + self.writeline("else:") + self.indent() + self.writeline("return concat(%s)" % frame.buffer) + self.outdent() + return + elif frame.eval_ctx.autoescape: + self.writeline("return Markup(concat(%s))" % frame.buffer) + return + self.writeline("return concat(%s)" % frame.buffer) + + def indent(self): + """Indent by one.""" + self._indentation += 1 + + def outdent(self, step=1): + """Outdent by step.""" + self._indentation -= step + + def start_write(self, frame, node=None): + """Yield or write into the frame buffer.""" + if frame.buffer is None: + self.writeline("yield ", node) + else: + self.writeline("%s.append(" % frame.buffer, node) + + def end_write(self, frame): + """End the writing process started by `start_write`.""" + if frame.buffer is not None: + self.write(")") + + def simple_write(self, s, frame, node=None): + """Simple shortcut for start_write + write + end_write.""" + self.start_write(frame, node) + self.write(s) + self.end_write(frame) + + def blockvisit(self, nodes, frame): + """Visit a list of nodes as block in a frame. If the current frame + is no buffer a dummy ``if 0: yield None`` is written automatically. + """ + try: + self.writeline("pass") + for node in nodes: + self.visit(node, frame) + except CompilerExit: + pass + + def write(self, x): + """Write a string into the output stream.""" + if self._new_lines: + if not self._first_write: + self.stream.write("\n" * self._new_lines) + self.code_lineno += self._new_lines + if self._write_debug_info is not None: + self.debug_info.append((self._write_debug_info, self.code_lineno)) + self._write_debug_info = None + self._first_write = False + self.stream.write(" " * self._indentation) + self._new_lines = 0 + self.stream.write(x) + + def writeline(self, x, node=None, extra=0): + """Combination of newline and write.""" + self.newline(node, extra) + self.write(x) + + def newline(self, node=None, extra=0): + """Add one or more newlines before the next write.""" + self._new_lines = max(self._new_lines, 1 + extra) + if node is not None and node.lineno != self._last_line: + self._write_debug_info = node.lineno + self._last_line = node.lineno + + def signature(self, node, frame, extra_kwargs=None): + """Writes a function call to the stream for the current node. + A leading comma is added automatically. The extra keyword + arguments may not include python keywords otherwise a syntax + error could occur. The extra keyword arguments should be given + as python dict. + """ + # if any of the given keyword arguments is a python keyword + # we have to make sure that no invalid call is created. + kwarg_workaround = False + for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()): + if is_python_keyword(kwarg): + kwarg_workaround = True + break + + for arg in node.args: + self.write(", ") + self.visit(arg, frame) + + if not kwarg_workaround: + for kwarg in node.kwargs: + self.write(", ") + self.visit(kwarg, frame) + if extra_kwargs is not None: + for key, value in iteritems(extra_kwargs): + self.write(", %s=%s" % (key, value)) + if node.dyn_args: + self.write(", *") + self.visit(node.dyn_args, frame) + + if kwarg_workaround: + if node.dyn_kwargs is not None: + self.write(", **dict({") + else: + self.write(", **{") + for kwarg in node.kwargs: + self.write("%r: " % kwarg.key) + self.visit(kwarg.value, frame) + self.write(", ") + if extra_kwargs is not None: + for key, value in iteritems(extra_kwargs): + self.write("%r: %s, " % (key, value)) + if node.dyn_kwargs is not None: + self.write("}, **") + self.visit(node.dyn_kwargs, frame) + self.write(")") + else: + self.write("}") + + elif node.dyn_kwargs is not None: + self.write(", **") + self.visit(node.dyn_kwargs, frame) + + def pull_dependencies(self, nodes): + """Pull all the dependencies.""" + visitor = DependencyFinderVisitor() + for node in nodes: + visitor.visit(node) + for dependency in "filters", "tests": + mapping = getattr(self, dependency) + for name in getattr(visitor, dependency): + if name not in mapping: + mapping[name] = self.temporary_identifier() + self.writeline( + "%s = environment.%s[%r]" % (mapping[name], dependency, name) + ) + + def enter_frame(self, frame): + undefs = [] + for target, (action, param) in iteritems(frame.symbols.loads): + if action == VAR_LOAD_PARAMETER: + pass + elif action == VAR_LOAD_RESOLVE: + self.writeline("%s = %s(%r)" % (target, self.get_resolve_func(), param)) + elif action == VAR_LOAD_ALIAS: + self.writeline("%s = %s" % (target, param)) + elif action == VAR_LOAD_UNDEFINED: + undefs.append(target) + else: + raise NotImplementedError("unknown load instruction") + if undefs: + self.writeline("%s = missing" % " = ".join(undefs)) + + def leave_frame(self, frame, with_python_scope=False): + if not with_python_scope: + undefs = [] + for target, _ in iteritems(frame.symbols.loads): + undefs.append(target) + if undefs: + self.writeline("%s = missing" % " = ".join(undefs)) + + def func(self, name): + if self.environment.is_async: + return "async def %s" % name + return "def %s" % name + + def macro_body(self, node, frame): + """Dump the function def of a macro or call block.""" + frame = frame.inner() + frame.symbols.analyze_node(node) + macro_ref = MacroRef(node) + + explicit_caller = None + skip_special_params = set() + args = [] + for idx, arg in enumerate(node.args): + if arg.name == "caller": + explicit_caller = idx + if arg.name in ("kwargs", "varargs"): + skip_special_params.add(arg.name) + args.append(frame.symbols.ref(arg.name)) + + undeclared = find_undeclared(node.body, ("caller", "kwargs", "varargs")) + + if "caller" in undeclared: + # In older Jinja versions there was a bug that allowed caller + # to retain the special behavior even if it was mentioned in + # the argument list. However thankfully this was only really + # working if it was the last argument. So we are explicitly + # checking this now and error out if it is anywhere else in + # the argument list. + if explicit_caller is not None: + try: + node.defaults[explicit_caller - len(node.args)] + except IndexError: + self.fail( + "When defining macros or call blocks the " + 'special "caller" argument must be omitted ' + "or be given a default.", + node.lineno, + ) + else: + args.append(frame.symbols.declare_parameter("caller")) + macro_ref.accesses_caller = True + if "kwargs" in undeclared and "kwargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("kwargs")) + macro_ref.accesses_kwargs = True + if "varargs" in undeclared and "varargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("varargs")) + macro_ref.accesses_varargs = True + + # macros are delayed, they never require output checks + frame.require_output_check = False + frame.symbols.analyze_node(node) + self.writeline("%s(%s):" % (self.func("macro"), ", ".join(args)), node) + self.indent() + + self.buffer(frame) + self.enter_frame(frame) + + self.push_parameter_definitions(frame) + for idx, arg in enumerate(node.args): + ref = frame.symbols.ref(arg.name) + self.writeline("if %s is missing:" % ref) + self.indent() + try: + default = node.defaults[idx - len(node.args)] + except IndexError: + self.writeline( + "%s = undefined(%r, name=%r)" + % (ref, "parameter %r was not provided" % arg.name, arg.name) + ) + else: + self.writeline("%s = " % ref) + self.visit(default, frame) + self.mark_parameter_stored(ref) + self.outdent() + self.pop_parameter_definitions() + + self.blockvisit(node.body, frame) + self.return_buffer_contents(frame, force_unescaped=True) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + return frame, macro_ref + + def macro_def(self, macro_ref, frame): + """Dump the macro definition for the def created by macro_body.""" + arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args) + name = getattr(macro_ref.node, "name", None) + if len(macro_ref.node.args) == 1: + arg_tuple += "," + self.write( + "Macro(environment, macro, %r, (%s), %r, %r, %r, " + "context.eval_ctx.autoescape)" + % ( + name, + arg_tuple, + macro_ref.accesses_kwargs, + macro_ref.accesses_varargs, + macro_ref.accesses_caller, + ) + ) + + def position(self, node): + """Return a human readable position for the node.""" + rv = "line %d" % node.lineno + if self.name is not None: + rv += " in " + repr(self.name) + return rv + + def dump_local_context(self, frame): + return "{%s}" % ", ".join( + "%r: %s" % (name, target) + for name, target in iteritems(frame.symbols.dump_stores()) + ) + + def write_commons(self): + """Writes a common preamble that is used by root and block functions. + Primarily this sets up common local helpers and enforces a generator + through a dead branch. + """ + self.writeline("resolve = context.resolve_or_missing") + self.writeline("undefined = environment.undefined") + # always use the standard Undefined class for the implicit else of + # conditional expressions + self.writeline("cond_expr_undefined = Undefined") + self.writeline("if 0: yield None") + + def push_parameter_definitions(self, frame): + """Pushes all parameter targets from the given frame into a local + stack that permits tracking of yet to be assigned parameters. In + particular this enables the optimization from `visit_Name` to skip + undefined expressions for parameters in macros as macros can reference + otherwise unbound parameters. + """ + self._param_def_block.append(frame.symbols.dump_param_targets()) + + def pop_parameter_definitions(self): + """Pops the current parameter definitions set.""" + self._param_def_block.pop() + + def mark_parameter_stored(self, target): + """Marks a parameter in the current parameter definitions as stored. + This will skip the enforced undefined checks. + """ + if self._param_def_block: + self._param_def_block[-1].discard(target) + + def push_context_reference(self, target): + self._context_reference_stack.append(target) + + def pop_context_reference(self): + self._context_reference_stack.pop() + + def get_context_ref(self): + return self._context_reference_stack[-1] + + def get_resolve_func(self): + target = self._context_reference_stack[-1] + if target == "context": + return "resolve" + return "%s.resolve" % target + + def derive_context(self, frame): + return "%s.derived(%s)" % ( + self.get_context_ref(), + self.dump_local_context(frame), + ) + + def parameter_is_undeclared(self, target): + """Checks if a given target is an undeclared parameter.""" + if not self._param_def_block: + return False + return target in self._param_def_block[-1] + + def push_assign_tracking(self): + """Pushes a new layer for assignment tracking.""" + self._assign_stack.append(set()) + + def pop_assign_tracking(self, frame): + """Pops the topmost level for assignment tracking and updates the + context variables if necessary. + """ + vars = self._assign_stack.pop() + if not frame.toplevel or not vars: + return + public_names = [x for x in vars if x[:1] != "_"] + if len(vars) == 1: + name = next(iter(vars)) + ref = frame.symbols.ref(name) + self.writeline("context.vars[%r] = %s" % (name, ref)) + else: + self.writeline("context.vars.update({") + for idx, name in enumerate(vars): + if idx: + self.write(", ") + ref = frame.symbols.ref(name) + self.write("%r: %s" % (name, ref)) + self.write("})") + if public_names: + if len(public_names) == 1: + self.writeline("context.exported_vars.add(%r)" % public_names[0]) + else: + self.writeline( + "context.exported_vars.update((%s))" + % ", ".join(imap(repr, public_names)) + ) + + # -- Statement Visitors + + def visit_Template(self, node, frame=None): + assert frame is None, "no root frame allowed" + eval_ctx = EvalContext(self.environment, self.name) + + from .runtime import exported + + self.writeline("from __future__ import %s" % ", ".join(code_features)) + self.writeline("from jinja2.runtime import " + ", ".join(exported)) + + if self.environment.is_async: + self.writeline( + "from jinja2.asyncsupport import auto_await, " + "auto_aiter, AsyncLoopContext" + ) + + # if we want a deferred initialization we cannot move the + # environment into a local name + envenv = not self.defer_init and ", environment=environment" or "" + + # do we have an extends tag at all? If not, we can save some + # overhead by just not processing any inheritance code. + have_extends = node.find(nodes.Extends) is not None + + # find all blocks + for block in node.find_all(nodes.Block): + if block.name in self.blocks: + self.fail("block %r defined twice" % block.name, block.lineno) + self.blocks[block.name] = block + + # find all imports and import them + for import_ in node.find_all(nodes.ImportedName): + if import_.importname not in self.import_aliases: + imp = import_.importname + self.import_aliases[imp] = alias = self.temporary_identifier() + if "." in imp: + module, obj = imp.rsplit(".", 1) + self.writeline("from %s import %s as %s" % (module, obj, alias)) + else: + self.writeline("import %s as %s" % (imp, alias)) + + # add the load name + self.writeline("name = %r" % self.name) + + # generate the root render function. + self.writeline( + "%s(context, missing=missing%s):" % (self.func("root"), envenv), extra=1 + ) + self.indent() + self.write_commons() + + # process the root + frame = Frame(eval_ctx) + if "self" in find_undeclared(node.body, ("self",)): + ref = frame.symbols.declare_parameter("self") + self.writeline("%s = TemplateReference(context)" % ref) + frame.symbols.analyze_node(node) + frame.toplevel = frame.rootlevel = True + frame.require_output_check = have_extends and not self.has_known_extends + if have_extends: + self.writeline("parent_template = None") + self.enter_frame(frame) + self.pull_dependencies(node.body) + self.blockvisit(node.body, frame) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + # make sure that the parent root is called. + if have_extends: + if not self.has_known_extends: + self.indent() + self.writeline("if parent_template is not None:") + self.indent() + if supports_yield_from and not self.environment.is_async: + self.writeline("yield from parent_template.root_render_func(context)") + else: + self.writeline( + "%sfor event in parent_template." + "root_render_func(context):" + % (self.environment.is_async and "async " or "") + ) + self.indent() + self.writeline("yield event") + self.outdent() + self.outdent(1 + (not self.has_known_extends)) + + # at this point we now have the blocks collected and can visit them too. + for name, block in iteritems(self.blocks): + self.writeline( + "%s(context, missing=missing%s):" + % (self.func("block_" + name), envenv), + block, + 1, + ) + self.indent() + self.write_commons() + # It's important that we do not make this frame a child of the + # toplevel template. This would cause a variety of + # interesting issues with identifier tracking. + block_frame = Frame(eval_ctx) + undeclared = find_undeclared(block.body, ("self", "super")) + if "self" in undeclared: + ref = block_frame.symbols.declare_parameter("self") + self.writeline("%s = TemplateReference(context)" % ref) + if "super" in undeclared: + ref = block_frame.symbols.declare_parameter("super") + self.writeline("%s = context.super(%r, block_%s)" % (ref, name, name)) + block_frame.symbols.analyze_node(block) + block_frame.block = name + self.enter_frame(block_frame) + self.pull_dependencies(block.body) + self.blockvisit(block.body, block_frame) + self.leave_frame(block_frame, with_python_scope=True) + self.outdent() + + self.writeline( + "blocks = {%s}" % ", ".join("%r: block_%s" % (x, x) for x in self.blocks), + extra=1, + ) + + # add a function that returns the debug info + self.writeline( + "debug_info = %r" % "&".join("%s=%s" % x for x in self.debug_info) + ) + + def visit_Block(self, node, frame): + """Call a block and register it for the template.""" + level = 0 + if frame.toplevel: + # if we know that we are a child template, there is no need to + # check if we are one + if self.has_known_extends: + return + if self.extends_so_far > 0: + self.writeline("if parent_template is None:") + self.indent() + level += 1 + + if node.scoped: + context = self.derive_context(frame) + else: + context = self.get_context_ref() + + if ( + supports_yield_from + and not self.environment.is_async + and frame.buffer is None + ): + self.writeline( + "yield from context.blocks[%r][0](%s)" % (node.name, context), node + ) + else: + loop = self.environment.is_async and "async for" or "for" + self.writeline( + "%s event in context.blocks[%r][0](%s):" % (loop, node.name, context), + node, + ) + self.indent() + self.simple_write("event", frame) + self.outdent() + + self.outdent(level) + + def visit_Extends(self, node, frame): + """Calls the extender.""" + if not frame.toplevel: + self.fail("cannot use extend from a non top-level scope", node.lineno) + + # if the number of extends statements in general is zero so + # far, we don't have to add a check if something extended + # the template before this one. + if self.extends_so_far > 0: + + # if we have a known extends we just add a template runtime + # error into the generated code. We could catch that at compile + # time too, but i welcome it not to confuse users by throwing the + # same error at different times just "because we can". + if not self.has_known_extends: + self.writeline("if parent_template is not None:") + self.indent() + self.writeline("raise TemplateRuntimeError(%r)" % "extended multiple times") + + # if we have a known extends already we don't need that code here + # as we know that the template execution will end here. + if self.has_known_extends: + raise CompilerExit() + else: + self.outdent() + + self.writeline("parent_template = environment.get_template(", node) + self.visit(node.template, frame) + self.write(", %r)" % self.name) + self.writeline( + "for name, parent_block in parent_template.blocks.%s():" % dict_item_iter + ) + self.indent() + self.writeline("context.blocks.setdefault(name, []).append(parent_block)") + self.outdent() + + # if this extends statement was in the root level we can take + # advantage of that information and simplify the generated code + # in the top level from this point onwards + if frame.rootlevel: + self.has_known_extends = True + + # and now we have one more + self.extends_so_far += 1 + + def visit_Include(self, node, frame): + """Handles includes.""" + if node.ignore_missing: + self.writeline("try:") + self.indent() + + func_name = "get_or_select_template" + if isinstance(node.template, nodes.Const): + if isinstance(node.template.value, string_types): + func_name = "get_template" + elif isinstance(node.template.value, (tuple, list)): + func_name = "select_template" + elif isinstance(node.template, (nodes.Tuple, nodes.List)): + func_name = "select_template" + + self.writeline("template = environment.%s(" % func_name, node) + self.visit(node.template, frame) + self.write(", %r)" % self.name) + if node.ignore_missing: + self.outdent() + self.writeline("except TemplateNotFound:") + self.indent() + self.writeline("pass") + self.outdent() + self.writeline("else:") + self.indent() + + skip_event_yield = False + if node.with_context: + loop = self.environment.is_async and "async for" or "for" + self.writeline( + "%s event in template.root_render_func(" + "template.new_context(context.get_all(), True, " + "%s)):" % (loop, self.dump_local_context(frame)) + ) + elif self.environment.is_async: + self.writeline( + "for event in (await " + "template._get_default_module_async())" + "._body_stream:" + ) + else: + if supports_yield_from: + self.writeline("yield from template._get_default_module()._body_stream") + skip_event_yield = True + else: + self.writeline( + "for event in template._get_default_module()._body_stream:" + ) + + if not skip_event_yield: + self.indent() + self.simple_write("event", frame) + self.outdent() + + if node.ignore_missing: + self.outdent() + + def visit_Import(self, node, frame): + """Visit regular imports.""" + self.writeline("%s = " % frame.symbols.ref(node.target), node) + if frame.toplevel: + self.write("context.vars[%r] = " % node.target) + if self.environment.is_async: + self.write("await ") + self.write("environment.get_template(") + self.visit(node.template, frame) + self.write(", %r)." % self.name) + if node.with_context: + self.write( + "make_module%s(context.get_all(), True, %s)" + % ( + self.environment.is_async and "_async" or "", + self.dump_local_context(frame), + ) + ) + elif self.environment.is_async: + self.write("_get_default_module_async()") + else: + self.write("_get_default_module()") + if frame.toplevel and not node.target.startswith("_"): + self.writeline("context.exported_vars.discard(%r)" % node.target) + + def visit_FromImport(self, node, frame): + """Visit named imports.""" + self.newline(node) + self.write( + "included_template = %senvironment.get_template(" + % (self.environment.is_async and "await " or "") + ) + self.visit(node.template, frame) + self.write(", %r)." % self.name) + if node.with_context: + self.write( + "make_module%s(context.get_all(), True, %s)" + % ( + self.environment.is_async and "_async" or "", + self.dump_local_context(frame), + ) + ) + elif self.environment.is_async: + self.write("_get_default_module_async()") + else: + self.write("_get_default_module()") + + var_names = [] + discarded_names = [] + for name in node.names: + if isinstance(name, tuple): + name, alias = name + else: + alias = name + self.writeline( + "%s = getattr(included_template, " + "%r, missing)" % (frame.symbols.ref(alias), name) + ) + self.writeline("if %s is missing:" % frame.symbols.ref(alias)) + self.indent() + self.writeline( + "%s = undefined(%r %% " + "included_template.__name__, " + "name=%r)" + % ( + frame.symbols.ref(alias), + "the template %%r (imported on %s) does " + "not export the requested name %s" + % (self.position(node), repr(name)), + name, + ) + ) + self.outdent() + if frame.toplevel: + var_names.append(alias) + if not alias.startswith("_"): + discarded_names.append(alias) + + if var_names: + if len(var_names) == 1: + name = var_names[0] + self.writeline( + "context.vars[%r] = %s" % (name, frame.symbols.ref(name)) + ) + else: + self.writeline( + "context.vars.update({%s})" + % ", ".join( + "%r: %s" % (name, frame.symbols.ref(name)) for name in var_names + ) + ) + if discarded_names: + if len(discarded_names) == 1: + self.writeline("context.exported_vars.discard(%r)" % discarded_names[0]) + else: + self.writeline( + "context.exported_vars.difference_" + "update((%s))" % ", ".join(imap(repr, discarded_names)) + ) + + def visit_For(self, node, frame): + loop_frame = frame.inner() + test_frame = frame.inner() + else_frame = frame.inner() + + # try to figure out if we have an extended loop. An extended loop + # is necessary if the loop is in recursive mode if the special loop + # variable is accessed in the body. + extended_loop = node.recursive or "loop" in find_undeclared( + node.iter_child_nodes(only=("body",)), ("loop",) + ) + + loop_ref = None + if extended_loop: + loop_ref = loop_frame.symbols.declare_parameter("loop") + + loop_frame.symbols.analyze_node(node, for_branch="body") + if node.else_: + else_frame.symbols.analyze_node(node, for_branch="else") + + if node.test: + loop_filter_func = self.temporary_identifier() + test_frame.symbols.analyze_node(node, for_branch="test") + self.writeline("%s(fiter):" % self.func(loop_filter_func), node.test) + self.indent() + self.enter_frame(test_frame) + self.writeline(self.environment.is_async and "async for " or "for ") + self.visit(node.target, loop_frame) + self.write(" in ") + self.write(self.environment.is_async and "auto_aiter(fiter)" or "fiter") + self.write(":") + self.indent() + self.writeline("if ", node.test) + self.visit(node.test, test_frame) + self.write(":") + self.indent() + self.writeline("yield ") + self.visit(node.target, loop_frame) + self.outdent(3) + self.leave_frame(test_frame, with_python_scope=True) + + # if we don't have an recursive loop we have to find the shadowed + # variables at that point. Because loops can be nested but the loop + # variable is a special one we have to enforce aliasing for it. + if node.recursive: + self.writeline( + "%s(reciter, loop_render_func, depth=0):" % self.func("loop"), node + ) + self.indent() + self.buffer(loop_frame) + + # Use the same buffer for the else frame + else_frame.buffer = loop_frame.buffer + + # make sure the loop variable is a special one and raise a template + # assertion error if a loop tries to write to loop + if extended_loop: + self.writeline("%s = missing" % loop_ref) + + for name in node.find_all(nodes.Name): + if name.ctx == "store" and name.name == "loop": + self.fail( + "Can't assign to special loop variable in for-loop target", + name.lineno, + ) + + if node.else_: + iteration_indicator = self.temporary_identifier() + self.writeline("%s = 1" % iteration_indicator) + + self.writeline(self.environment.is_async and "async for " or "for ", node) + self.visit(node.target, loop_frame) + if extended_loop: + if self.environment.is_async: + self.write(", %s in AsyncLoopContext(" % loop_ref) + else: + self.write(", %s in LoopContext(" % loop_ref) + else: + self.write(" in ") + + if node.test: + self.write("%s(" % loop_filter_func) + if node.recursive: + self.write("reciter") + else: + if self.environment.is_async and not extended_loop: + self.write("auto_aiter(") + self.visit(node.iter, frame) + if self.environment.is_async and not extended_loop: + self.write(")") + if node.test: + self.write(")") + + if node.recursive: + self.write(", undefined, loop_render_func, depth):") + else: + self.write(extended_loop and ", undefined):" or ":") + + self.indent() + self.enter_frame(loop_frame) + + self.blockvisit(node.body, loop_frame) + if node.else_: + self.writeline("%s = 0" % iteration_indicator) + self.outdent() + self.leave_frame( + loop_frame, with_python_scope=node.recursive and not node.else_ + ) + + if node.else_: + self.writeline("if %s:" % iteration_indicator) + self.indent() + self.enter_frame(else_frame) + self.blockvisit(node.else_, else_frame) + self.leave_frame(else_frame) + self.outdent() + + # if the node was recursive we have to return the buffer contents + # and start the iteration code + if node.recursive: + self.return_buffer_contents(loop_frame) + self.outdent() + self.start_write(frame, node) + if self.environment.is_async: + self.write("await ") + self.write("loop(") + if self.environment.is_async: + self.write("auto_aiter(") + self.visit(node.iter, frame) + if self.environment.is_async: + self.write(")") + self.write(", loop)") + self.end_write(frame) + + def visit_If(self, node, frame): + if_frame = frame.soft() + self.writeline("if ", node) + self.visit(node.test, if_frame) + self.write(":") + self.indent() + self.blockvisit(node.body, if_frame) + self.outdent() + for elif_ in node.elif_: + self.writeline("elif ", elif_) + self.visit(elif_.test, if_frame) + self.write(":") + self.indent() + self.blockvisit(elif_.body, if_frame) + self.outdent() + if node.else_: + self.writeline("else:") + self.indent() + self.blockvisit(node.else_, if_frame) + self.outdent() + + def visit_Macro(self, node, frame): + macro_frame, macro_ref = self.macro_body(node, frame) + self.newline() + if frame.toplevel: + if not node.name.startswith("_"): + self.write("context.exported_vars.add(%r)" % node.name) + self.writeline("context.vars[%r] = " % node.name) + self.write("%s = " % frame.symbols.ref(node.name)) + self.macro_def(macro_ref, macro_frame) + + def visit_CallBlock(self, node, frame): + call_frame, macro_ref = self.macro_body(node, frame) + self.writeline("caller = ") + self.macro_def(macro_ref, call_frame) + self.start_write(frame, node) + self.visit_Call(node.call, frame, forward_caller=True) + self.end_write(frame) + + def visit_FilterBlock(self, node, frame): + filter_frame = frame.inner() + filter_frame.symbols.analyze_node(node) + self.enter_frame(filter_frame) + self.buffer(filter_frame) + self.blockvisit(node.body, filter_frame) + self.start_write(frame, node) + self.visit_Filter(node.filter, filter_frame) + self.end_write(frame) + self.leave_frame(filter_frame) + + def visit_With(self, node, frame): + with_frame = frame.inner() + with_frame.symbols.analyze_node(node) + self.enter_frame(with_frame) + for target, expr in izip(node.targets, node.values): + self.newline() + self.visit(target, with_frame) + self.write(" = ") + self.visit(expr, frame) + self.blockvisit(node.body, with_frame) + self.leave_frame(with_frame) + + def visit_ExprStmt(self, node, frame): + self.newline(node) + self.visit(node.node, frame) + + _FinalizeInfo = namedtuple("_FinalizeInfo", ("const", "src")) + #: The default finalize function if the environment isn't configured + #: with one. Or if the environment has one, this is called on that + #: function's output for constants. + _default_finalize = text_type + _finalize = None + + def _make_finalize(self): + """Build the finalize function to be used on constants and at + runtime. Cached so it's only created once for all output nodes. + + Returns a ``namedtuple`` with the following attributes: + + ``const`` + A function to finalize constant data at compile time. + + ``src`` + Source code to output around nodes to be evaluated at + runtime. + """ + if self._finalize is not None: + return self._finalize + + finalize = default = self._default_finalize + src = None + + if self.environment.finalize: + src = "environment.finalize(" + env_finalize = self.environment.finalize + + def finalize(value): + return default(env_finalize(value)) + + if getattr(env_finalize, "contextfunction", False) is True: + src += "context, " + finalize = None # noqa: F811 + elif getattr(env_finalize, "evalcontextfunction", False) is True: + src += "context.eval_ctx, " + finalize = None + elif getattr(env_finalize, "environmentfunction", False) is True: + src += "environment, " + + def finalize(value): + return default(env_finalize(self.environment, value)) + + self._finalize = self._FinalizeInfo(finalize, src) + return self._finalize + + def _output_const_repr(self, group): + """Given a group of constant values converted from ``Output`` + child nodes, produce a string to write to the template module + source. + """ + return repr(concat(group)) + + def _output_child_to_const(self, node, frame, finalize): + """Try to optimize a child of an ``Output`` node by trying to + convert it to constant, finalized data at compile time. + + If :exc:`Impossible` is raised, the node is not constant and + will be evaluated at runtime. Any other exception will also be + evaluated at runtime for easier debugging. + """ + const = node.as_const(frame.eval_ctx) + + if frame.eval_ctx.autoescape: + const = escape(const) + + # Template data doesn't go through finalize. + if isinstance(node, nodes.TemplateData): + return text_type(const) + + return finalize.const(const) + + def _output_child_pre(self, node, frame, finalize): + """Output extra source code before visiting a child of an + ``Output`` node. + """ + if frame.eval_ctx.volatile: + self.write("(escape if context.eval_ctx.autoescape else to_string)(") + elif frame.eval_ctx.autoescape: + self.write("escape(") + else: + self.write("to_string(") + + if finalize.src is not None: + self.write(finalize.src) + + def _output_child_post(self, node, frame, finalize): + """Output extra source code after visiting a child of an + ``Output`` node. + """ + self.write(")") + + if finalize.src is not None: + self.write(")") + + def visit_Output(self, node, frame): + # If an extends is active, don't render outside a block. + if frame.require_output_check: + # A top-level extends is known to exist at compile time. + if self.has_known_extends: + return + + self.writeline("if parent_template is None:") + self.indent() + + finalize = self._make_finalize() + body = [] + + # Evaluate constants at compile time if possible. Each item in + # body will be either a list of static data or a node to be + # evaluated at runtime. + for child in node.nodes: + try: + if not ( + # If the finalize function requires runtime context, + # constants can't be evaluated at compile time. + finalize.const + # Unless it's basic template data that won't be + # finalized anyway. + or isinstance(child, nodes.TemplateData) + ): + raise nodes.Impossible() + + const = self._output_child_to_const(child, frame, finalize) + except (nodes.Impossible, Exception): + # The node was not constant and needs to be evaluated at + # runtime. Or another error was raised, which is easier + # to debug at runtime. + body.append(child) + continue + + if body and isinstance(body[-1], list): + body[-1].append(const) + else: + body.append([const]) + + if frame.buffer is not None: + if len(body) == 1: + self.writeline("%s.append(" % frame.buffer) + else: + self.writeline("%s.extend((" % frame.buffer) + + self.indent() + + for item in body: + if isinstance(item, list): + # A group of constant data to join and output. + val = self._output_const_repr(item) + + if frame.buffer is None: + self.writeline("yield " + val) + else: + self.writeline(val + ",") + else: + if frame.buffer is None: + self.writeline("yield ", item) + else: + self.newline(item) + + # A node to be evaluated at runtime. + self._output_child_pre(item, frame, finalize) + self.visit(item, frame) + self._output_child_post(item, frame, finalize) + + if frame.buffer is not None: + self.write(",") + + if frame.buffer is not None: + self.outdent() + self.writeline(")" if len(body) == 1 else "))") + + if frame.require_output_check: + self.outdent() + + def visit_Assign(self, node, frame): + self.push_assign_tracking() + self.newline(node) + self.visit(node.target, frame) + self.write(" = ") + self.visit(node.node, frame) + self.pop_assign_tracking(frame) + + def visit_AssignBlock(self, node, frame): + self.push_assign_tracking() + block_frame = frame.inner() + # This is a special case. Since a set block always captures we + # will disable output checks. This way one can use set blocks + # toplevel even in extended templates. + block_frame.require_output_check = False + block_frame.symbols.analyze_node(node) + self.enter_frame(block_frame) + self.buffer(block_frame) + self.blockvisit(node.body, block_frame) + self.newline(node) + self.visit(node.target, frame) + self.write(" = (Markup if context.eval_ctx.autoescape else identity)(") + if node.filter is not None: + self.visit_Filter(node.filter, block_frame) + else: + self.write("concat(%s)" % block_frame.buffer) + self.write(")") + self.pop_assign_tracking(frame) + self.leave_frame(block_frame) + + # -- Expression Visitors + + def visit_Name(self, node, frame): + if node.ctx == "store" and frame.toplevel: + if self._assign_stack: + self._assign_stack[-1].add(node.name) + ref = frame.symbols.ref(node.name) + + # If we are looking up a variable we might have to deal with the + # case where it's undefined. We can skip that case if the load + # instruction indicates a parameter which are always defined. + if node.ctx == "load": + load = frame.symbols.find_load(ref) + if not ( + load is not None + and load[0] == VAR_LOAD_PARAMETER + and not self.parameter_is_undeclared(ref) + ): + self.write( + "(undefined(name=%r) if %s is missing else %s)" + % (node.name, ref, ref) + ) + return + + self.write(ref) + + def visit_NSRef(self, node, frame): + # NSRefs can only be used to store values; since they use the normal + # `foo.bar` notation they will be parsed as a normal attribute access + # when used anywhere but in a `set` context + ref = frame.symbols.ref(node.name) + self.writeline("if not isinstance(%s, Namespace):" % ref) + self.indent() + self.writeline( + "raise TemplateRuntimeError(%r)" + % "cannot assign attribute on non-namespace object" + ) + self.outdent() + self.writeline("%s[%r]" % (ref, node.attr)) + + def visit_Const(self, node, frame): + val = node.as_const(frame.eval_ctx) + if isinstance(val, float): + self.write(str(val)) + else: + self.write(repr(val)) + + def visit_TemplateData(self, node, frame): + try: + self.write(repr(node.as_const(frame.eval_ctx))) + except nodes.Impossible: + self.write( + "(Markup if context.eval_ctx.autoescape else identity)(%r)" % node.data + ) + + def visit_Tuple(self, node, frame): + self.write("(") + idx = -1 + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item, frame) + self.write(idx == 0 and ",)" or ")") + + def visit_List(self, node, frame): + self.write("[") + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item, frame) + self.write("]") + + def visit_Dict(self, node, frame): + self.write("{") + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item.key, frame) + self.write(": ") + self.visit(item.value, frame) + self.write("}") + + def binop(operator, interceptable=True): # noqa: B902 + @optimizeconst + def visitor(self, node, frame): + if ( + self.environment.sandboxed + and operator in self.environment.intercepted_binops + ): + self.write("environment.call_binop(context, %r, " % operator) + self.visit(node.left, frame) + self.write(", ") + self.visit(node.right, frame) + else: + self.write("(") + self.visit(node.left, frame) + self.write(" %s " % operator) + self.visit(node.right, frame) + self.write(")") + + return visitor + + def uaop(operator, interceptable=True): # noqa: B902 + @optimizeconst + def visitor(self, node, frame): + if ( + self.environment.sandboxed + and operator in self.environment.intercepted_unops + ): + self.write("environment.call_unop(context, %r, " % operator) + self.visit(node.node, frame) + else: + self.write("(" + operator) + self.visit(node.node, frame) + self.write(")") + + return visitor + + visit_Add = binop("+") + visit_Sub = binop("-") + visit_Mul = binop("*") + visit_Div = binop("/") + visit_FloorDiv = binop("//") + visit_Pow = binop("**") + visit_Mod = binop("%") + visit_And = binop("and", interceptable=False) + visit_Or = binop("or", interceptable=False) + visit_Pos = uaop("+") + visit_Neg = uaop("-") + visit_Not = uaop("not ", interceptable=False) + del binop, uaop + + @optimizeconst + def visit_Concat(self, node, frame): + if frame.eval_ctx.volatile: + func_name = "(context.eval_ctx.volatile and markup_join or unicode_join)" + elif frame.eval_ctx.autoescape: + func_name = "markup_join" + else: + func_name = "unicode_join" + self.write("%s((" % func_name) + for arg in node.nodes: + self.visit(arg, frame) + self.write(", ") + self.write("))") + + @optimizeconst + def visit_Compare(self, node, frame): + self.write("(") + self.visit(node.expr, frame) + for op in node.ops: + self.visit(op, frame) + self.write(")") + + def visit_Operand(self, node, frame): + self.write(" %s " % operators[node.op]) + self.visit(node.expr, frame) + + @optimizeconst + def visit_Getattr(self, node, frame): + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getattr(") + self.visit(node.node, frame) + self.write(", %r)" % node.attr) + + if self.environment.is_async: + self.write("))") + + @optimizeconst + def visit_Getitem(self, node, frame): + # slices bypass the environment getitem method. + if isinstance(node.arg, nodes.Slice): + self.visit(node.node, frame) + self.write("[") + self.visit(node.arg, frame) + self.write("]") + else: + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getitem(") + self.visit(node.node, frame) + self.write(", ") + self.visit(node.arg, frame) + self.write(")") + + if self.environment.is_async: + self.write("))") + + def visit_Slice(self, node, frame): + if node.start is not None: + self.visit(node.start, frame) + self.write(":") + if node.stop is not None: + self.visit(node.stop, frame) + if node.step is not None: + self.write(":") + self.visit(node.step, frame) + + @optimizeconst + def visit_Filter(self, node, frame): + if self.environment.is_async: + self.write("await auto_await(") + self.write(self.filters[node.name] + "(") + func = self.environment.filters.get(node.name) + if func is None: + self.fail("no filter named %r" % node.name, node.lineno) + if getattr(func, "contextfilter", False) is True: + self.write("context, ") + elif getattr(func, "evalcontextfilter", False) is True: + self.write("context.eval_ctx, ") + elif getattr(func, "environmentfilter", False) is True: + self.write("environment, ") + + # if the filter node is None we are inside a filter block + # and want to write to the current buffer + if node.node is not None: + self.visit(node.node, frame) + elif frame.eval_ctx.volatile: + self.write( + "(context.eval_ctx.autoescape and" + " Markup(concat(%s)) or concat(%s))" % (frame.buffer, frame.buffer) + ) + elif frame.eval_ctx.autoescape: + self.write("Markup(concat(%s))" % frame.buffer) + else: + self.write("concat(%s)" % frame.buffer) + self.signature(node, frame) + self.write(")") + if self.environment.is_async: + self.write(")") + + @optimizeconst + def visit_Test(self, node, frame): + self.write(self.tests[node.name] + "(") + if node.name not in self.environment.tests: + self.fail("no test named %r" % node.name, node.lineno) + self.visit(node.node, frame) + self.signature(node, frame) + self.write(")") + + @optimizeconst + def visit_CondExpr(self, node, frame): + def write_expr2(): + if node.expr2 is not None: + return self.visit(node.expr2, frame) + self.write( + "cond_expr_undefined(%r)" + % ( + "the inline if-" + "expression on %s evaluated to false and " + "no else section was defined." % self.position(node) + ) + ) + + self.write("(") + self.visit(node.expr1, frame) + self.write(" if ") + self.visit(node.test, frame) + self.write(" else ") + write_expr2() + self.write(")") + + @optimizeconst + def visit_Call(self, node, frame, forward_caller=False): + if self.environment.is_async: + self.write("await auto_await(") + if self.environment.sandboxed: + self.write("environment.call(context, ") + else: + self.write("context.call(") + self.visit(node.node, frame) + extra_kwargs = forward_caller and {"caller": "caller"} or None + self.signature(node, frame, extra_kwargs) + self.write(")") + if self.environment.is_async: + self.write(")") + + def visit_Keyword(self, node, frame): + self.write(node.key + "=") + self.visit(node.value, frame) + + # -- Unused nodes for extensions + + def visit_MarkSafe(self, node, frame): + self.write("Markup(") + self.visit(node.expr, frame) + self.write(")") + + def visit_MarkSafeIfAutoescape(self, node, frame): + self.write("(context.eval_ctx.autoescape and Markup or identity)(") + self.visit(node.expr, frame) + self.write(")") + + def visit_EnvironmentAttribute(self, node, frame): + self.write("environment." + node.name) + + def visit_ExtensionAttribute(self, node, frame): + self.write("environment.extensions[%r].%s" % (node.identifier, node.name)) + + def visit_ImportedName(self, node, frame): + self.write(self.import_aliases[node.importname]) + + def visit_InternalName(self, node, frame): + self.write(node.name) + + def visit_ContextReference(self, node, frame): + self.write("context") + + def visit_DerivedContextReference(self, node, frame): + self.write(self.derive_context(frame)) + + def visit_Continue(self, node, frame): + self.writeline("continue", node) + + def visit_Break(self, node, frame): + self.writeline("break", node) + + def visit_Scope(self, node, frame): + scope_frame = frame.inner() + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + + def visit_OverlayScope(self, node, frame): + ctx = self.temporary_identifier() + self.writeline("%s = %s" % (ctx, self.derive_context(frame))) + self.writeline("%s.vars = " % ctx) + self.visit(node.context, frame) + self.push_context_reference(ctx) + + scope_frame = frame.inner(isolated=True) + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + self.pop_context_reference() + + def visit_EvalContextModifier(self, node, frame): + for keyword in node.options: + self.writeline("context.eval_ctx.%s = " % keyword.key) + self.visit(keyword.value, frame) + try: + val = keyword.value.as_const(frame.eval_ctx) + except nodes.Impossible: + frame.eval_ctx.volatile = True + else: + setattr(frame.eval_ctx, keyword.key, val) + + def visit_ScopedEvalContextModifier(self, node, frame): + old_ctx_name = self.temporary_identifier() + saved_ctx = frame.eval_ctx.save() + self.writeline("%s = context.eval_ctx.save()" % old_ctx_name) + self.visit_EvalContextModifier(node, frame) + for child in node.body: + self.visit(child, frame) + frame.eval_ctx.revert(saved_ctx) + self.writeline("context.eval_ctx.revert(%s)" % old_ctx_name) diff --git a/contrib/python/Jinja2/py2/jinja2/constants.py b/contrib/python/Jinja2/py2/jinja2/constants.py new file mode 100644 index 0000000000..bf7f2ca721 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/constants.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +#: list of lorem ipsum words used by the lipsum() helper function +LOREM_IPSUM_WORDS = u"""\ +a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at +auctor augue bibendum blandit class commodo condimentum congue consectetuer +consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus +diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend +elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames +faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac +hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum +justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem +luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie +mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non +nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque +penatibus per pharetra phasellus placerat platea porta porttitor posuere +potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus +ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit +sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor +tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices +ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus +viverra volutpat vulputate""" diff --git a/contrib/python/Jinja2/py2/jinja2/debug.py b/contrib/python/Jinja2/py2/jinja2/debug.py new file mode 100644 index 0000000000..5d8aec31d0 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/debug.py @@ -0,0 +1,268 @@ +import sys +from types import CodeType + +from . import TemplateSyntaxError +from ._compat import PYPY +from .utils import internal_code +from .utils import missing + + +def rewrite_traceback_stack(source=None): + """Rewrite the current exception to replace any tracebacks from + within compiled template code with tracebacks that look like they + came from the template source. + + This must be called within an ``except`` block. + + :param exc_info: A :meth:`sys.exc_info` tuple. If not provided, + the current ``exc_info`` is used. + :param source: For ``TemplateSyntaxError``, the original source if + known. + :return: A :meth:`sys.exc_info` tuple that can be re-raised. + """ + exc_type, exc_value, tb = sys.exc_info() + + if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated: + exc_value.translated = True + exc_value.source = source + + try: + # Remove the old traceback on Python 3, otherwise the frames + # from the compiler still show up. + exc_value.with_traceback(None) + except AttributeError: + pass + + # Outside of runtime, so the frame isn't executing template + # code, but it still needs to point at the template. + tb = fake_traceback( + exc_value, None, exc_value.filename or "<unknown>", exc_value.lineno + ) + else: + # Skip the frame for the render function. + tb = tb.tb_next + + stack = [] + + # Build the stack of traceback object, replacing any in template + # code with the source file and line information. + while tb is not None: + # Skip frames decorated with @internalcode. These are internal + # calls that aren't useful in template debugging output. + if tb.tb_frame.f_code in internal_code: + tb = tb.tb_next + continue + + template = tb.tb_frame.f_globals.get("__jinja_template__") + + if template is not None: + lineno = template.get_corresponding_lineno(tb.tb_lineno) + fake_tb = fake_traceback(exc_value, tb, template.filename, lineno) + stack.append(fake_tb) + else: + stack.append(tb) + + tb = tb.tb_next + + tb_next = None + + # Assign tb_next in reverse to avoid circular references. + for tb in reversed(stack): + tb_next = tb_set_next(tb, tb_next) + + return exc_type, exc_value, tb_next + + +def fake_traceback(exc_value, tb, filename, lineno): + """Produce a new traceback object that looks like it came from the + template source instead of the compiled code. The filename, line + number, and location name will point to the template, and the local + variables will be the current template context. + + :param exc_value: The original exception to be re-raised to create + the new traceback. + :param tb: The original traceback to get the local variables and + code info from. + :param filename: The template filename. + :param lineno: The line number in the template source. + """ + if tb is not None: + # Replace the real locals with the context that would be + # available at that point in the template. + locals = get_template_locals(tb.tb_frame.f_locals) + locals.pop("__jinja_exception__", None) + else: + locals = {} + + globals = { + "__name__": filename, + "__file__": filename, + "__jinja_exception__": exc_value, + } + # Raise an exception at the correct line number. + code = compile("\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec") + + # Build a new code object that points to the template file and + # replaces the location with a block name. + try: + location = "template" + + if tb is not None: + function = tb.tb_frame.f_code.co_name + + if function == "root": + location = "top-level template code" + elif function.startswith("block_"): + location = 'block "%s"' % function[6:] + + # Collect arguments for the new code object. CodeType only + # accepts positional arguments, and arguments were inserted in + # new Python versions. + code_args = [] + + for attr in ( + "argcount", + "posonlyargcount", # Python 3.8 + "kwonlyargcount", # Python 3 + "nlocals", + "stacksize", + "flags", + "code", # codestring + "consts", # constants + "names", + "varnames", + ("filename", filename), + ("name", location), + "firstlineno", + "lnotab", + "freevars", + "cellvars", + ): + if isinstance(attr, tuple): + # Replace with given value. + code_args.append(attr[1]) + continue + + try: + # Copy original value if it exists. + code_args.append(getattr(code, "co_" + attr)) + except AttributeError: + # Some arguments were added later. + continue + + code = CodeType(*code_args) + except Exception: + # Some environments such as Google App Engine don't support + # modifying code objects. + pass + + # Execute the new code, which is guaranteed to raise, and return + # the new traceback without this frame. + try: + exec(code, globals, locals) + except BaseException: + return sys.exc_info()[2].tb_next + + +def get_template_locals(real_locals): + """Based on the runtime locals, get the context that would be + available at that point in the template. + """ + # Start with the current template context. + ctx = real_locals.get("context") + + if ctx: + data = ctx.get_all().copy() + else: + data = {} + + # Might be in a derived context that only sets local variables + # rather than pushing a context. Local variables follow the scheme + # l_depth_name. Find the highest-depth local that has a value for + # each name. + local_overrides = {} + + for name, value in real_locals.items(): + if not name.startswith("l_") or value is missing: + # Not a template variable, or no longer relevant. + continue + + try: + _, depth, name = name.split("_", 2) + depth = int(depth) + except ValueError: + continue + + cur_depth = local_overrides.get(name, (-1,))[0] + + if cur_depth < depth: + local_overrides[name] = (depth, value) + + # Modify the context with any derived context. + for name, (_, value) in local_overrides.items(): + if value is missing: + data.pop(name, None) + else: + data[name] = value + + return data + + +if sys.version_info >= (3, 7): + # tb_next is directly assignable as of Python 3.7 + def tb_set_next(tb, tb_next): + tb.tb_next = tb_next + return tb + + +elif PYPY: + # PyPy might have special support, and won't work with ctypes. + try: + import tputil + except ImportError: + # Without tproxy support, use the original traceback. + def tb_set_next(tb, tb_next): + return tb + + else: + # With tproxy support, create a proxy around the traceback that + # returns the new tb_next. + def tb_set_next(tb, tb_next): + def controller(op): + if op.opname == "__getattribute__" and op.args[0] == "tb_next": + return tb_next + + return op.delegate() + + return tputil.make_proxy(controller, obj=tb) + + +else: + # Use ctypes to assign tb_next at the C level since it's read-only + # from Python. + import ctypes + + class _CTraceback(ctypes.Structure): + _fields_ = [ + # Extra PyObject slots when compiled with Py_TRACE_REFS. + ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()), + # Only care about tb_next as an object, not a traceback. + ("tb_next", ctypes.py_object), + ] + + def tb_set_next(tb, tb_next): + c_tb = _CTraceback.from_address(id(tb)) + + # Clear out the old tb_next. + if tb.tb_next is not None: + c_tb_next = ctypes.py_object(tb.tb_next) + c_tb.tb_next = ctypes.py_object() + ctypes.pythonapi.Py_DecRef(c_tb_next) + + # Assign the new tb_next. + if tb_next is not None: + c_tb_next = ctypes.py_object(tb_next) + ctypes.pythonapi.Py_IncRef(c_tb_next) + c_tb.tb_next = c_tb_next + + return tb diff --git a/contrib/python/Jinja2/py2/jinja2/defaults.py b/contrib/python/Jinja2/py2/jinja2/defaults.py new file mode 100644 index 0000000000..8e0e7d7710 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/defaults.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from ._compat import range_type +from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 +from .tests import TESTS as DEFAULT_TESTS # noqa: F401 +from .utils import Cycler +from .utils import generate_lorem_ipsum +from .utils import Joiner +from .utils import Namespace + +# defaults for the parser / lexer +BLOCK_START_STRING = "{%" +BLOCK_END_STRING = "%}" +VARIABLE_START_STRING = "{{" +VARIABLE_END_STRING = "}}" +COMMENT_START_STRING = "{#" +COMMENT_END_STRING = "#}" +LINE_STATEMENT_PREFIX = None +LINE_COMMENT_PREFIX = None +TRIM_BLOCKS = False +LSTRIP_BLOCKS = False +NEWLINE_SEQUENCE = "\n" +KEEP_TRAILING_NEWLINE = False + +# default filters, tests and namespace + +DEFAULT_NAMESPACE = { + "range": range_type, + "dict": dict, + "lipsum": generate_lorem_ipsum, + "cycler": Cycler, + "joiner": Joiner, + "namespace": Namespace, +} + +# default policies +DEFAULT_POLICIES = { + "compiler.ascii_str": True, + "urlize.rel": "noopener", + "urlize.target": None, + "truncate.leeway": 5, + "json.dumps_function": None, + "json.dumps_kwargs": {"sort_keys": True}, + "ext.i18n.trimmed": False, +} diff --git a/contrib/python/Jinja2/py2/jinja2/environment.py b/contrib/python/Jinja2/py2/jinja2/environment.py new file mode 100644 index 0000000000..8430390eea --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/environment.py @@ -0,0 +1,1362 @@ +# -*- coding: utf-8 -*- +"""Classes for managing templates and their runtime and compile time +options. +""" +import os +import sys +import weakref +from functools import partial +from functools import reduce + +from markupsafe import Markup + +from . import nodes +from ._compat import encode_filename +from ._compat import implements_iterator +from ._compat import implements_to_string +from ._compat import iteritems +from ._compat import PY2 +from ._compat import PYPY +from ._compat import reraise +from ._compat import string_types +from ._compat import text_type +from .compiler import CodeGenerator +from .compiler import generate +from .defaults import BLOCK_END_STRING +from .defaults import BLOCK_START_STRING +from .defaults import COMMENT_END_STRING +from .defaults import COMMENT_START_STRING +from .defaults import DEFAULT_FILTERS +from .defaults import DEFAULT_NAMESPACE +from .defaults import DEFAULT_POLICIES +from .defaults import DEFAULT_TESTS +from .defaults import KEEP_TRAILING_NEWLINE +from .defaults import LINE_COMMENT_PREFIX +from .defaults import LINE_STATEMENT_PREFIX +from .defaults import LSTRIP_BLOCKS +from .defaults import NEWLINE_SEQUENCE +from .defaults import TRIM_BLOCKS +from .defaults import VARIABLE_END_STRING +from .defaults import VARIABLE_START_STRING +from .exceptions import TemplateNotFound +from .exceptions import TemplateRuntimeError +from .exceptions import TemplatesNotFound +from .exceptions import TemplateSyntaxError +from .exceptions import UndefinedError +from .lexer import get_lexer +from .lexer import TokenStream +from .nodes import EvalContext +from .parser import Parser +from .runtime import Context +from .runtime import new_context +from .runtime import Undefined +from .utils import concat +from .utils import consume +from .utils import have_async_gen +from .utils import import_string +from .utils import internalcode +from .utils import LRUCache +from .utils import missing + +# for direct template usage we have up to ten living environments +_spontaneous_environments = LRUCache(10) + + +def get_spontaneous_environment(cls, *args): + """Return a new spontaneous environment. A spontaneous environment + is used for templates created directly rather than through an + existing environment. + + :param cls: Environment class to create. + :param args: Positional arguments passed to environment. + """ + key = (cls, args) + + try: + return _spontaneous_environments[key] + except KeyError: + _spontaneous_environments[key] = env = cls(*args) + env.shared = True + return env + + +def create_cache(size): + """Return the cache class for the given size.""" + if size == 0: + return None + if size < 0: + return {} + return LRUCache(size) + + +def copy_cache(cache): + """Create an empty copy of the given cache.""" + if cache is None: + return None + elif type(cache) is dict: + return {} + return LRUCache(cache.capacity) + + +def load_extensions(environment, extensions): + """Load the extensions from the list and bind it to the environment. + Returns a dict of instantiated environments. + """ + result = {} + for extension in extensions: + if isinstance(extension, string_types): + extension = import_string(extension) + result[extension.identifier] = extension(environment) + return result + + +def fail_for_missing_callable(string, name): + msg = string % name + if isinstance(name, Undefined): + try: + name._fail_with_undefined_error() + except Exception as e: + msg = "%s (%s; did you forget to quote the callable name?)" % (msg, e) + raise TemplateRuntimeError(msg) + + +def _environment_sanity_check(environment): + """Perform a sanity check on the environment.""" + assert issubclass( + environment.undefined, Undefined + ), "undefined must be a subclass of undefined because filters depend on it." + assert ( + environment.block_start_string + != environment.variable_start_string + != environment.comment_start_string + ), "block, variable and comment start strings must be different" + assert environment.newline_sequence in ( + "\r", + "\r\n", + "\n", + ), "newline_sequence set to unknown line ending string." + return environment + + +class Environment(object): + r"""The core component of Jinja is the `Environment`. It contains + important shared variables like configuration, filters, tests, + globals and others. Instances of this class may be modified if + they are not shared and if no template was loaded so far. + Modifications on environments after the first template was loaded + will lead to surprising effects and undefined behavior. + + Here are the possible initialization parameters: + + `block_start_string` + The string marking the beginning of a block. Defaults to ``'{%'``. + + `block_end_string` + The string marking the end of a block. Defaults to ``'%}'``. + + `variable_start_string` + The string marking the beginning of a print statement. + Defaults to ``'{{'``. + + `variable_end_string` + The string marking the end of a print statement. Defaults to + ``'}}'``. + + `comment_start_string` + The string marking the beginning of a comment. Defaults to ``'{#'``. + + `comment_end_string` + The string marking the end of a comment. Defaults to ``'#}'``. + + `line_statement_prefix` + If given and a string, this will be used as prefix for line based + statements. See also :ref:`line-statements`. + + `line_comment_prefix` + If given and a string, this will be used as prefix for line based + comments. See also :ref:`line-statements`. + + .. versionadded:: 2.2 + + `trim_blocks` + If this is set to ``True`` the first newline after a block is + removed (block, not variable tag!). Defaults to `False`. + + `lstrip_blocks` + If this is set to ``True`` leading spaces and tabs are stripped + from the start of a line to a block. Defaults to `False`. + + `newline_sequence` + The sequence that starts a newline. Must be one of ``'\r'``, + ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a + useful default for Linux and OS X systems as well as web + applications. + + `keep_trailing_newline` + Preserve the trailing newline when rendering templates. + The default is ``False``, which causes a single newline, + if present, to be stripped from the end of the template. + + .. versionadded:: 2.7 + + `extensions` + List of Jinja extensions to use. This can either be import paths + as strings or extension classes. For more information have a + look at :ref:`the extensions documentation <jinja-extensions>`. + + `optimized` + should the optimizer be enabled? Default is ``True``. + + `undefined` + :class:`Undefined` or a subclass of it that is used to represent + undefined values in the template. + + `finalize` + A callable that can be used to process the result of a variable + expression before it is output. For example one can convert + ``None`` implicitly into an empty string here. + + `autoescape` + If set to ``True`` the XML/HTML autoescaping feature is enabled by + default. For more details about autoescaping see + :class:`~markupsafe.Markup`. As of Jinja 2.4 this can also + be a callable that is passed the template name and has to + return ``True`` or ``False`` depending on autoescape should be + enabled by default. + + .. versionchanged:: 2.4 + `autoescape` can now be a function + + `loader` + The template loader for this environment. + + `cache_size` + The size of the cache. Per default this is ``400`` which means + that if more than 400 templates are loaded the loader will clean + out the least recently used template. If the cache size is set to + ``0`` templates are recompiled all the time, if the cache size is + ``-1`` the cache will not be cleaned. + + .. versionchanged:: 2.8 + The cache size was increased to 400 from a low 50. + + `auto_reload` + Some loaders load templates from locations where the template + sources may change (ie: file system or database). If + ``auto_reload`` is set to ``True`` (default) every time a template is + requested the loader checks if the source changed and if yes, it + will reload the template. For higher performance it's possible to + disable that. + + `bytecode_cache` + If set to a bytecode cache object, this object will provide a + cache for the internal Jinja bytecode so that templates don't + have to be parsed if they were not changed. + + See :ref:`bytecode-cache` for more information. + + `enable_async` + If set to true this enables async template execution which allows + you to take advantage of newer Python features. This requires + Python 3.6 or later. + """ + + #: if this environment is sandboxed. Modifying this variable won't make + #: the environment sandboxed though. For a real sandboxed environment + #: have a look at jinja2.sandbox. This flag alone controls the code + #: generation by the compiler. + sandboxed = False + + #: True if the environment is just an overlay + overlayed = False + + #: the environment this environment is linked to if it is an overlay + linked_to = None + + #: shared environments have this set to `True`. A shared environment + #: must not be modified + shared = False + + #: the class that is used for code generation. See + #: :class:`~jinja2.compiler.CodeGenerator` for more information. + code_generator_class = CodeGenerator + + #: the context class thatis used for templates. See + #: :class:`~jinja2.runtime.Context` for more information. + context_class = Context + + def __init__( + self, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + loader=None, + cache_size=400, + auto_reload=True, + bytecode_cache=None, + enable_async=False, + ): + # !!Important notice!! + # The constructor accepts quite a few arguments that should be + # passed by keyword rather than position. However it's important to + # not change the order of arguments because it's used at least + # internally in those cases: + # - spontaneous environments (i18n extension and Template) + # - unittests + # If parameter changes are required only add parameters at the end + # and don't change the arguments (or the defaults!) of the arguments + # existing already. + + # lexer / parser information + self.block_start_string = block_start_string + self.block_end_string = block_end_string + self.variable_start_string = variable_start_string + self.variable_end_string = variable_end_string + self.comment_start_string = comment_start_string + self.comment_end_string = comment_end_string + self.line_statement_prefix = line_statement_prefix + self.line_comment_prefix = line_comment_prefix + self.trim_blocks = trim_blocks + self.lstrip_blocks = lstrip_blocks + self.newline_sequence = newline_sequence + self.keep_trailing_newline = keep_trailing_newline + + # runtime information + self.undefined = undefined + self.optimized = optimized + self.finalize = finalize + self.autoescape = autoescape + + # defaults + self.filters = DEFAULT_FILTERS.copy() + self.tests = DEFAULT_TESTS.copy() + self.globals = DEFAULT_NAMESPACE.copy() + + # set the loader provided + self.loader = loader + self.cache = create_cache(cache_size) + self.bytecode_cache = bytecode_cache + self.auto_reload = auto_reload + + # configurable policies + self.policies = DEFAULT_POLICIES.copy() + + # load extensions + self.extensions = load_extensions(self, extensions) + + self.enable_async = enable_async + self.is_async = self.enable_async and have_async_gen + if self.is_async: + # runs patch_all() to enable async support + from . import asyncsupport # noqa: F401 + + _environment_sanity_check(self) + + def add_extension(self, extension): + """Adds an extension after the environment was created. + + .. versionadded:: 2.5 + """ + self.extensions.update(load_extensions(self, [extension])) + + def extend(self, **attributes): + """Add the items to the instance of the environment if they do not exist + yet. This is used by :ref:`extensions <writing-extensions>` to register + callbacks and configuration values without breaking inheritance. + """ + for key, value in iteritems(attributes): + if not hasattr(self, key): + setattr(self, key, value) + + def overlay( + self, + block_start_string=missing, + block_end_string=missing, + variable_start_string=missing, + variable_end_string=missing, + comment_start_string=missing, + comment_end_string=missing, + line_statement_prefix=missing, + line_comment_prefix=missing, + trim_blocks=missing, + lstrip_blocks=missing, + extensions=missing, + optimized=missing, + undefined=missing, + finalize=missing, + autoescape=missing, + loader=missing, + cache_size=missing, + auto_reload=missing, + bytecode_cache=missing, + ): + """Create a new overlay environment that shares all the data with the + current environment except for cache and the overridden attributes. + Extensions cannot be removed for an overlayed environment. An overlayed + environment automatically gets all the extensions of the environment it + is linked to plus optional extra extensions. + + Creating overlays should happen after the initial environment was set + up completely. Not all attributes are truly linked, some are just + copied over so modifications on the original environment may not shine + through. + """ + args = dict(locals()) + del args["self"], args["cache_size"], args["extensions"] + + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.overlayed = True + rv.linked_to = self + + for key, value in iteritems(args): + if value is not missing: + setattr(rv, key, value) + + if cache_size is not missing: + rv.cache = create_cache(cache_size) + else: + rv.cache = copy_cache(self.cache) + + rv.extensions = {} + for key, value in iteritems(self.extensions): + rv.extensions[key] = value.bind(rv) + if extensions is not missing: + rv.extensions.update(load_extensions(rv, extensions)) + + return _environment_sanity_check(rv) + + lexer = property(get_lexer, doc="The lexer for this environment.") + + def iter_extensions(self): + """Iterates over the extensions by priority.""" + return iter(sorted(self.extensions.values(), key=lambda x: x.priority)) + + def getitem(self, obj, argument): + """Get an item or attribute of an object but prefer the item.""" + try: + return obj[argument] + except (AttributeError, TypeError, LookupError): + if isinstance(argument, string_types): + try: + attr = str(argument) + except Exception: + pass + else: + try: + return getattr(obj, attr) + except AttributeError: + pass + return self.undefined(obj=obj, name=argument) + + def getattr(self, obj, attribute): + """Get an item or attribute of an object but prefer the attribute. + Unlike :meth:`getitem` the attribute *must* be a bytestring. + """ + try: + return getattr(obj, attribute) + except AttributeError: + pass + try: + return obj[attribute] + except (TypeError, LookupError, AttributeError): + return self.undefined(obj=obj, name=attribute) + + def call_filter( + self, name, value, args=None, kwargs=None, context=None, eval_ctx=None + ): + """Invokes a filter on a value the same way the compiler does it. + + Note that on Python 3 this might return a coroutine in case the + filter is running from an environment in async mode and the filter + supports async execution. It's your responsibility to await this + if needed. + + .. versionadded:: 2.7 + """ + func = self.filters.get(name) + if func is None: + fail_for_missing_callable("no filter named %r", name) + args = [value] + list(args or ()) + if getattr(func, "contextfilter", False) is True: + if context is None: + raise TemplateRuntimeError( + "Attempted to invoke context filter without context" + ) + args.insert(0, context) + elif getattr(func, "evalcontextfilter", False) is True: + if eval_ctx is None: + if context is not None: + eval_ctx = context.eval_ctx + else: + eval_ctx = EvalContext(self) + args.insert(0, eval_ctx) + elif getattr(func, "environmentfilter", False) is True: + args.insert(0, self) + return func(*args, **(kwargs or {})) + + def call_test(self, name, value, args=None, kwargs=None): + """Invokes a test on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.tests.get(name) + if func is None: + fail_for_missing_callable("no test named %r", name) + return func(value, *(args or ()), **(kwargs or {})) + + @internalcode + def parse(self, source, name=None, filename=None): + """Parse the sourcecode and return the abstract syntax tree. This + tree of nodes is used by the compiler to convert the template into + executable source- or bytecode. This is useful for debugging or to + extract information from templates. + + If you are :ref:`developing Jinja extensions <writing-extensions>` + this gives you a good overview of the node tree generated. + """ + try: + return self._parse(source, name, filename) + except TemplateSyntaxError: + self.handle_exception(source=source) + + def _parse(self, source, name, filename): + """Internal parsing function used by `parse` and `compile`.""" + return Parser(self, source, name, encode_filename(filename)).parse() + + def lex(self, source, name=None, filename=None): + """Lex the given sourcecode and return a generator that yields + tokens as tuples in the form ``(lineno, token_type, value)``. + This can be useful for :ref:`extension development <writing-extensions>` + and debugging templates. + + This does not perform preprocessing. If you want the preprocessing + of the extensions to be applied you have to filter source through + the :meth:`preprocess` method. + """ + source = text_type(source) + try: + return self.lexer.tokeniter(source, name, filename) + except TemplateSyntaxError: + self.handle_exception(source=source) + + def preprocess(self, source, name=None, filename=None): + """Preprocesses the source with all extensions. This is automatically + called for all parsing and compiling methods but *not* for :meth:`lex` + because there you usually only want the actual source tokenized. + """ + return reduce( + lambda s, e: e.preprocess(s, name, filename), + self.iter_extensions(), + text_type(source), + ) + + def _tokenize(self, source, name, filename=None, state=None): + """Called by the parser to do the preprocessing and filtering + for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`. + """ + source = self.preprocess(source, name, filename) + stream = self.lexer.tokenize(source, name, filename, state) + for ext in self.iter_extensions(): + stream = ext.filter_stream(stream) + if not isinstance(stream, TokenStream): + stream = TokenStream(stream, name, filename) + return stream + + def _generate(self, source, name, filename, defer_init=False): + """Internal hook that can be overridden to hook a different generate + method in. + + .. versionadded:: 2.5 + """ + return generate( + source, + self, + name, + filename, + defer_init=defer_init, + optimized=self.optimized, + ) + + def _compile(self, source, filename): + """Internal hook that can be overridden to hook a different compile + method in. + + .. versionadded:: 2.5 + """ + return compile(source, filename, "exec") + + @internalcode + def compile(self, source, name=None, filename=None, raw=False, defer_init=False): + """Compile a node or template source code. The `name` parameter is + the load name of the template after it was joined using + :meth:`join_path` if necessary, not the filename on the file system. + the `filename` parameter is the estimated filename of the template on + the file system. If the template came from a database or memory this + can be omitted. + + The return value of this method is a python code object. If the `raw` + parameter is `True` the return value will be a string with python + code equivalent to the bytecode returned otherwise. This method is + mainly used internally. + + `defer_init` is use internally to aid the module code generator. This + causes the generated code to be able to import without the global + environment variable to be set. + + .. versionadded:: 2.4 + `defer_init` parameter added. + """ + source_hint = None + try: + if isinstance(source, string_types): + source_hint = source + source = self._parse(source, name, filename) + source = self._generate(source, name, filename, defer_init=defer_init) + if raw: + return source + if filename is None: + filename = "<template>" + else: + filename = encode_filename(filename) + return self._compile(source, filename) + except TemplateSyntaxError: + self.handle_exception(source=source_hint) + + def compile_expression(self, source, undefined_to_none=True): + """A handy helper method that returns a callable that accepts keyword + arguments that appear as variables in the expression. If called it + returns the result of the expression. + + This is useful if applications want to use the same rules as Jinja + in template "configuration files" or similar situations. + + Example usage: + + >>> env = Environment() + >>> expr = env.compile_expression('foo == 42') + >>> expr(foo=23) + False + >>> expr(foo=42) + True + + Per default the return value is converted to `None` if the + expression returns an undefined value. This can be changed + by setting `undefined_to_none` to `False`. + + >>> env.compile_expression('var')() is None + True + >>> env.compile_expression('var', undefined_to_none=False)() + Undefined + + .. versionadded:: 2.1 + """ + parser = Parser(self, source, state="variable") + try: + expr = parser.parse_expression() + if not parser.stream.eos: + raise TemplateSyntaxError( + "chunk after expression", parser.stream.current.lineno, None, None + ) + expr.set_environment(self) + except TemplateSyntaxError: + if sys.exc_info() is not None: + self.handle_exception(source=source) + + body = [nodes.Assign(nodes.Name("result", "store"), expr, lineno=1)] + template = self.from_string(nodes.Template(body, lineno=1)) + return TemplateExpression(template, undefined_to_none) + + def compile_templates( + self, + target, + extensions=None, + filter_func=None, + zip="deflated", + log_function=None, + ignore_errors=True, + py_compile=False, + ): + """Finds all the templates the loader can find, compiles them + and stores them in `target`. If `zip` is `None`, instead of in a + zipfile, the templates will be stored in a directory. + By default a deflate zip algorithm is used. To switch to + the stored algorithm, `zip` can be set to ``'stored'``. + + `extensions` and `filter_func` are passed to :meth:`list_templates`. + Each template returned will be compiled to the target folder or + zipfile. + + By default template compilation errors are ignored. In case a + log function is provided, errors are logged. If you want template + syntax errors to abort the compilation you can set `ignore_errors` + to `False` and you will get an exception on syntax errors. + + If `py_compile` is set to `True` .pyc files will be written to the + target instead of standard .py files. This flag does not do anything + on pypy and Python 3 where pyc files are not picked up by itself and + don't give much benefit. + + .. versionadded:: 2.4 + """ + from .loaders import ModuleLoader + + if log_function is None: + + def log_function(x): + pass + + if py_compile: + if not PY2 or PYPY: + import warnings + + warnings.warn( + "'py_compile=True' has no effect on PyPy or Python" + " 3 and will be removed in version 3.0", + DeprecationWarning, + stacklevel=2, + ) + py_compile = False + else: + import imp + import marshal + + py_header = imp.get_magic() + u"\xff\xff\xff\xff".encode("iso-8859-15") + + # Python 3.3 added a source filesize to the header + if sys.version_info >= (3, 3): + py_header += u"\x00\x00\x00\x00".encode("iso-8859-15") + + def write_file(filename, data): + if zip: + info = ZipInfo(filename) + info.external_attr = 0o755 << 16 + zip_file.writestr(info, data) + else: + if isinstance(data, text_type): + data = data.encode("utf8") + + with open(os.path.join(target, filename), "wb") as f: + f.write(data) + + if zip is not None: + from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED, ZIP_STORED + + zip_file = ZipFile( + target, "w", dict(deflated=ZIP_DEFLATED, stored=ZIP_STORED)[zip] + ) + log_function('Compiling into Zip archive "%s"' % target) + else: + if not os.path.isdir(target): + os.makedirs(target) + log_function('Compiling into folder "%s"' % target) + + try: + for name in self.list_templates(extensions, filter_func): + source, filename, _ = self.loader.get_source(self, name) + try: + code = self.compile(source, name, filename, True, True) + except TemplateSyntaxError as e: + if not ignore_errors: + raise + log_function('Could not compile "%s": %s' % (name, e)) + continue + + filename = ModuleLoader.get_module_filename(name) + + if py_compile: + c = self._compile(code, encode_filename(filename)) + write_file(filename + "c", py_header + marshal.dumps(c)) + log_function('Byte-compiled "%s" as %s' % (name, filename + "c")) + else: + write_file(filename, code) + log_function('Compiled "%s" as %s' % (name, filename)) + finally: + if zip: + zip_file.close() + + log_function("Finished compiling templates") + + def list_templates(self, extensions=None, filter_func=None): + """Returns a list of templates for this environment. This requires + that the loader supports the loader's + :meth:`~BaseLoader.list_templates` method. + + If there are other files in the template folder besides the + actual templates, the returned list can be filtered. There are two + ways: either `extensions` is set to a list of file extensions for + templates, or a `filter_func` can be provided which is a callable that + is passed a template name and should return `True` if it should end up + in the result list. + + If the loader does not support that, a :exc:`TypeError` is raised. + + .. versionadded:: 2.4 + """ + names = self.loader.list_templates() + + if extensions is not None: + if filter_func is not None: + raise TypeError( + "either extensions or filter_func can be passed, but not both" + ) + + def filter_func(x): + return "." in x and x.rsplit(".", 1)[1] in extensions + + if filter_func is not None: + names = [name for name in names if filter_func(name)] + + return names + + def handle_exception(self, source=None): + """Exception handling helper. This is used internally to either raise + rewritten exceptions or return a rendered traceback for the template. + """ + from .debug import rewrite_traceback_stack + + reraise(*rewrite_traceback_stack(source=source)) + + def join_path(self, template, parent): + """Join a template with the parent. By default all the lookups are + relative to the loader root so this method returns the `template` + parameter unchanged, but if the paths should be relative to the + parent template, this function can be used to calculate the real + template name. + + Subclasses may override this method and implement template path + joining here. + """ + return template + + @internalcode + def _load_template(self, name, globals): + if self.loader is None: + raise TypeError("no loader for this environment specified") + cache_key = (weakref.ref(self.loader), name) + if self.cache is not None: + template = self.cache.get(cache_key) + if template is not None and ( + not self.auto_reload or template.is_up_to_date + ): + return template + template = self.loader.load(self, name, globals) + if self.cache is not None: + self.cache[cache_key] = template + return template + + @internalcode + def get_template(self, name, parent=None, globals=None): + """Load a template from the loader. If a loader is configured this + method asks the loader for the template and returns a :class:`Template`. + If the `parent` parameter is not `None`, :meth:`join_path` is called + to get the real template name before loading. + + The `globals` parameter can be used to provide template wide globals. + These variables are available in the context at render time. + + If the template does not exist a :exc:`TemplateNotFound` exception is + raised. + + .. versionchanged:: 2.4 + If `name` is a :class:`Template` object it is returned from the + function unchanged. + """ + if isinstance(name, Template): + return name + if parent is not None: + name = self.join_path(name, parent) + return self._load_template(name, self.make_globals(globals)) + + @internalcode + def select_template(self, names, parent=None, globals=None): + """Works like :meth:`get_template` but tries a number of templates + before it fails. If it cannot find any of the templates, it will + raise a :exc:`TemplatesNotFound` exception. + + .. versionchanged:: 2.11 + If names is :class:`Undefined`, an :exc:`UndefinedError` is + raised instead. If no templates were found and names + contains :class:`Undefined`, the message is more helpful. + + .. versionchanged:: 2.4 + If `names` contains a :class:`Template` object it is returned + from the function unchanged. + + .. versionadded:: 2.3 + """ + if isinstance(names, Undefined): + names._fail_with_undefined_error() + + if not names: + raise TemplatesNotFound( + message=u"Tried to select from an empty list " u"of templates." + ) + globals = self.make_globals(globals) + for name in names: + if isinstance(name, Template): + return name + if parent is not None: + name = self.join_path(name, parent) + try: + return self._load_template(name, globals) + except (TemplateNotFound, UndefinedError): + pass + raise TemplatesNotFound(names) + + @internalcode + def get_or_select_template(self, template_name_or_list, parent=None, globals=None): + """Does a typecheck and dispatches to :meth:`select_template` + if an iterable of template names is given, otherwise to + :meth:`get_template`. + + .. versionadded:: 2.3 + """ + if isinstance(template_name_or_list, (string_types, Undefined)): + return self.get_template(template_name_or_list, parent, globals) + elif isinstance(template_name_or_list, Template): + return template_name_or_list + return self.select_template(template_name_or_list, parent, globals) + + def from_string(self, source, globals=None, template_class=None): + """Load a template from a string. This parses the source given and + returns a :class:`Template` object. + """ + globals = self.make_globals(globals) + cls = template_class or self.template_class + return cls.from_code(self, self.compile(source), globals, None) + + def make_globals(self, d): + """Return a dict for the globals.""" + if not d: + return self.globals + return dict(self.globals, **d) + + +class Template(object): + """The central template object. This class represents a compiled template + and is used to evaluate it. + + Normally the template object is generated from an :class:`Environment` but + it also has a constructor that makes it possible to create a template + instance directly using the constructor. It takes the same arguments as + the environment constructor but it's not possible to specify a loader. + + Every template object has a few methods and members that are guaranteed + to exist. However it's important that a template object should be + considered immutable. Modifications on the object are not supported. + + Template objects created from the constructor rather than an environment + do have an `environment` attribute that points to a temporary environment + that is probably shared with other templates created with the constructor + and compatible settings. + + >>> template = Template('Hello {{ name }}!') + >>> template.render(name='John Doe') == u'Hello John Doe!' + True + >>> stream = template.stream(name='John Doe') + >>> next(stream) == u'Hello John Doe!' + True + >>> next(stream) + Traceback (most recent call last): + ... + StopIteration + """ + + #: Type of environment to create when creating a template directly + #: rather than through an existing environment. + environment_class = Environment + + def __new__( + cls, + source, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + enable_async=False, + ): + env = get_spontaneous_environment( + cls.environment_class, + block_start_string, + block_end_string, + variable_start_string, + variable_end_string, + comment_start_string, + comment_end_string, + line_statement_prefix, + line_comment_prefix, + trim_blocks, + lstrip_blocks, + newline_sequence, + keep_trailing_newline, + frozenset(extensions), + optimized, + undefined, + finalize, + autoescape, + None, + 0, + False, + None, + enable_async, + ) + return env.from_string(source, template_class=cls) + + @classmethod + def from_code(cls, environment, code, globals, uptodate=None): + """Creates a template object from compiled code and the globals. This + is used by the loaders and environment to create a template object. + """ + namespace = {"environment": environment, "__file__": code.co_filename} + exec(code, namespace) + rv = cls._from_namespace(environment, namespace, globals) + rv._uptodate = uptodate + return rv + + @classmethod + def from_module_dict(cls, environment, module_dict, globals): + """Creates a template object from a module. This is used by the + module loader to create a template object. + + .. versionadded:: 2.4 + """ + return cls._from_namespace(environment, module_dict, globals) + + @classmethod + def _from_namespace(cls, environment, namespace, globals): + t = object.__new__(cls) + t.environment = environment + t.globals = globals + t.name = namespace["name"] + t.filename = namespace["__file__"] + t.blocks = namespace["blocks"] + + # render function and module + t.root_render_func = namespace["root"] + t._module = None + + # debug and loader helpers + t._debug_info = namespace["debug_info"] + t._uptodate = None + + # store the reference + namespace["environment"] = environment + namespace["__jinja_template__"] = t + + return t + + def render(self, *args, **kwargs): + """This method accepts the same arguments as the `dict` constructor: + A dict, a dict subclass or some keyword arguments. If no arguments + are given the context will be empty. These two calls do the same:: + + template.render(knights='that say nih') + template.render({'knights': 'that say nih'}) + + This will return the rendered template as unicode string. + """ + vars = dict(*args, **kwargs) + try: + return concat(self.root_render_func(self.new_context(vars))) + except Exception: + self.environment.handle_exception() + + def render_async(self, *args, **kwargs): + """This works similar to :meth:`render` but returns a coroutine + that when awaited returns the entire rendered template string. This + requires the async feature to be enabled. + + Example usage:: + + await template.render_async(knights='that say nih; asynchronously') + """ + # see asyncsupport for the actual implementation + raise NotImplementedError( + "This feature is not available for this version of Python" + ) + + def stream(self, *args, **kwargs): + """Works exactly like :meth:`generate` but returns a + :class:`TemplateStream`. + """ + return TemplateStream(self.generate(*args, **kwargs)) + + def generate(self, *args, **kwargs): + """For very large templates it can be useful to not render the whole + template at once but evaluate each statement after another and yield + piece for piece. This method basically does exactly that and returns + a generator that yields one item after another as unicode strings. + + It accepts the same arguments as :meth:`render`. + """ + vars = dict(*args, **kwargs) + try: + for event in self.root_render_func(self.new_context(vars)): + yield event + except Exception: + yield self.environment.handle_exception() + + def generate_async(self, *args, **kwargs): + """An async version of :meth:`generate`. Works very similarly but + returns an async iterator instead. + """ + # see asyncsupport for the actual implementation + raise NotImplementedError( + "This feature is not available for this version of Python" + ) + + def new_context(self, vars=None, shared=False, locals=None): + """Create a new :class:`Context` for this template. The vars + provided will be passed to the template. Per default the globals + are added to the context. If shared is set to `True` the data + is passed as is to the context without adding the globals. + + `locals` can be a dict of local variables for internal usage. + """ + return new_context( + self.environment, self.name, self.blocks, vars, shared, self.globals, locals + ) + + def make_module(self, vars=None, shared=False, locals=None): + """This method works like the :attr:`module` attribute when called + without arguments but it will evaluate the template on every call + rather than caching it. It's also possible to provide + a dict which is then used as context. The arguments are the same + as for the :meth:`new_context` method. + """ + return TemplateModule(self, self.new_context(vars, shared, locals)) + + def make_module_async(self, vars=None, shared=False, locals=None): + """As template module creation can invoke template code for + asynchronous executions this method must be used instead of the + normal :meth:`make_module` one. Likewise the module attribute + becomes unavailable in async mode. + """ + # see asyncsupport for the actual implementation + raise NotImplementedError( + "This feature is not available for this version of Python" + ) + + @internalcode + def _get_default_module(self): + if self._module is not None: + return self._module + self._module = rv = self.make_module() + return rv + + @property + def module(self): + """The template as module. This is used for imports in the + template runtime but is also useful if one wants to access + exported template variables from the Python layer: + + >>> t = Template('{% macro foo() %}42{% endmacro %}23') + >>> str(t.module) + '23' + >>> t.module.foo() == u'42' + True + + This attribute is not available if async mode is enabled. + """ + return self._get_default_module() + + def get_corresponding_lineno(self, lineno): + """Return the source line number of a line number in the + generated bytecode as they are not in sync. + """ + for template_line, code_line in reversed(self.debug_info): + if code_line <= lineno: + return template_line + return 1 + + @property + def is_up_to_date(self): + """If this variable is `False` there is a newer version available.""" + if self._uptodate is None: + return True + return self._uptodate() + + @property + def debug_info(self): + """The debug info mapping.""" + if self._debug_info: + return [tuple(map(int, x.split("="))) for x in self._debug_info.split("&")] + return [] + + def __repr__(self): + if self.name is None: + name = "memory:%x" % id(self) + else: + name = repr(self.name) + return "<%s %s>" % (self.__class__.__name__, name) + + +@implements_to_string +class TemplateModule(object): + """Represents an imported template. All the exported names of the + template are available as attributes on this object. Additionally + converting it into an unicode- or bytestrings renders the contents. + """ + + def __init__(self, template, context, body_stream=None): + if body_stream is None: + if context.environment.is_async: + raise RuntimeError( + "Async mode requires a body stream " + "to be passed to a template module. Use " + "the async methods of the API you are " + "using." + ) + body_stream = list(template.root_render_func(context)) + self._body_stream = body_stream + self.__dict__.update(context.get_exported()) + self.__name__ = template.name + + def __html__(self): + return Markup(concat(self._body_stream)) + + def __str__(self): + return concat(self._body_stream) + + def __repr__(self): + if self.__name__ is None: + name = "memory:%x" % id(self) + else: + name = repr(self.__name__) + return "<%s %s>" % (self.__class__.__name__, name) + + +class TemplateExpression(object): + """The :meth:`jinja2.Environment.compile_expression` method returns an + instance of this object. It encapsulates the expression-like access + to the template with an expression it wraps. + """ + + def __init__(self, template, undefined_to_none): + self._template = template + self._undefined_to_none = undefined_to_none + + def __call__(self, *args, **kwargs): + context = self._template.new_context(dict(*args, **kwargs)) + consume(self._template.root_render_func(context)) + rv = context.vars["result"] + if self._undefined_to_none and isinstance(rv, Undefined): + rv = None + return rv + + +@implements_iterator +class TemplateStream(object): + """A template stream works pretty much like an ordinary python generator + but it can buffer multiple items to reduce the number of total iterations. + Per default the output is unbuffered which means that for every unbuffered + instruction in the template one unicode string is yielded. + + If buffering is enabled with a buffer size of 5, five items are combined + into a new unicode string. This is mainly useful if you are streaming + big templates to a client via WSGI which flushes after each iteration. + """ + + def __init__(self, gen): + self._gen = gen + self.disable_buffering() + + def dump(self, fp, encoding=None, errors="strict"): + """Dump the complete stream into a file or file-like object. + Per default unicode strings are written, if you want to encode + before writing specify an `encoding`. + + Example usage:: + + Template('Hello {{ name }}!').stream(name='foo').dump('hello.html') + """ + close = False + if isinstance(fp, string_types): + if encoding is None: + encoding = "utf-8" + fp = open(fp, "wb") + close = True + try: + if encoding is not None: + iterable = (x.encode(encoding, errors) for x in self) + else: + iterable = self + if hasattr(fp, "writelines"): + fp.writelines(iterable) + else: + for item in iterable: + fp.write(item) + finally: + if close: + fp.close() + + def disable_buffering(self): + """Disable the output buffering.""" + self._next = partial(next, self._gen) + self.buffered = False + + def _buffered_generator(self, size): + buf = [] + c_size = 0 + push = buf.append + + while 1: + try: + while c_size < size: + c = next(self._gen) + push(c) + if c: + c_size += 1 + except StopIteration: + if not c_size: + return + yield concat(buf) + del buf[:] + c_size = 0 + + def enable_buffering(self, size=5): + """Enable buffering. Buffer `size` items before yielding them.""" + if size <= 1: + raise ValueError("buffer size too small") + + self.buffered = True + self._next = partial(next, self._buffered_generator(size)) + + def __iter__(self): + return self + + def __next__(self): + return self._next() + + +# hook in default template class. if anyone reads this comment: ignore that +# it's possible to use custom templates ;-) +Environment.template_class = Template diff --git a/contrib/python/Jinja2/py2/jinja2/exceptions.py b/contrib/python/Jinja2/py2/jinja2/exceptions.py new file mode 100644 index 0000000000..0bf2003e30 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/exceptions.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +from ._compat import imap +from ._compat import implements_to_string +from ._compat import PY2 +from ._compat import text_type + + +class TemplateError(Exception): + """Baseclass for all template errors.""" + + if PY2: + + def __init__(self, message=None): + if message is not None: + message = text_type(message).encode("utf-8") + Exception.__init__(self, message) + + @property + def message(self): + if self.args: + message = self.args[0] + if message is not None: + return message.decode("utf-8", "replace") + + def __unicode__(self): + return self.message or u"" + + else: + + def __init__(self, message=None): + Exception.__init__(self, message) + + @property + def message(self): + if self.args: + message = self.args[0] + if message is not None: + return message + + +@implements_to_string +class TemplateNotFound(IOError, LookupError, TemplateError): + """Raised if a template does not exist. + + .. versionchanged:: 2.11 + If the given name is :class:`Undefined` and no message was + provided, an :exc:`UndefinedError` is raised. + """ + + # looks weird, but removes the warning descriptor that just + # bogusly warns us about message being deprecated + message = None + + def __init__(self, name, message=None): + IOError.__init__(self, name) + + if message is None: + from .runtime import Undefined + + if isinstance(name, Undefined): + name._fail_with_undefined_error() + + message = name + + self.message = message + self.name = name + self.templates = [name] + + def __str__(self): + return self.message + + +class TemplatesNotFound(TemplateNotFound): + """Like :class:`TemplateNotFound` but raised if multiple templates + are selected. This is a subclass of :class:`TemplateNotFound` + exception, so just catching the base exception will catch both. + + .. versionchanged:: 2.11 + If a name in the list of names is :class:`Undefined`, a message + about it being undefined is shown rather than the empty string. + + .. versionadded:: 2.2 + """ + + def __init__(self, names=(), message=None): + if message is None: + from .runtime import Undefined + + parts = [] + + for name in names: + if isinstance(name, Undefined): + parts.append(name._undefined_message) + else: + parts.append(name) + + message = u"none of the templates given were found: " + u", ".join( + imap(text_type, parts) + ) + TemplateNotFound.__init__(self, names and names[-1] or None, message) + self.templates = list(names) + + +@implements_to_string +class TemplateSyntaxError(TemplateError): + """Raised to tell the user that there is a problem with the template.""" + + def __init__(self, message, lineno, name=None, filename=None): + TemplateError.__init__(self, message) + self.lineno = lineno + self.name = name + self.filename = filename + self.source = None + + # this is set to True if the debug.translate_syntax_error + # function translated the syntax error into a new traceback + self.translated = False + + def __str__(self): + # for translated errors we only return the message + if self.translated: + return self.message + + # otherwise attach some stuff + location = "line %d" % self.lineno + name = self.filename or self.name + if name: + location = 'File "%s", %s' % (name, location) + lines = [self.message, " " + location] + + # if the source is set, add the line to the output + if self.source is not None: + try: + line = self.source.splitlines()[self.lineno - 1] + except IndexError: + line = None + if line: + lines.append(" " + line.strip()) + + return u"\n".join(lines) + + def __reduce__(self): + # https://bugs.python.org/issue1692335 Exceptions that take + # multiple required arguments have problems with pickling. + # Without this, raises TypeError: __init__() missing 1 required + # positional argument: 'lineno' + return self.__class__, (self.message, self.lineno, self.name, self.filename) + + +class TemplateAssertionError(TemplateSyntaxError): + """Like a template syntax error, but covers cases where something in the + template caused an error at compile time that wasn't necessarily caused + by a syntax error. However it's a direct subclass of + :exc:`TemplateSyntaxError` and has the same attributes. + """ + + +class TemplateRuntimeError(TemplateError): + """A generic runtime error in the template engine. Under some situations + Jinja may raise this exception. + """ + + +class UndefinedError(TemplateRuntimeError): + """Raised if a template tries to operate on :class:`Undefined`.""" + + +class SecurityError(TemplateRuntimeError): + """Raised if a template tries to do something insecure if the + sandbox is enabled. + """ + + +class FilterArgumentError(TemplateRuntimeError): + """This error is raised if a filter was called with inappropriate + arguments + """ diff --git a/contrib/python/Jinja2/py2/jinja2/ext.py b/contrib/python/Jinja2/py2/jinja2/ext.py new file mode 100644 index 0000000000..9141be4dac --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/ext.py @@ -0,0 +1,704 @@ +# -*- coding: utf-8 -*- +"""Extension API for adding custom tags and behavior.""" +import pprint +import re +from sys import version_info + +from markupsafe import Markup + +from . import nodes +from ._compat import iteritems +from ._compat import string_types +from ._compat import with_metaclass +from .defaults import BLOCK_END_STRING +from .defaults import BLOCK_START_STRING +from .defaults import COMMENT_END_STRING +from .defaults import COMMENT_START_STRING +from .defaults import KEEP_TRAILING_NEWLINE +from .defaults import LINE_COMMENT_PREFIX +from .defaults import LINE_STATEMENT_PREFIX +from .defaults import LSTRIP_BLOCKS +from .defaults import NEWLINE_SEQUENCE +from .defaults import TRIM_BLOCKS +from .defaults import VARIABLE_END_STRING +from .defaults import VARIABLE_START_STRING +from .environment import Environment +from .exceptions import TemplateAssertionError +from .exceptions import TemplateSyntaxError +from .nodes import ContextReference +from .runtime import concat +from .utils import contextfunction +from .utils import import_string + +# the only real useful gettext functions for a Jinja template. Note +# that ugettext must be assigned to gettext as Jinja doesn't support +# non unicode strings. +GETTEXT_FUNCTIONS = ("_", "gettext", "ngettext") + +_ws_re = re.compile(r"\s*\n\s*") + + +class ExtensionRegistry(type): + """Gives the extension an unique identifier.""" + + def __new__(mcs, name, bases, d): + rv = type.__new__(mcs, name, bases, d) + rv.identifier = rv.__module__ + "." + rv.__name__ + return rv + + +class Extension(with_metaclass(ExtensionRegistry, object)): + """Extensions can be used to add extra functionality to the Jinja template + system at the parser level. Custom extensions are bound to an environment + but may not store environment specific data on `self`. The reason for + this is that an extension can be bound to another environment (for + overlays) by creating a copy and reassigning the `environment` attribute. + + As extensions are created by the environment they cannot accept any + arguments for configuration. One may want to work around that by using + a factory function, but that is not possible as extensions are identified + by their import name. The correct way to configure the extension is + storing the configuration values on the environment. Because this way the + environment ends up acting as central configuration storage the + attributes may clash which is why extensions have to ensure that the names + they choose for configuration are not too generic. ``prefix`` for example + is a terrible name, ``fragment_cache_prefix`` on the other hand is a good + name as includes the name of the extension (fragment cache). + """ + + #: if this extension parses this is the list of tags it's listening to. + tags = set() + + #: the priority of that extension. This is especially useful for + #: extensions that preprocess values. A lower value means higher + #: priority. + #: + #: .. versionadded:: 2.4 + priority = 100 + + def __init__(self, environment): + self.environment = environment + + def bind(self, environment): + """Create a copy of this extension bound to another environment.""" + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.environment = environment + return rv + + def preprocess(self, source, name, filename=None): + """This method is called before the actual lexing and can be used to + preprocess the source. The `filename` is optional. The return value + must be the preprocessed source. + """ + return source + + def filter_stream(self, stream): + """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used + to filter tokens returned. This method has to return an iterable of + :class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a + :class:`~jinja2.lexer.TokenStream`. + """ + return stream + + def parse(self, parser): + """If any of the :attr:`tags` matched this method is called with the + parser as first argument. The token the parser stream is pointing at + is the name token that matched. This method has to return one or a + list of multiple nodes. + """ + raise NotImplementedError() + + def attr(self, name, lineno=None): + """Return an attribute node for the current extension. This is useful + to pass constants on extensions to generated template code. + + :: + + self.attr('_my_attribute', lineno=lineno) + """ + return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno) + + def call_method( + self, name, args=None, kwargs=None, dyn_args=None, dyn_kwargs=None, lineno=None + ): + """Call a method of the extension. This is a shortcut for + :meth:`attr` + :class:`jinja2.nodes.Call`. + """ + if args is None: + args = [] + if kwargs is None: + kwargs = [] + return nodes.Call( + self.attr(name, lineno=lineno), + args, + kwargs, + dyn_args, + dyn_kwargs, + lineno=lineno, + ) + + +@contextfunction +def _gettext_alias(__context, *args, **kwargs): + return __context.call(__context.resolve("gettext"), *args, **kwargs) + + +def _make_new_gettext(func): + @contextfunction + def gettext(__context, __string, **variables): + rv = __context.call(func, __string) + if __context.eval_ctx.autoescape: + rv = Markup(rv) + # Always treat as a format string, even if there are no + # variables. This makes translation strings more consistent + # and predictable. This requires escaping + return rv % variables + + return gettext + + +def _make_new_ngettext(func): + @contextfunction + def ngettext(__context, __singular, __plural, __num, **variables): + variables.setdefault("num", __num) + rv = __context.call(func, __singular, __plural, __num) + if __context.eval_ctx.autoescape: + rv = Markup(rv) + # Always treat as a format string, see gettext comment above. + return rv % variables + + return ngettext + + +class InternationalizationExtension(Extension): + """This extension adds gettext support to Jinja.""" + + tags = {"trans"} + + # TODO: the i18n extension is currently reevaluating values in a few + # situations. Take this example: + # {% trans count=something() %}{{ count }} foo{% pluralize + # %}{{ count }} fooss{% endtrans %} + # something is called twice here. One time for the gettext value and + # the other time for the n-parameter of the ngettext function. + + def __init__(self, environment): + Extension.__init__(self, environment) + environment.globals["_"] = _gettext_alias + environment.extend( + install_gettext_translations=self._install, + install_null_translations=self._install_null, + install_gettext_callables=self._install_callables, + uninstall_gettext_translations=self._uninstall, + extract_translations=self._extract, + newstyle_gettext=False, + ) + + def _install(self, translations, newstyle=None): + gettext = getattr(translations, "ugettext", None) + if gettext is None: + gettext = translations.gettext + ngettext = getattr(translations, "ungettext", None) + if ngettext is None: + ngettext = translations.ngettext + self._install_callables(gettext, ngettext, newstyle) + + def _install_null(self, newstyle=None): + self._install_callables( + lambda x: x, lambda s, p, n: (n != 1 and (p,) or (s,))[0], newstyle + ) + + def _install_callables(self, gettext, ngettext, newstyle=None): + if newstyle is not None: + self.environment.newstyle_gettext = newstyle + if self.environment.newstyle_gettext: + gettext = _make_new_gettext(gettext) + ngettext = _make_new_ngettext(ngettext) + self.environment.globals.update(gettext=gettext, ngettext=ngettext) + + def _uninstall(self, translations): + for key in "gettext", "ngettext": + self.environment.globals.pop(key, None) + + def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS): + if isinstance(source, string_types): + source = self.environment.parse(source) + return extract_from_ast(source, gettext_functions) + + def parse(self, parser): + """Parse a translatable tag.""" + lineno = next(parser.stream).lineno + num_called_num = False + + # find all the variables referenced. Additionally a variable can be + # defined in the body of the trans block too, but this is checked at + # a later state. + plural_expr = None + plural_expr_assignment = None + variables = {} + trimmed = None + while parser.stream.current.type != "block_end": + if variables: + parser.stream.expect("comma") + + # skip colon for python compatibility + if parser.stream.skip_if("colon"): + break + + name = parser.stream.expect("name") + if name.value in variables: + parser.fail( + "translatable variable %r defined twice." % name.value, + name.lineno, + exc=TemplateAssertionError, + ) + + # expressions + if parser.stream.current.type == "assign": + next(parser.stream) + variables[name.value] = var = parser.parse_expression() + elif trimmed is None and name.value in ("trimmed", "notrimmed"): + trimmed = name.value == "trimmed" + continue + else: + variables[name.value] = var = nodes.Name(name.value, "load") + + if plural_expr is None: + if isinstance(var, nodes.Call): + plural_expr = nodes.Name("_trans", "load") + variables[name.value] = plural_expr + plural_expr_assignment = nodes.Assign( + nodes.Name("_trans", "store"), var + ) + else: + plural_expr = var + num_called_num = name.value == "num" + + parser.stream.expect("block_end") + + plural = None + have_plural = False + referenced = set() + + # now parse until endtrans or pluralize + singular_names, singular = self._parse_block(parser, True) + if singular_names: + referenced.update(singular_names) + if plural_expr is None: + plural_expr = nodes.Name(singular_names[0], "load") + num_called_num = singular_names[0] == "num" + + # if we have a pluralize block, we parse that too + if parser.stream.current.test("name:pluralize"): + have_plural = True + next(parser.stream) + if parser.stream.current.type != "block_end": + name = parser.stream.expect("name") + if name.value not in variables: + parser.fail( + "unknown variable %r for pluralization" % name.value, + name.lineno, + exc=TemplateAssertionError, + ) + plural_expr = variables[name.value] + num_called_num = name.value == "num" + parser.stream.expect("block_end") + plural_names, plural = self._parse_block(parser, False) + next(parser.stream) + referenced.update(plural_names) + else: + next(parser.stream) + + # register free names as simple name expressions + for var in referenced: + if var not in variables: + variables[var] = nodes.Name(var, "load") + + if not have_plural: + plural_expr = None + elif plural_expr is None: + parser.fail("pluralize without variables", lineno) + + if trimmed is None: + trimmed = self.environment.policies["ext.i18n.trimmed"] + if trimmed: + singular = self._trim_whitespace(singular) + if plural: + plural = self._trim_whitespace(plural) + + node = self._make_node( + singular, + plural, + variables, + plural_expr, + bool(referenced), + num_called_num and have_plural, + ) + node.set_lineno(lineno) + if plural_expr_assignment is not None: + return [plural_expr_assignment, node] + else: + return node + + def _trim_whitespace(self, string, _ws_re=_ws_re): + return _ws_re.sub(" ", string.strip()) + + def _parse_block(self, parser, allow_pluralize): + """Parse until the next block tag with a given name.""" + referenced = [] + buf = [] + while 1: + if parser.stream.current.type == "data": + buf.append(parser.stream.current.value.replace("%", "%%")) + next(parser.stream) + elif parser.stream.current.type == "variable_begin": + next(parser.stream) + name = parser.stream.expect("name").value + referenced.append(name) + buf.append("%%(%s)s" % name) + parser.stream.expect("variable_end") + elif parser.stream.current.type == "block_begin": + next(parser.stream) + if parser.stream.current.test("name:endtrans"): + break + elif parser.stream.current.test("name:pluralize"): + if allow_pluralize: + break + parser.fail( + "a translatable section can have only one pluralize section" + ) + parser.fail( + "control structures in translatable sections are not allowed" + ) + elif parser.stream.eos: + parser.fail("unclosed translation block") + else: + raise RuntimeError("internal parser error") + + return referenced, concat(buf) + + def _make_node( + self, singular, plural, variables, plural_expr, vars_referenced, num_called_num + ): + """Generates a useful node from the data provided.""" + # no variables referenced? no need to escape for old style + # gettext invocations only if there are vars. + if not vars_referenced and not self.environment.newstyle_gettext: + singular = singular.replace("%%", "%") + if plural: + plural = plural.replace("%%", "%") + + # singular only: + if plural_expr is None: + gettext = nodes.Name("gettext", "load") + node = nodes.Call(gettext, [nodes.Const(singular)], [], None, None) + + # singular and plural + else: + ngettext = nodes.Name("ngettext", "load") + node = nodes.Call( + ngettext, + [nodes.Const(singular), nodes.Const(plural), plural_expr], + [], + None, + None, + ) + + # in case newstyle gettext is used, the method is powerful + # enough to handle the variable expansion and autoescape + # handling itself + if self.environment.newstyle_gettext: + for key, value in iteritems(variables): + # the function adds that later anyways in case num was + # called num, so just skip it. + if num_called_num and key == "num": + continue + node.kwargs.append(nodes.Keyword(key, value)) + + # otherwise do that here + else: + # mark the return value as safe if we are in an + # environment with autoescaping turned on + node = nodes.MarkSafeIfAutoescape(node) + if variables: + node = nodes.Mod( + node, + nodes.Dict( + [ + nodes.Pair(nodes.Const(key), value) + for key, value in variables.items() + ] + ), + ) + return nodes.Output([node]) + + +class ExprStmtExtension(Extension): + """Adds a `do` tag to Jinja that works like the print statement just + that it doesn't print the return value. + """ + + tags = set(["do"]) + + def parse(self, parser): + node = nodes.ExprStmt(lineno=next(parser.stream).lineno) + node.node = parser.parse_tuple() + return node + + +class LoopControlExtension(Extension): + """Adds break and continue to the template engine.""" + + tags = set(["break", "continue"]) + + def parse(self, parser): + token = next(parser.stream) + if token.value == "break": + return nodes.Break(lineno=token.lineno) + return nodes.Continue(lineno=token.lineno) + + +class WithExtension(Extension): + pass + + +class AutoEscapeExtension(Extension): + pass + + +class DebugExtension(Extension): + """A ``{% debug %}`` tag that dumps the available variables, + filters, and tests. + + .. code-block:: html+jinja + + <pre>{% debug %}</pre> + + .. code-block:: text + + {'context': {'cycler': <class 'jinja2.utils.Cycler'>, + ..., + 'namespace': <class 'jinja2.utils.Namespace'>}, + 'filters': ['abs', 'attr', 'batch', 'capitalize', 'center', 'count', 'd', + ..., 'urlencode', 'urlize', 'wordcount', 'wordwrap', 'xmlattr'], + 'tests': ['!=', '<', '<=', '==', '>', '>=', 'callable', 'defined', + ..., 'odd', 'sameas', 'sequence', 'string', 'undefined', 'upper']} + + .. versionadded:: 2.11.0 + """ + + tags = {"debug"} + + def parse(self, parser): + lineno = parser.stream.expect("name:debug").lineno + context = ContextReference() + result = self.call_method("_render", [context], lineno=lineno) + return nodes.Output([result], lineno=lineno) + + def _render(self, context): + result = { + "context": context.get_all(), + "filters": sorted(self.environment.filters.keys()), + "tests": sorted(self.environment.tests.keys()), + } + + # Set the depth since the intent is to show the top few names. + if version_info[:2] >= (3, 4): + return pprint.pformat(result, depth=3, compact=True) + else: + return pprint.pformat(result, depth=3) + + +def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, babel_style=True): + """Extract localizable strings from the given template node. Per + default this function returns matches in babel style that means non string + parameters as well as keyword arguments are returned as `None`. This + allows Babel to figure out what you really meant if you are using + gettext functions that allow keyword arguments for placeholder expansion. + If you don't want that behavior set the `babel_style` parameter to `False` + which causes only strings to be returned and parameters are always stored + in tuples. As a consequence invalid gettext calls (calls without a single + string parameter or string parameters after non-string parameters) are + skipped. + + This example explains the behavior: + + >>> from jinja2 import Environment + >>> env = Environment() + >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}') + >>> list(extract_from_ast(node)) + [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))] + >>> list(extract_from_ast(node, babel_style=False)) + [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))] + + For every string found this function yields a ``(lineno, function, + message)`` tuple, where: + + * ``lineno`` is the number of the line on which the string was found, + * ``function`` is the name of the ``gettext`` function used (if the + string was extracted from embedded Python code), and + * ``message`` is the string itself (a ``unicode`` object, or a tuple + of ``unicode`` objects for functions with multiple string arguments). + + This extraction function operates on the AST and is because of that unable + to extract any comments. For comment support you have to use the babel + extraction interface or extract comments yourself. + """ + for node in node.find_all(nodes.Call): + if ( + not isinstance(node.node, nodes.Name) + or node.node.name not in gettext_functions + ): + continue + + strings = [] + for arg in node.args: + if isinstance(arg, nodes.Const) and isinstance(arg.value, string_types): + strings.append(arg.value) + else: + strings.append(None) + + for _ in node.kwargs: + strings.append(None) + if node.dyn_args is not None: + strings.append(None) + if node.dyn_kwargs is not None: + strings.append(None) + + if not babel_style: + strings = tuple(x for x in strings if x is not None) + if not strings: + continue + else: + if len(strings) == 1: + strings = strings[0] + else: + strings = tuple(strings) + yield node.lineno, node.node.name, strings + + +class _CommentFinder(object): + """Helper class to find comments in a token stream. Can only + find comments for gettext calls forwards. Once the comment + from line 4 is found, a comment for line 1 will not return a + usable value. + """ + + def __init__(self, tokens, comment_tags): + self.tokens = tokens + self.comment_tags = comment_tags + self.offset = 0 + self.last_lineno = 0 + + def find_backwards(self, offset): + try: + for _, token_type, token_value in reversed( + self.tokens[self.offset : offset] + ): + if token_type in ("comment", "linecomment"): + try: + prefix, comment = token_value.split(None, 1) + except ValueError: + continue + if prefix in self.comment_tags: + return [comment.rstrip()] + return [] + finally: + self.offset = offset + + def find_comments(self, lineno): + if not self.comment_tags or self.last_lineno > lineno: + return [] + for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset :]): + if token_lineno > lineno: + return self.find_backwards(self.offset + idx) + return self.find_backwards(len(self.tokens)) + + +def babel_extract(fileobj, keywords, comment_tags, options): + """Babel extraction method for Jinja templates. + + .. versionchanged:: 2.3 + Basic support for translation comments was added. If `comment_tags` + is now set to a list of keywords for extraction, the extractor will + try to find the best preceding comment that begins with one of the + keywords. For best results, make sure to not have more than one + gettext call in one line of code and the matching comment in the + same line or the line before. + + .. versionchanged:: 2.5.1 + The `newstyle_gettext` flag can be set to `True` to enable newstyle + gettext calls. + + .. versionchanged:: 2.7 + A `silent` option can now be provided. If set to `False` template + syntax errors are propagated instead of being ignored. + + :param fileobj: the file-like object the messages should be extracted from + :param keywords: a list of keywords (i.e. function names) that should be + recognized as translation functions + :param comment_tags: a list of translator tags to search for and include + in the results. + :param options: a dictionary of additional options (optional) + :return: an iterator over ``(lineno, funcname, message, comments)`` tuples. + (comments will be empty currently) + """ + extensions = set() + for extension in options.get("extensions", "").split(","): + extension = extension.strip() + if not extension: + continue + extensions.add(import_string(extension)) + if InternationalizationExtension not in extensions: + extensions.add(InternationalizationExtension) + + def getbool(options, key, default=False): + return options.get(key, str(default)).lower() in ("1", "on", "yes", "true") + + silent = getbool(options, "silent", True) + environment = Environment( + options.get("block_start_string", BLOCK_START_STRING), + options.get("block_end_string", BLOCK_END_STRING), + options.get("variable_start_string", VARIABLE_START_STRING), + options.get("variable_end_string", VARIABLE_END_STRING), + options.get("comment_start_string", COMMENT_START_STRING), + options.get("comment_end_string", COMMENT_END_STRING), + options.get("line_statement_prefix") or LINE_STATEMENT_PREFIX, + options.get("line_comment_prefix") or LINE_COMMENT_PREFIX, + getbool(options, "trim_blocks", TRIM_BLOCKS), + getbool(options, "lstrip_blocks", LSTRIP_BLOCKS), + NEWLINE_SEQUENCE, + getbool(options, "keep_trailing_newline", KEEP_TRAILING_NEWLINE), + frozenset(extensions), + cache_size=0, + auto_reload=False, + ) + + if getbool(options, "trimmed"): + environment.policies["ext.i18n.trimmed"] = True + if getbool(options, "newstyle_gettext"): + environment.newstyle_gettext = True + + source = fileobj.read().decode(options.get("encoding", "utf-8")) + try: + node = environment.parse(source) + tokens = list(environment.lex(environment.preprocess(source))) + except TemplateSyntaxError: + if not silent: + raise + # skip templates with syntax errors + return + + finder = _CommentFinder(tokens, comment_tags) + for lineno, func, message in extract_from_ast(node, keywords): + yield lineno, func, message, finder.find_comments(lineno) + + +#: nicer import names +i18n = InternationalizationExtension +do = ExprStmtExtension +loopcontrols = LoopControlExtension +with_ = WithExtension +autoescape = AutoEscapeExtension +debug = DebugExtension diff --git a/contrib/python/Jinja2/py2/jinja2/filters.py b/contrib/python/Jinja2/py2/jinja2/filters.py new file mode 100644 index 0000000000..74b108dcec --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/filters.py @@ -0,0 +1,1382 @@ +# -*- coding: utf-8 -*- +"""Built-in template filters used with the ``|`` operator.""" +import math +import random +import re +import warnings +from collections import namedtuple +from itertools import chain +from itertools import groupby + +from markupsafe import escape +from markupsafe import Markup +from markupsafe import soft_unicode + +from ._compat import abc +from ._compat import imap +from ._compat import iteritems +from ._compat import string_types +from ._compat import text_type +from .exceptions import FilterArgumentError +from .runtime import Undefined +from .utils import htmlsafe_json_dumps +from .utils import pformat +from .utils import unicode_urlencode +from .utils import urlize + +_word_re = re.compile(r"\w+", re.UNICODE) +_word_beginning_split_re = re.compile(r"([-\s\(\{\[\<]+)", re.UNICODE) + + +def contextfilter(f): + """Decorator for marking context dependent filters. The current + :class:`Context` will be passed as first argument. + """ + f.contextfilter = True + return f + + +def evalcontextfilter(f): + """Decorator for marking eval-context dependent filters. An eval + context object is passed as first argument. For more information + about the eval context, see :ref:`eval-context`. + + .. versionadded:: 2.4 + """ + f.evalcontextfilter = True + return f + + +def environmentfilter(f): + """Decorator for marking environment dependent filters. The current + :class:`Environment` is passed to the filter as first argument. + """ + f.environmentfilter = True + return f + + +def ignore_case(value): + """For use as a postprocessor for :func:`make_attrgetter`. Converts strings + to lowercase and returns other types as-is.""" + return value.lower() if isinstance(value, string_types) else value + + +def make_attrgetter(environment, attribute, postprocess=None, default=None): + """Returns a callable that looks up the given attribute from a + passed object with the rules of the environment. Dots are allowed + to access attributes of attributes. Integer parts in paths are + looked up as integers. + """ + attribute = _prepare_attribute_parts(attribute) + + def attrgetter(item): + for part in attribute: + item = environment.getitem(item, part) + + if default and isinstance(item, Undefined): + item = default + + if postprocess is not None: + item = postprocess(item) + + return item + + return attrgetter + + +def make_multi_attrgetter(environment, attribute, postprocess=None): + """Returns a callable that looks up the given comma separated + attributes from a passed object with the rules of the environment. + Dots are allowed to access attributes of each attribute. Integer + parts in paths are looked up as integers. + + The value returned by the returned callable is a list of extracted + attribute values. + + Examples of attribute: "attr1,attr2", "attr1.inner1.0,attr2.inner2.0", etc. + """ + attribute_parts = ( + attribute.split(",") if isinstance(attribute, string_types) else [attribute] + ) + attribute = [ + _prepare_attribute_parts(attribute_part) for attribute_part in attribute_parts + ] + + def attrgetter(item): + items = [None] * len(attribute) + for i, attribute_part in enumerate(attribute): + item_i = item + for part in attribute_part: + item_i = environment.getitem(item_i, part) + + if postprocess is not None: + item_i = postprocess(item_i) + + items[i] = item_i + return items + + return attrgetter + + +def _prepare_attribute_parts(attr): + if attr is None: + return [] + elif isinstance(attr, string_types): + return [int(x) if x.isdigit() else x for x in attr.split(".")] + else: + return [attr] + + +def do_forceescape(value): + """Enforce HTML escaping. This will probably double escape variables.""" + if hasattr(value, "__html__"): + value = value.__html__() + return escape(text_type(value)) + + +def do_urlencode(value): + """Quote data for use in a URL path or query using UTF-8. + + Basic wrapper around :func:`urllib.parse.quote` when given a + string, or :func:`urllib.parse.urlencode` for a dict or iterable. + + :param value: Data to quote. A string will be quoted directly. A + dict or iterable of ``(key, value)`` pairs will be joined as a + query string. + + When given a string, "/" is not quoted. HTTP servers treat "/" and + "%2F" equivalently in paths. If you need quoted slashes, use the + ``|replace("/", "%2F")`` filter. + + .. versionadded:: 2.7 + """ + if isinstance(value, string_types) or not isinstance(value, abc.Iterable): + return unicode_urlencode(value) + + if isinstance(value, dict): + items = iteritems(value) + else: + items = iter(value) + + return u"&".join( + "%s=%s" % (unicode_urlencode(k, for_qs=True), unicode_urlencode(v, for_qs=True)) + for k, v in items + ) + + +@evalcontextfilter +def do_replace(eval_ctx, s, old, new, count=None): + """Return a copy of the value with all occurrences of a substring + replaced with a new one. The first argument is the substring + that should be replaced, the second is the replacement string. + If the optional third argument ``count`` is given, only the first + ``count`` occurrences are replaced: + + .. sourcecode:: jinja + + {{ "Hello World"|replace("Hello", "Goodbye") }} + -> Goodbye World + + {{ "aaaaargh"|replace("a", "d'oh, ", 2) }} + -> d'oh, d'oh, aaargh + """ + if count is None: + count = -1 + if not eval_ctx.autoescape: + return text_type(s).replace(text_type(old), text_type(new), count) + if ( + hasattr(old, "__html__") + or hasattr(new, "__html__") + and not hasattr(s, "__html__") + ): + s = escape(s) + else: + s = soft_unicode(s) + return s.replace(soft_unicode(old), soft_unicode(new), count) + + +def do_upper(s): + """Convert a value to uppercase.""" + return soft_unicode(s).upper() + + +def do_lower(s): + """Convert a value to lowercase.""" + return soft_unicode(s).lower() + + +@evalcontextfilter +def do_xmlattr(_eval_ctx, d, autospace=True): + """Create an SGML/XML attribute string based on the items in a dict. + All values that are neither `none` nor `undefined` are automatically + escaped: + + .. sourcecode:: html+jinja + + <ul{{ {'class': 'my_list', 'missing': none, + 'id': 'list-%d'|format(variable)}|xmlattr }}> + ... + </ul> + + Results in something like this: + + .. sourcecode:: html + + <ul class="my_list" id="list-42"> + ... + </ul> + + As you can see it automatically prepends a space in front of the item + if the filter returned something unless the second parameter is false. + """ + rv = u" ".join( + u'%s="%s"' % (escape(key), escape(value)) + for key, value in iteritems(d) + if value is not None and not isinstance(value, Undefined) + ) + if autospace and rv: + rv = u" " + rv + if _eval_ctx.autoescape: + rv = Markup(rv) + return rv + + +def do_capitalize(s): + """Capitalize a value. The first character will be uppercase, all others + lowercase. + """ + return soft_unicode(s).capitalize() + + +def do_title(s): + """Return a titlecased version of the value. I.e. words will start with + uppercase letters, all remaining characters are lowercase. + """ + return "".join( + [ + item[0].upper() + item[1:].lower() + for item in _word_beginning_split_re.split(soft_unicode(s)) + if item + ] + ) + + +def do_dictsort(value, case_sensitive=False, by="key", reverse=False): + """Sort a dict and yield (key, value) pairs. Because python dicts are + unsorted you may want to use this function to order them by either + key or value: + + .. sourcecode:: jinja + + {% for key, value in mydict|dictsort %} + sort the dict by key, case insensitive + + {% for key, value in mydict|dictsort(reverse=true) %} + sort the dict by key, case insensitive, reverse order + + {% for key, value in mydict|dictsort(true) %} + sort the dict by key, case sensitive + + {% for key, value in mydict|dictsort(false, 'value') %} + sort the dict by value, case insensitive + """ + if by == "key": + pos = 0 + elif by == "value": + pos = 1 + else: + raise FilterArgumentError('You can only sort by either "key" or "value"') + + def sort_func(item): + value = item[pos] + + if not case_sensitive: + value = ignore_case(value) + + return value + + return sorted(value.items(), key=sort_func, reverse=reverse) + + +@environmentfilter +def do_sort(environment, value, reverse=False, case_sensitive=False, attribute=None): + """Sort an iterable using Python's :func:`sorted`. + + .. sourcecode:: jinja + + {% for city in cities|sort %} + ... + {% endfor %} + + :param reverse: Sort descending instead of ascending. + :param case_sensitive: When sorting strings, sort upper and lower + case separately. + :param attribute: When sorting objects or dicts, an attribute or + key to sort by. Can use dot notation like ``"address.city"``. + Can be a list of attributes like ``"age,name"``. + + The sort is stable, it does not change the relative order of + elements that compare equal. This makes it is possible to chain + sorts on different attributes and ordering. + + .. sourcecode:: jinja + + {% for user in users|sort(attribute="name") + |sort(reverse=true, attribute="age") %} + ... + {% endfor %} + + As a shortcut to chaining when the direction is the same for all + attributes, pass a comma separate list of attributes. + + .. sourcecode:: jinja + + {% for user users|sort(attribute="age,name") %} + ... + {% endfor %} + + .. versionchanged:: 2.11.0 + The ``attribute`` parameter can be a comma separated list of + attributes, e.g. ``"age,name"``. + + .. versionchanged:: 2.6 + The ``attribute`` parameter was added. + """ + key_func = make_multi_attrgetter( + environment, attribute, postprocess=ignore_case if not case_sensitive else None + ) + return sorted(value, key=key_func, reverse=reverse) + + +@environmentfilter +def do_unique(environment, value, case_sensitive=False, attribute=None): + """Returns a list of unique items from the given iterable. + + .. sourcecode:: jinja + + {{ ['foo', 'bar', 'foobar', 'FooBar']|unique|list }} + -> ['foo', 'bar', 'foobar'] + + The unique items are yielded in the same order as their first occurrence in + the iterable passed to the filter. + + :param case_sensitive: Treat upper and lower case strings as distinct. + :param attribute: Filter objects with unique values for this attribute. + """ + getter = make_attrgetter( + environment, attribute, postprocess=ignore_case if not case_sensitive else None + ) + seen = set() + + for item in value: + key = getter(item) + + if key not in seen: + seen.add(key) + yield item + + +def _min_or_max(environment, value, func, case_sensitive, attribute): + it = iter(value) + + try: + first = next(it) + except StopIteration: + return environment.undefined("No aggregated item, sequence was empty.") + + key_func = make_attrgetter( + environment, attribute, postprocess=ignore_case if not case_sensitive else None + ) + return func(chain([first], it), key=key_func) + + +@environmentfilter +def do_min(environment, value, case_sensitive=False, attribute=None): + """Return the smallest item from the sequence. + + .. sourcecode:: jinja + + {{ [1, 2, 3]|min }} + -> 1 + + :param case_sensitive: Treat upper and lower case strings as distinct. + :param attribute: Get the object with the min value of this attribute. + """ + return _min_or_max(environment, value, min, case_sensitive, attribute) + + +@environmentfilter +def do_max(environment, value, case_sensitive=False, attribute=None): + """Return the largest item from the sequence. + + .. sourcecode:: jinja + + {{ [1, 2, 3]|max }} + -> 3 + + :param case_sensitive: Treat upper and lower case strings as distinct. + :param attribute: Get the object with the max value of this attribute. + """ + return _min_or_max(environment, value, max, case_sensitive, attribute) + + +def do_default(value, default_value=u"", boolean=False): + """If the value is undefined it will return the passed default value, + otherwise the value of the variable: + + .. sourcecode:: jinja + + {{ my_variable|default('my_variable is not defined') }} + + This will output the value of ``my_variable`` if the variable was + defined, otherwise ``'my_variable is not defined'``. If you want + to use default with variables that evaluate to false you have to + set the second parameter to `true`: + + .. sourcecode:: jinja + + {{ ''|default('the string was empty', true) }} + + .. versionchanged:: 2.11 + It's now possible to configure the :class:`~jinja2.Environment` with + :class:`~jinja2.ChainableUndefined` to make the `default` filter work + on nested elements and attributes that may contain undefined values + in the chain without getting an :exc:`~jinja2.UndefinedError`. + """ + if isinstance(value, Undefined) or (boolean and not value): + return default_value + return value + + +@evalcontextfilter +def do_join(eval_ctx, value, d=u"", attribute=None): + """Return a string which is the concatenation of the strings in the + sequence. The separator between elements is an empty string per + default, you can define it with the optional parameter: + + .. sourcecode:: jinja + + {{ [1, 2, 3]|join('|') }} + -> 1|2|3 + + {{ [1, 2, 3]|join }} + -> 123 + + It is also possible to join certain attributes of an object: + + .. sourcecode:: jinja + + {{ users|join(', ', attribute='username') }} + + .. versionadded:: 2.6 + The `attribute` parameter was added. + """ + if attribute is not None: + value = imap(make_attrgetter(eval_ctx.environment, attribute), value) + + # no automatic escaping? joining is a lot easier then + if not eval_ctx.autoescape: + return text_type(d).join(imap(text_type, value)) + + # if the delimiter doesn't have an html representation we check + # if any of the items has. If yes we do a coercion to Markup + if not hasattr(d, "__html__"): + value = list(value) + do_escape = False + for idx, item in enumerate(value): + if hasattr(item, "__html__"): + do_escape = True + else: + value[idx] = text_type(item) + if do_escape: + d = escape(d) + else: + d = text_type(d) + return d.join(value) + + # no html involved, to normal joining + return soft_unicode(d).join(imap(soft_unicode, value)) + + +def do_center(value, width=80): + """Centers the value in a field of a given width.""" + return text_type(value).center(width) + + +@environmentfilter +def do_first(environment, seq): + """Return the first item of a sequence.""" + try: + return next(iter(seq)) + except StopIteration: + return environment.undefined("No first item, sequence was empty.") + + +@environmentfilter +def do_last(environment, seq): + """ + Return the last item of a sequence. + + Note: Does not work with generators. You may want to explicitly + convert it to a list: + + .. sourcecode:: jinja + + {{ data | selectattr('name', '==', 'Jinja') | list | last }} + """ + try: + return next(iter(reversed(seq))) + except StopIteration: + return environment.undefined("No last item, sequence was empty.") + + +@contextfilter +def do_random(context, seq): + """Return a random item from the sequence.""" + try: + return random.choice(seq) + except IndexError: + return context.environment.undefined("No random item, sequence was empty.") + + +def do_filesizeformat(value, binary=False): + """Format the value like a 'human-readable' file size (i.e. 13 kB, + 4.1 MB, 102 Bytes, etc). Per default decimal prefixes are used (Mega, + Giga, etc.), if the second parameter is set to `True` the binary + prefixes are used (Mebi, Gibi). + """ + bytes = float(value) + base = binary and 1024 or 1000 + prefixes = [ + (binary and "KiB" or "kB"), + (binary and "MiB" or "MB"), + (binary and "GiB" or "GB"), + (binary and "TiB" or "TB"), + (binary and "PiB" or "PB"), + (binary and "EiB" or "EB"), + (binary and "ZiB" or "ZB"), + (binary and "YiB" or "YB"), + ] + if bytes == 1: + return "1 Byte" + elif bytes < base: + return "%d Bytes" % bytes + else: + for i, prefix in enumerate(prefixes): + unit = base ** (i + 2) + if bytes < unit: + return "%.1f %s" % ((base * bytes / unit), prefix) + return "%.1f %s" % ((base * bytes / unit), prefix) + + +def do_pprint(value, verbose=False): + """Pretty print a variable. Useful for debugging. + + With Jinja 1.2 onwards you can pass it a parameter. If this parameter + is truthy the output will be more verbose (this requires `pretty`) + """ + return pformat(value, verbose=verbose) + + +@evalcontextfilter +def do_urlize( + eval_ctx, value, trim_url_limit=None, nofollow=False, target=None, rel=None +): + """Converts URLs in plain text into clickable links. + + If you pass the filter an additional integer it will shorten the urls + to that number. Also a third argument exists that makes the urls + "nofollow": + + .. sourcecode:: jinja + + {{ mytext|urlize(40, true) }} + links are shortened to 40 chars and defined with rel="nofollow" + + If *target* is specified, the ``target`` attribute will be added to the + ``<a>`` tag: + + .. sourcecode:: jinja + + {{ mytext|urlize(40, target='_blank') }} + + .. versionchanged:: 2.8+ + The *target* parameter was added. + """ + policies = eval_ctx.environment.policies + rel = set((rel or "").split() or []) + if nofollow: + rel.add("nofollow") + rel.update((policies["urlize.rel"] or "").split()) + if target is None: + target = policies["urlize.target"] + rel = " ".join(sorted(rel)) or None + rv = urlize(value, trim_url_limit, rel=rel, target=target) + if eval_ctx.autoescape: + rv = Markup(rv) + return rv + + +def do_indent(s, width=4, first=False, blank=False, indentfirst=None): + """Return a copy of the string with each line indented by 4 spaces. The + first line and blank lines are not indented by default. + + :param width: Number of spaces to indent by. + :param first: Don't skip indenting the first line. + :param blank: Don't skip indenting empty lines. + + .. versionchanged:: 2.10 + Blank lines are not indented by default. + + Rename the ``indentfirst`` argument to ``first``. + """ + if indentfirst is not None: + warnings.warn( + "The 'indentfirst' argument is renamed to 'first' and will" + " be removed in version 3.0.", + DeprecationWarning, + stacklevel=2, + ) + first = indentfirst + + indention = u" " * width + newline = u"\n" + + if isinstance(s, Markup): + indention = Markup(indention) + newline = Markup(newline) + + s += newline # this quirk is necessary for splitlines method + + if blank: + rv = (newline + indention).join(s.splitlines()) + else: + lines = s.splitlines() + rv = lines.pop(0) + + if lines: + rv += newline + newline.join( + indention + line if line else line for line in lines + ) + + if first: + rv = indention + rv + + return rv + + +@environmentfilter +def do_truncate(env, s, length=255, killwords=False, end="...", leeway=None): + """Return a truncated copy of the string. The length is specified + with the first parameter which defaults to ``255``. If the second + parameter is ``true`` the filter will cut the text at length. Otherwise + it will discard the last word. If the text was in fact + truncated it will append an ellipsis sign (``"..."``). If you want a + different ellipsis sign than ``"..."`` you can specify it using the + third parameter. Strings that only exceed the length by the tolerance + margin given in the fourth parameter will not be truncated. + + .. sourcecode:: jinja + + {{ "foo bar baz qux"|truncate(9) }} + -> "foo..." + {{ "foo bar baz qux"|truncate(9, True) }} + -> "foo ba..." + {{ "foo bar baz qux"|truncate(11) }} + -> "foo bar baz qux" + {{ "foo bar baz qux"|truncate(11, False, '...', 0) }} + -> "foo bar..." + + The default leeway on newer Jinja versions is 5 and was 0 before but + can be reconfigured globally. + """ + if leeway is None: + leeway = env.policies["truncate.leeway"] + assert length >= len(end), "expected length >= %s, got %s" % (len(end), length) + assert leeway >= 0, "expected leeway >= 0, got %s" % leeway + if len(s) <= length + leeway: + return s + if killwords: + return s[: length - len(end)] + end + result = s[: length - len(end)].rsplit(" ", 1)[0] + return result + end + + +@environmentfilter +def do_wordwrap( + environment, + s, + width=79, + break_long_words=True, + wrapstring=None, + break_on_hyphens=True, +): + """Wrap a string to the given width. Existing newlines are treated + as paragraphs to be wrapped separately. + + :param s: Original text to wrap. + :param width: Maximum length of wrapped lines. + :param break_long_words: If a word is longer than ``width``, break + it across lines. + :param break_on_hyphens: If a word contains hyphens, it may be split + across lines. + :param wrapstring: String to join each wrapped line. Defaults to + :attr:`Environment.newline_sequence`. + + .. versionchanged:: 2.11 + Existing newlines are treated as paragraphs wrapped separately. + + .. versionchanged:: 2.11 + Added the ``break_on_hyphens`` parameter. + + .. versionchanged:: 2.7 + Added the ``wrapstring`` parameter. + """ + + import textwrap + + if not wrapstring: + wrapstring = environment.newline_sequence + + # textwrap.wrap doesn't consider existing newlines when wrapping. + # If the string has a newline before width, wrap will still insert + # a newline at width, resulting in a short line. Instead, split and + # wrap each paragraph individually. + return wrapstring.join( + [ + wrapstring.join( + textwrap.wrap( + line, + width=width, + expand_tabs=False, + replace_whitespace=False, + break_long_words=break_long_words, + break_on_hyphens=break_on_hyphens, + ) + ) + for line in s.splitlines() + ] + ) + + +def do_wordcount(s): + """Count the words in that string.""" + return len(_word_re.findall(soft_unicode(s))) + + +def do_int(value, default=0, base=10): + """Convert the value into an integer. If the + conversion doesn't work it will return ``0``. You can + override this default using the first parameter. You + can also override the default base (10) in the second + parameter, which handles input with prefixes such as + 0b, 0o and 0x for bases 2, 8 and 16 respectively. + The base is ignored for decimal numbers and non-string values. + """ + try: + if isinstance(value, string_types): + return int(value, base) + return int(value) + except (TypeError, ValueError): + # this quirk is necessary so that "42.23"|int gives 42. + try: + return int(float(value)) + except (TypeError, ValueError): + return default + + +def do_float(value, default=0.0): + """Convert the value into a floating point number. If the + conversion doesn't work it will return ``0.0``. You can + override this default using the first parameter. + """ + try: + return float(value) + except (TypeError, ValueError): + return default + + +def do_format(value, *args, **kwargs): + """Apply the given values to a `printf-style`_ format string, like + ``string % values``. + + .. sourcecode:: jinja + + {{ "%s, %s!"|format(greeting, name) }} + Hello, World! + + In most cases it should be more convenient and efficient to use the + ``%`` operator or :meth:`str.format`. + + .. code-block:: text + + {{ "%s, %s!" % (greeting, name) }} + {{ "{}, {}!".format(greeting, name) }} + + .. _printf-style: https://docs.python.org/library/stdtypes.html + #printf-style-string-formatting + """ + if args and kwargs: + raise FilterArgumentError( + "can't handle positional and keyword arguments at the same time" + ) + return soft_unicode(value) % (kwargs or args) + + +def do_trim(value, chars=None): + """Strip leading and trailing characters, by default whitespace.""" + return soft_unicode(value).strip(chars) + + +def do_striptags(value): + """Strip SGML/XML tags and replace adjacent whitespace by one space.""" + if hasattr(value, "__html__"): + value = value.__html__() + return Markup(text_type(value)).striptags() + + +def do_slice(value, slices, fill_with=None): + """Slice an iterator and return a list of lists containing + those items. Useful if you want to create a div containing + three ul tags that represent columns: + + .. sourcecode:: html+jinja + + <div class="columnwrapper"> + {%- for column in items|slice(3) %} + <ul class="column-{{ loop.index }}"> + {%- for item in column %} + <li>{{ item }}</li> + {%- endfor %} + </ul> + {%- endfor %} + </div> + + If you pass it a second argument it's used to fill missing + values on the last iteration. + """ + seq = list(value) + length = len(seq) + items_per_slice = length // slices + slices_with_extra = length % slices + offset = 0 + for slice_number in range(slices): + start = offset + slice_number * items_per_slice + if slice_number < slices_with_extra: + offset += 1 + end = offset + (slice_number + 1) * items_per_slice + tmp = seq[start:end] + if fill_with is not None and slice_number >= slices_with_extra: + tmp.append(fill_with) + yield tmp + + +def do_batch(value, linecount, fill_with=None): + """ + A filter that batches items. It works pretty much like `slice` + just the other way round. It returns a list of lists with the + given number of items. If you provide a second parameter this + is used to fill up missing items. See this example: + + .. sourcecode:: html+jinja + + <table> + {%- for row in items|batch(3, ' ') %} + <tr> + {%- for column in row %} + <td>{{ column }}</td> + {%- endfor %} + </tr> + {%- endfor %} + </table> + """ + tmp = [] + for item in value: + if len(tmp) == linecount: + yield tmp + tmp = [] + tmp.append(item) + if tmp: + if fill_with is not None and len(tmp) < linecount: + tmp += [fill_with] * (linecount - len(tmp)) + yield tmp + + +def do_round(value, precision=0, method="common"): + """Round the number to a given precision. The first + parameter specifies the precision (default is ``0``), the + second the rounding method: + + - ``'common'`` rounds either up or down + - ``'ceil'`` always rounds up + - ``'floor'`` always rounds down + + If you don't specify a method ``'common'`` is used. + + .. sourcecode:: jinja + + {{ 42.55|round }} + -> 43.0 + {{ 42.55|round(1, 'floor') }} + -> 42.5 + + Note that even if rounded to 0 precision, a float is returned. If + you need a real integer, pipe it through `int`: + + .. sourcecode:: jinja + + {{ 42.55|round|int }} + -> 43 + """ + if method not in {"common", "ceil", "floor"}: + raise FilterArgumentError("method must be common, ceil or floor") + if method == "common": + return round(value, precision) + func = getattr(math, method) + return func(value * (10 ** precision)) / (10 ** precision) + + +# Use a regular tuple repr here. This is what we did in the past and we +# really want to hide this custom type as much as possible. In particular +# we do not want to accidentally expose an auto generated repr in case +# people start to print this out in comments or something similar for +# debugging. +_GroupTuple = namedtuple("_GroupTuple", ["grouper", "list"]) +_GroupTuple.__repr__ = tuple.__repr__ +_GroupTuple.__str__ = tuple.__str__ + + +@environmentfilter +def do_groupby(environment, value, attribute): + """Group a sequence of objects by an attribute using Python's + :func:`itertools.groupby`. The attribute can use dot notation for + nested access, like ``"address.city"``. Unlike Python's ``groupby``, + the values are sorted first so only one group is returned for each + unique value. + + For example, a list of ``User`` objects with a ``city`` attribute + can be rendered in groups. In this example, ``grouper`` refers to + the ``city`` value of the group. + + .. sourcecode:: html+jinja + + <ul>{% for city, items in users|groupby("city") %} + <li>{{ city }} + <ul>{% for user in items %} + <li>{{ user.name }} + {% endfor %}</ul> + </li> + {% endfor %}</ul> + + ``groupby`` yields namedtuples of ``(grouper, list)``, which + can be used instead of the tuple unpacking above. ``grouper`` is the + value of the attribute, and ``list`` is the items with that value. + + .. sourcecode:: html+jinja + + <ul>{% for group in users|groupby("city") %} + <li>{{ group.grouper }}: {{ group.list|join(", ") }} + {% endfor %}</ul> + + .. versionchanged:: 2.6 + The attribute supports dot notation for nested access. + """ + expr = make_attrgetter(environment, attribute) + return [ + _GroupTuple(key, list(values)) + for key, values in groupby(sorted(value, key=expr), expr) + ] + + +@environmentfilter +def do_sum(environment, iterable, attribute=None, start=0): + """Returns the sum of a sequence of numbers plus the value of parameter + 'start' (which defaults to 0). When the sequence is empty it returns + start. + + It is also possible to sum up only certain attributes: + + .. sourcecode:: jinja + + Total: {{ items|sum(attribute='price') }} + + .. versionchanged:: 2.6 + The `attribute` parameter was added to allow suming up over + attributes. Also the `start` parameter was moved on to the right. + """ + if attribute is not None: + iterable = imap(make_attrgetter(environment, attribute), iterable) + return sum(iterable, start) + + +def do_list(value): + """Convert the value into a list. If it was a string the returned list + will be a list of characters. + """ + return list(value) + + +def do_mark_safe(value): + """Mark the value as safe which means that in an environment with automatic + escaping enabled this variable will not be escaped. + """ + return Markup(value) + + +def do_mark_unsafe(value): + """Mark a value as unsafe. This is the reverse operation for :func:`safe`.""" + return text_type(value) + + +def do_reverse(value): + """Reverse the object or return an iterator that iterates over it the other + way round. + """ + if isinstance(value, string_types): + return value[::-1] + try: + return reversed(value) + except TypeError: + try: + rv = list(value) + rv.reverse() + return rv + except TypeError: + raise FilterArgumentError("argument must be iterable") + + +@environmentfilter +def do_attr(environment, obj, name): + """Get an attribute of an object. ``foo|attr("bar")`` works like + ``foo.bar`` just that always an attribute is returned and items are not + looked up. + + See :ref:`Notes on subscriptions <notes-on-subscriptions>` for more details. + """ + try: + name = str(name) + except UnicodeError: + pass + else: + try: + value = getattr(obj, name) + except AttributeError: + pass + else: + if environment.sandboxed and not environment.is_safe_attribute( + obj, name, value + ): + return environment.unsafe_undefined(obj, name) + return value + return environment.undefined(obj=obj, name=name) + + +@contextfilter +def do_map(*args, **kwargs): + """Applies a filter on a sequence of objects or looks up an attribute. + This is useful when dealing with lists of objects but you are really + only interested in a certain value of it. + + The basic usage is mapping on an attribute. Imagine you have a list + of users but you are only interested in a list of usernames: + + .. sourcecode:: jinja + + Users on this page: {{ users|map(attribute='username')|join(', ') }} + + You can specify a ``default`` value to use if an object in the list + does not have the given attribute. + + .. sourcecode:: jinja + + {{ users|map(attribute="username", default="Anonymous")|join(", ") }} + + Alternatively you can let it invoke a filter by passing the name of the + filter and the arguments afterwards. A good example would be applying a + text conversion filter on a sequence: + + .. sourcecode:: jinja + + Users on this page: {{ titles|map('lower')|join(', ') }} + + Similar to a generator comprehension such as: + + .. code-block:: python + + (u.username for u in users) + (u.username or "Anonymous" for u in users) + (do_lower(x) for x in titles) + + .. versionchanged:: 2.11.0 + Added the ``default`` parameter. + + .. versionadded:: 2.7 + """ + seq, func = prepare_map(args, kwargs) + if seq: + for item in seq: + yield func(item) + + +@contextfilter +def do_select(*args, **kwargs): + """Filters a sequence of objects by applying a test to each object, + and only selecting the objects with the test succeeding. + + If no test is specified, each object will be evaluated as a boolean. + + Example usage: + + .. sourcecode:: jinja + + {{ numbers|select("odd") }} + {{ numbers|select("odd") }} + {{ numbers|select("divisibleby", 3) }} + {{ numbers|select("lessthan", 42) }} + {{ strings|select("equalto", "mystring") }} + + Similar to a generator comprehension such as: + + .. code-block:: python + + (n for n in numbers if test_odd(n)) + (n for n in numbers if test_divisibleby(n, 3)) + + .. versionadded:: 2.7 + """ + return select_or_reject(args, kwargs, lambda x: x, False) + + +@contextfilter +def do_reject(*args, **kwargs): + """Filters a sequence of objects by applying a test to each object, + and rejecting the objects with the test succeeding. + + If no test is specified, each object will be evaluated as a boolean. + + Example usage: + + .. sourcecode:: jinja + + {{ numbers|reject("odd") }} + + Similar to a generator comprehension such as: + + .. code-block:: python + + (n for n in numbers if not test_odd(n)) + + .. versionadded:: 2.7 + """ + return select_or_reject(args, kwargs, lambda x: not x, False) + + +@contextfilter +def do_selectattr(*args, **kwargs): + """Filters a sequence of objects by applying a test to the specified + attribute of each object, and only selecting the objects with the + test succeeding. + + If no test is specified, the attribute's value will be evaluated as + a boolean. + + Example usage: + + .. sourcecode:: jinja + + {{ users|selectattr("is_active") }} + {{ users|selectattr("email", "none") }} + + Similar to a generator comprehension such as: + + .. code-block:: python + + (u for user in users if user.is_active) + (u for user in users if test_none(user.email)) + + .. versionadded:: 2.7 + """ + return select_or_reject(args, kwargs, lambda x: x, True) + + +@contextfilter +def do_rejectattr(*args, **kwargs): + """Filters a sequence of objects by applying a test to the specified + attribute of each object, and rejecting the objects with the test + succeeding. + + If no test is specified, the attribute's value will be evaluated as + a boolean. + + .. sourcecode:: jinja + + {{ users|rejectattr("is_active") }} + {{ users|rejectattr("email", "none") }} + + Similar to a generator comprehension such as: + + .. code-block:: python + + (u for user in users if not user.is_active) + (u for user in users if not test_none(user.email)) + + .. versionadded:: 2.7 + """ + return select_or_reject(args, kwargs, lambda x: not x, True) + + +@evalcontextfilter +def do_tojson(eval_ctx, value, indent=None): + """Dumps a structure to JSON so that it's safe to use in ``<script>`` + tags. It accepts the same arguments and returns a JSON string. Note that + this is available in templates through the ``|tojson`` filter which will + also mark the result as safe. Due to how this function escapes certain + characters this is safe even if used outside of ``<script>`` tags. + + The following characters are escaped in strings: + + - ``<`` + - ``>`` + - ``&`` + - ``'`` + + This makes it safe to embed such strings in any place in HTML with the + notable exception of double quoted attributes. In that case single + quote your attributes or HTML escape it in addition. + + The indent parameter can be used to enable pretty printing. Set it to + the number of spaces that the structures should be indented with. + + Note that this filter is for use in HTML contexts only. + + .. versionadded:: 2.9 + """ + policies = eval_ctx.environment.policies + dumper = policies["json.dumps_function"] + options = policies["json.dumps_kwargs"] + if indent is not None: + options = dict(options) + options["indent"] = indent + return htmlsafe_json_dumps(value, dumper=dumper, **options) + + +def prepare_map(args, kwargs): + context = args[0] + seq = args[1] + default = None + + if len(args) == 2 and "attribute" in kwargs: + attribute = kwargs.pop("attribute") + default = kwargs.pop("default", None) + if kwargs: + raise FilterArgumentError( + "Unexpected keyword argument %r" % next(iter(kwargs)) + ) + func = make_attrgetter(context.environment, attribute, default=default) + else: + try: + name = args[2] + args = args[3:] + except LookupError: + raise FilterArgumentError("map requires a filter argument") + + def func(item): + return context.environment.call_filter( + name, item, args, kwargs, context=context + ) + + return seq, func + + +def prepare_select_or_reject(args, kwargs, modfunc, lookup_attr): + context = args[0] + seq = args[1] + if lookup_attr: + try: + attr = args[2] + except LookupError: + raise FilterArgumentError("Missing parameter for attribute name") + transfunc = make_attrgetter(context.environment, attr) + off = 1 + else: + off = 0 + + def transfunc(x): + return x + + try: + name = args[2 + off] + args = args[3 + off :] + + def func(item): + return context.environment.call_test(name, item, args, kwargs) + + except LookupError: + func = bool + + return seq, lambda item: modfunc(func(transfunc(item))) + + +def select_or_reject(args, kwargs, modfunc, lookup_attr): + seq, func = prepare_select_or_reject(args, kwargs, modfunc, lookup_attr) + if seq: + for item in seq: + if func(item): + yield item + + +FILTERS = { + "abs": abs, + "attr": do_attr, + "batch": do_batch, + "capitalize": do_capitalize, + "center": do_center, + "count": len, + "d": do_default, + "default": do_default, + "dictsort": do_dictsort, + "e": escape, + "escape": escape, + "filesizeformat": do_filesizeformat, + "first": do_first, + "float": do_float, + "forceescape": do_forceescape, + "format": do_format, + "groupby": do_groupby, + "indent": do_indent, + "int": do_int, + "join": do_join, + "last": do_last, + "length": len, + "list": do_list, + "lower": do_lower, + "map": do_map, + "min": do_min, + "max": do_max, + "pprint": do_pprint, + "random": do_random, + "reject": do_reject, + "rejectattr": do_rejectattr, + "replace": do_replace, + "reverse": do_reverse, + "round": do_round, + "safe": do_mark_safe, + "select": do_select, + "selectattr": do_selectattr, + "slice": do_slice, + "sort": do_sort, + "string": soft_unicode, + "striptags": do_striptags, + "sum": do_sum, + "title": do_title, + "trim": do_trim, + "truncate": do_truncate, + "unique": do_unique, + "upper": do_upper, + "urlencode": do_urlencode, + "urlize": do_urlize, + "wordcount": do_wordcount, + "wordwrap": do_wordwrap, + "xmlattr": do_xmlattr, + "tojson": do_tojson, +} diff --git a/contrib/python/Jinja2/py2/jinja2/idtracking.py b/contrib/python/Jinja2/py2/jinja2/idtracking.py new file mode 100644 index 0000000000..9a0d838017 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/idtracking.py @@ -0,0 +1,290 @@ +from ._compat import iteritems +from .visitor import NodeVisitor + +VAR_LOAD_PARAMETER = "param" +VAR_LOAD_RESOLVE = "resolve" +VAR_LOAD_ALIAS = "alias" +VAR_LOAD_UNDEFINED = "undefined" + + +def find_symbols(nodes, parent_symbols=None): + sym = Symbols(parent=parent_symbols) + visitor = FrameSymbolVisitor(sym) + for node in nodes: + visitor.visit(node) + return sym + + +def symbols_for_node(node, parent_symbols=None): + sym = Symbols(parent=parent_symbols) + sym.analyze_node(node) + return sym + + +class Symbols(object): + def __init__(self, parent=None, level=None): + if level is None: + if parent is None: + level = 0 + else: + level = parent.level + 1 + self.level = level + self.parent = parent + self.refs = {} + self.loads = {} + self.stores = set() + + def analyze_node(self, node, **kwargs): + visitor = RootVisitor(self) + visitor.visit(node, **kwargs) + + def _define_ref(self, name, load=None): + ident = "l_%d_%s" % (self.level, name) + self.refs[name] = ident + if load is not None: + self.loads[ident] = load + return ident + + def find_load(self, target): + if target in self.loads: + return self.loads[target] + if self.parent is not None: + return self.parent.find_load(target) + + def find_ref(self, name): + if name in self.refs: + return self.refs[name] + if self.parent is not None: + return self.parent.find_ref(name) + + def ref(self, name): + rv = self.find_ref(name) + if rv is None: + raise AssertionError( + "Tried to resolve a name to a reference that " + "was unknown to the frame (%r)" % name + ) + return rv + + def copy(self): + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.refs = self.refs.copy() + rv.loads = self.loads.copy() + rv.stores = self.stores.copy() + return rv + + def store(self, name): + self.stores.add(name) + + # If we have not see the name referenced yet, we need to figure + # out what to set it to. + if name not in self.refs: + # If there is a parent scope we check if the name has a + # reference there. If it does it means we might have to alias + # to a variable there. + if self.parent is not None: + outer_ref = self.parent.find_ref(name) + if outer_ref is not None: + self._define_ref(name, load=(VAR_LOAD_ALIAS, outer_ref)) + return + + # Otherwise we can just set it to undefined. + self._define_ref(name, load=(VAR_LOAD_UNDEFINED, None)) + + def declare_parameter(self, name): + self.stores.add(name) + return self._define_ref(name, load=(VAR_LOAD_PARAMETER, None)) + + def load(self, name): + target = self.find_ref(name) + if target is None: + self._define_ref(name, load=(VAR_LOAD_RESOLVE, name)) + + def branch_update(self, branch_symbols): + stores = {} + for branch in branch_symbols: + for target in branch.stores: + if target in self.stores: + continue + stores[target] = stores.get(target, 0) + 1 + + for sym in branch_symbols: + self.refs.update(sym.refs) + self.loads.update(sym.loads) + self.stores.update(sym.stores) + + for name, branch_count in iteritems(stores): + if branch_count == len(branch_symbols): + continue + target = self.find_ref(name) + assert target is not None, "should not happen" + + if self.parent is not None: + outer_target = self.parent.find_ref(name) + if outer_target is not None: + self.loads[target] = (VAR_LOAD_ALIAS, outer_target) + continue + self.loads[target] = (VAR_LOAD_RESOLVE, name) + + def dump_stores(self): + rv = {} + node = self + while node is not None: + for name in node.stores: + if name not in rv: + rv[name] = self.find_ref(name) + node = node.parent + return rv + + def dump_param_targets(self): + rv = set() + node = self + while node is not None: + for target, (instr, _) in iteritems(self.loads): + if instr == VAR_LOAD_PARAMETER: + rv.add(target) + node = node.parent + return rv + + +class RootVisitor(NodeVisitor): + def __init__(self, symbols): + self.sym_visitor = FrameSymbolVisitor(symbols) + + def _simple_visit(self, node, **kwargs): + for child in node.iter_child_nodes(): + self.sym_visitor.visit(child) + + visit_Template = ( + visit_Block + ) = ( + visit_Macro + ) = ( + visit_FilterBlock + ) = visit_Scope = visit_If = visit_ScopedEvalContextModifier = _simple_visit + + def visit_AssignBlock(self, node, **kwargs): + for child in node.body: + self.sym_visitor.visit(child) + + def visit_CallBlock(self, node, **kwargs): + for child in node.iter_child_nodes(exclude=("call",)): + self.sym_visitor.visit(child) + + def visit_OverlayScope(self, node, **kwargs): + for child in node.body: + self.sym_visitor.visit(child) + + def visit_For(self, node, for_branch="body", **kwargs): + if for_branch == "body": + self.sym_visitor.visit(node.target, store_as_param=True) + branch = node.body + elif for_branch == "else": + branch = node.else_ + elif for_branch == "test": + self.sym_visitor.visit(node.target, store_as_param=True) + if node.test is not None: + self.sym_visitor.visit(node.test) + return + else: + raise RuntimeError("Unknown for branch") + for item in branch or (): + self.sym_visitor.visit(item) + + def visit_With(self, node, **kwargs): + for target in node.targets: + self.sym_visitor.visit(target) + for child in node.body: + self.sym_visitor.visit(child) + + def generic_visit(self, node, *args, **kwargs): + raise NotImplementedError( + "Cannot find symbols for %r" % node.__class__.__name__ + ) + + +class FrameSymbolVisitor(NodeVisitor): + """A visitor for `Frame.inspect`.""" + + def __init__(self, symbols): + self.symbols = symbols + + def visit_Name(self, node, store_as_param=False, **kwargs): + """All assignments to names go through this function.""" + if store_as_param or node.ctx == "param": + self.symbols.declare_parameter(node.name) + elif node.ctx == "store": + self.symbols.store(node.name) + elif node.ctx == "load": + self.symbols.load(node.name) + + def visit_NSRef(self, node, **kwargs): + self.symbols.load(node.name) + + def visit_If(self, node, **kwargs): + self.visit(node.test, **kwargs) + + original_symbols = self.symbols + + def inner_visit(nodes): + self.symbols = rv = original_symbols.copy() + for subnode in nodes: + self.visit(subnode, **kwargs) + self.symbols = original_symbols + return rv + + body_symbols = inner_visit(node.body) + elif_symbols = inner_visit(node.elif_) + else_symbols = inner_visit(node.else_ or ()) + + self.symbols.branch_update([body_symbols, elif_symbols, else_symbols]) + + def visit_Macro(self, node, **kwargs): + self.symbols.store(node.name) + + def visit_Import(self, node, **kwargs): + self.generic_visit(node, **kwargs) + self.symbols.store(node.target) + + def visit_FromImport(self, node, **kwargs): + self.generic_visit(node, **kwargs) + for name in node.names: + if isinstance(name, tuple): + self.symbols.store(name[1]) + else: + self.symbols.store(name) + + def visit_Assign(self, node, **kwargs): + """Visit assignments in the correct order.""" + self.visit(node.node, **kwargs) + self.visit(node.target, **kwargs) + + def visit_For(self, node, **kwargs): + """Visiting stops at for blocks. However the block sequence + is visited as part of the outer scope. + """ + self.visit(node.iter, **kwargs) + + def visit_CallBlock(self, node, **kwargs): + self.visit(node.call, **kwargs) + + def visit_FilterBlock(self, node, **kwargs): + self.visit(node.filter, **kwargs) + + def visit_With(self, node, **kwargs): + for target in node.values: + self.visit(target) + + def visit_AssignBlock(self, node, **kwargs): + """Stop visiting at block assigns.""" + self.visit(node.target, **kwargs) + + def visit_Scope(self, node, **kwargs): + """Stop visiting at scopes.""" + + def visit_Block(self, node, **kwargs): + """Stop visiting at blocks.""" + + def visit_OverlayScope(self, node, **kwargs): + """Do not visit into overlay scopes.""" diff --git a/contrib/python/Jinja2/py2/jinja2/lexer.py b/contrib/python/Jinja2/py2/jinja2/lexer.py new file mode 100644 index 0000000000..552356a12d --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/lexer.py @@ -0,0 +1,848 @@ +# -*- coding: utf-8 -*- +"""Implements a Jinja / Python combination lexer. The ``Lexer`` class +is used to do some preprocessing. It filters out invalid operators like +the bitshift operators we don't allow in templates. It separates +template code and python code in expressions. +""" +import re +from ast import literal_eval +from collections import deque +from operator import itemgetter + +from ._compat import implements_iterator +from ._compat import intern +from ._compat import iteritems +from ._compat import text_type +from .exceptions import TemplateSyntaxError +from .utils import LRUCache + +# cache for the lexers. Exists in order to be able to have multiple +# environments with the same lexer +_lexer_cache = LRUCache(50) + +# static regular expressions +whitespace_re = re.compile(r"\s+", re.U) +newline_re = re.compile(r"(\r\n|\r|\n)") +string_re = re.compile( + r"('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S +) +integer_re = re.compile(r"(\d+_)*\d+") +float_re = re.compile( + r""" + (?<!\.) # doesn't start with a . + (\d+_)*\d+ # digits, possibly _ separated + ( + (\.(\d+_)*\d+)? # optional fractional part + e[+\-]?(\d+_)*\d+ # exponent part + | + \.(\d+_)*\d+ # required fractional part + ) + """, + re.IGNORECASE | re.VERBOSE, +) + +try: + # check if this Python supports Unicode identifiers + compile("föö", "<unknown>", "eval") +except SyntaxError: + # Python 2, no Unicode support, use ASCII identifiers + name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*") + check_ident = False +else: + # Unicode support, import generated re pattern and set flag to use + # str.isidentifier to validate during lexing. + from ._identifier import pattern as name_re + + check_ident = True + +# internal the tokens and keep references to them +TOKEN_ADD = intern("add") +TOKEN_ASSIGN = intern("assign") +TOKEN_COLON = intern("colon") +TOKEN_COMMA = intern("comma") +TOKEN_DIV = intern("div") +TOKEN_DOT = intern("dot") +TOKEN_EQ = intern("eq") +TOKEN_FLOORDIV = intern("floordiv") +TOKEN_GT = intern("gt") +TOKEN_GTEQ = intern("gteq") +TOKEN_LBRACE = intern("lbrace") +TOKEN_LBRACKET = intern("lbracket") +TOKEN_LPAREN = intern("lparen") +TOKEN_LT = intern("lt") +TOKEN_LTEQ = intern("lteq") +TOKEN_MOD = intern("mod") +TOKEN_MUL = intern("mul") +TOKEN_NE = intern("ne") +TOKEN_PIPE = intern("pipe") +TOKEN_POW = intern("pow") +TOKEN_RBRACE = intern("rbrace") +TOKEN_RBRACKET = intern("rbracket") +TOKEN_RPAREN = intern("rparen") +TOKEN_SEMICOLON = intern("semicolon") +TOKEN_SUB = intern("sub") +TOKEN_TILDE = intern("tilde") +TOKEN_WHITESPACE = intern("whitespace") +TOKEN_FLOAT = intern("float") +TOKEN_INTEGER = intern("integer") +TOKEN_NAME = intern("name") +TOKEN_STRING = intern("string") +TOKEN_OPERATOR = intern("operator") +TOKEN_BLOCK_BEGIN = intern("block_begin") +TOKEN_BLOCK_END = intern("block_end") +TOKEN_VARIABLE_BEGIN = intern("variable_begin") +TOKEN_VARIABLE_END = intern("variable_end") +TOKEN_RAW_BEGIN = intern("raw_begin") +TOKEN_RAW_END = intern("raw_end") +TOKEN_COMMENT_BEGIN = intern("comment_begin") +TOKEN_COMMENT_END = intern("comment_end") +TOKEN_COMMENT = intern("comment") +TOKEN_LINESTATEMENT_BEGIN = intern("linestatement_begin") +TOKEN_LINESTATEMENT_END = intern("linestatement_end") +TOKEN_LINECOMMENT_BEGIN = intern("linecomment_begin") +TOKEN_LINECOMMENT_END = intern("linecomment_end") +TOKEN_LINECOMMENT = intern("linecomment") +TOKEN_DATA = intern("data") +TOKEN_INITIAL = intern("initial") +TOKEN_EOF = intern("eof") + +# bind operators to token types +operators = { + "+": TOKEN_ADD, + "-": TOKEN_SUB, + "/": TOKEN_DIV, + "//": TOKEN_FLOORDIV, + "*": TOKEN_MUL, + "%": TOKEN_MOD, + "**": TOKEN_POW, + "~": TOKEN_TILDE, + "[": TOKEN_LBRACKET, + "]": TOKEN_RBRACKET, + "(": TOKEN_LPAREN, + ")": TOKEN_RPAREN, + "{": TOKEN_LBRACE, + "}": TOKEN_RBRACE, + "==": TOKEN_EQ, + "!=": TOKEN_NE, + ">": TOKEN_GT, + ">=": TOKEN_GTEQ, + "<": TOKEN_LT, + "<=": TOKEN_LTEQ, + "=": TOKEN_ASSIGN, + ".": TOKEN_DOT, + ":": TOKEN_COLON, + "|": TOKEN_PIPE, + ",": TOKEN_COMMA, + ";": TOKEN_SEMICOLON, +} + +reverse_operators = dict([(v, k) for k, v in iteritems(operators)]) +assert len(operators) == len(reverse_operators), "operators dropped" +operator_re = re.compile( + "(%s)" % "|".join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x))) +) + +ignored_tokens = frozenset( + [ + TOKEN_COMMENT_BEGIN, + TOKEN_COMMENT, + TOKEN_COMMENT_END, + TOKEN_WHITESPACE, + TOKEN_LINECOMMENT_BEGIN, + TOKEN_LINECOMMENT_END, + TOKEN_LINECOMMENT, + ] +) +ignore_if_empty = frozenset( + [TOKEN_WHITESPACE, TOKEN_DATA, TOKEN_COMMENT, TOKEN_LINECOMMENT] +) + + +def _describe_token_type(token_type): + if token_type in reverse_operators: + return reverse_operators[token_type] + return { + TOKEN_COMMENT_BEGIN: "begin of comment", + TOKEN_COMMENT_END: "end of comment", + TOKEN_COMMENT: "comment", + TOKEN_LINECOMMENT: "comment", + TOKEN_BLOCK_BEGIN: "begin of statement block", + TOKEN_BLOCK_END: "end of statement block", + TOKEN_VARIABLE_BEGIN: "begin of print statement", + TOKEN_VARIABLE_END: "end of print statement", + TOKEN_LINESTATEMENT_BEGIN: "begin of line statement", + TOKEN_LINESTATEMENT_END: "end of line statement", + TOKEN_DATA: "template data / text", + TOKEN_EOF: "end of template", + }.get(token_type, token_type) + + +def describe_token(token): + """Returns a description of the token.""" + if token.type == TOKEN_NAME: + return token.value + return _describe_token_type(token.type) + + +def describe_token_expr(expr): + """Like `describe_token` but for token expressions.""" + if ":" in expr: + type, value = expr.split(":", 1) + if type == TOKEN_NAME: + return value + else: + type = expr + return _describe_token_type(type) + + +def count_newlines(value): + """Count the number of newline characters in the string. This is + useful for extensions that filter a stream. + """ + return len(newline_re.findall(value)) + + +def compile_rules(environment): + """Compiles all the rules from the environment into a list of rules.""" + e = re.escape + rules = [ + ( + len(environment.comment_start_string), + TOKEN_COMMENT_BEGIN, + e(environment.comment_start_string), + ), + ( + len(environment.block_start_string), + TOKEN_BLOCK_BEGIN, + e(environment.block_start_string), + ), + ( + len(environment.variable_start_string), + TOKEN_VARIABLE_BEGIN, + e(environment.variable_start_string), + ), + ] + + if environment.line_statement_prefix is not None: + rules.append( + ( + len(environment.line_statement_prefix), + TOKEN_LINESTATEMENT_BEGIN, + r"^[ \t\v]*" + e(environment.line_statement_prefix), + ) + ) + if environment.line_comment_prefix is not None: + rules.append( + ( + len(environment.line_comment_prefix), + TOKEN_LINECOMMENT_BEGIN, + r"(?:^|(?<=\S))[^\S\r\n]*" + e(environment.line_comment_prefix), + ) + ) + + return [x[1:] for x in sorted(rules, reverse=True)] + + +class Failure(object): + """Class that raises a `TemplateSyntaxError` if called. + Used by the `Lexer` to specify known errors. + """ + + def __init__(self, message, cls=TemplateSyntaxError): + self.message = message + self.error_class = cls + + def __call__(self, lineno, filename): + raise self.error_class(self.message, lineno, filename) + + +class Token(tuple): + """Token class.""" + + __slots__ = () + lineno, type, value = (property(itemgetter(x)) for x in range(3)) + + def __new__(cls, lineno, type, value): + return tuple.__new__(cls, (lineno, intern(str(type)), value)) + + def __str__(self): + if self.type in reverse_operators: + return reverse_operators[self.type] + elif self.type == "name": + return self.value + return self.type + + def test(self, expr): + """Test a token against a token expression. This can either be a + token type or ``'token_type:token_value'``. This can only test + against string values and types. + """ + # here we do a regular string equality check as test_any is usually + # passed an iterable of not interned strings. + if self.type == expr: + return True + elif ":" in expr: + return expr.split(":", 1) == [self.type, self.value] + return False + + def test_any(self, *iterable): + """Test against multiple token expressions.""" + for expr in iterable: + if self.test(expr): + return True + return False + + def __repr__(self): + return "Token(%r, %r, %r)" % (self.lineno, self.type, self.value) + + +@implements_iterator +class TokenStreamIterator(object): + """The iterator for tokenstreams. Iterate over the stream + until the eof token is reached. + """ + + def __init__(self, stream): + self.stream = stream + + def __iter__(self): + return self + + def __next__(self): + token = self.stream.current + if token.type is TOKEN_EOF: + self.stream.close() + raise StopIteration() + next(self.stream) + return token + + +@implements_iterator +class TokenStream(object): + """A token stream is an iterable that yields :class:`Token`\\s. The + parser however does not iterate over it but calls :meth:`next` to go + one token ahead. The current active token is stored as :attr:`current`. + """ + + def __init__(self, generator, name, filename): + self._iter = iter(generator) + self._pushed = deque() + self.name = name + self.filename = filename + self.closed = False + self.current = Token(1, TOKEN_INITIAL, "") + next(self) + + def __iter__(self): + return TokenStreamIterator(self) + + def __bool__(self): + return bool(self._pushed) or self.current.type is not TOKEN_EOF + + __nonzero__ = __bool__ # py2 + + @property + def eos(self): + """Are we at the end of the stream?""" + return not self + + def push(self, token): + """Push a token back to the stream.""" + self._pushed.append(token) + + def look(self): + """Look at the next token.""" + old_token = next(self) + result = self.current + self.push(result) + self.current = old_token + return result + + def skip(self, n=1): + """Got n tokens ahead.""" + for _ in range(n): + next(self) + + def next_if(self, expr): + """Perform the token test and return the token if it matched. + Otherwise the return value is `None`. + """ + if self.current.test(expr): + return next(self) + + def skip_if(self, expr): + """Like :meth:`next_if` but only returns `True` or `False`.""" + return self.next_if(expr) is not None + + def __next__(self): + """Go one token ahead and return the old one. + + Use the built-in :func:`next` instead of calling this directly. + """ + rv = self.current + if self._pushed: + self.current = self._pushed.popleft() + elif self.current.type is not TOKEN_EOF: + try: + self.current = next(self._iter) + except StopIteration: + self.close() + return rv + + def close(self): + """Close the stream.""" + self.current = Token(self.current.lineno, TOKEN_EOF, "") + self._iter = None + self.closed = True + + def expect(self, expr): + """Expect a given token type and return it. This accepts the same + argument as :meth:`jinja2.lexer.Token.test`. + """ + if not self.current.test(expr): + expr = describe_token_expr(expr) + if self.current.type is TOKEN_EOF: + raise TemplateSyntaxError( + "unexpected end of template, expected %r." % expr, + self.current.lineno, + self.name, + self.filename, + ) + raise TemplateSyntaxError( + "expected token %r, got %r" % (expr, describe_token(self.current)), + self.current.lineno, + self.name, + self.filename, + ) + try: + return self.current + finally: + next(self) + + +def get_lexer(environment): + """Return a lexer which is probably cached.""" + key = ( + environment.block_start_string, + environment.block_end_string, + environment.variable_start_string, + environment.variable_end_string, + environment.comment_start_string, + environment.comment_end_string, + environment.line_statement_prefix, + environment.line_comment_prefix, + environment.trim_blocks, + environment.lstrip_blocks, + environment.newline_sequence, + environment.keep_trailing_newline, + ) + lexer = _lexer_cache.get(key) + if lexer is None: + lexer = Lexer(environment) + _lexer_cache[key] = lexer + return lexer + + +class OptionalLStrip(tuple): + """A special tuple for marking a point in the state that can have + lstrip applied. + """ + + __slots__ = () + + # Even though it looks like a no-op, creating instances fails + # without this. + def __new__(cls, *members, **kwargs): + return super(OptionalLStrip, cls).__new__(cls, members) + + +class Lexer(object): + """Class that implements a lexer for a given environment. Automatically + created by the environment class, usually you don't have to do that. + + Note that the lexer is not automatically bound to an environment. + Multiple environments can share the same lexer. + """ + + def __init__(self, environment): + # shortcuts + e = re.escape + + def c(x): + return re.compile(x, re.M | re.S) + + # lexing rules for tags + tag_rules = [ + (whitespace_re, TOKEN_WHITESPACE, None), + (float_re, TOKEN_FLOAT, None), + (integer_re, TOKEN_INTEGER, None), + (name_re, TOKEN_NAME, None), + (string_re, TOKEN_STRING, None), + (operator_re, TOKEN_OPERATOR, None), + ] + + # assemble the root lexing rule. because "|" is ungreedy + # we have to sort by length so that the lexer continues working + # as expected when we have parsing rules like <% for block and + # <%= for variables. (if someone wants asp like syntax) + # variables are just part of the rules if variable processing + # is required. + root_tag_rules = compile_rules(environment) + + # block suffix if trimming is enabled + block_suffix_re = environment.trim_blocks and "\\n?" or "" + + # If lstrip is enabled, it should not be applied if there is any + # non-whitespace between the newline and block. + self.lstrip_unless_re = c(r"[^ \t]") if environment.lstrip_blocks else None + + self.newline_sequence = environment.newline_sequence + self.keep_trailing_newline = environment.keep_trailing_newline + + # global lexing rules + self.rules = { + "root": [ + # directives + ( + c( + "(.*?)(?:%s)" + % "|".join( + [ + r"(?P<raw_begin>%s(\-|\+|)\s*raw\s*(?:\-%s\s*|%s))" + % ( + e(environment.block_start_string), + e(environment.block_end_string), + e(environment.block_end_string), + ) + ] + + [ + r"(?P<%s>%s(\-|\+|))" % (n, r) + for n, r in root_tag_rules + ] + ) + ), + OptionalLStrip(TOKEN_DATA, "#bygroup"), + "#bygroup", + ), + # data + (c(".+"), TOKEN_DATA, None), + ], + # comments + TOKEN_COMMENT_BEGIN: [ + ( + c( + r"(.*?)((?:\-%s\s*|%s)%s)" + % ( + e(environment.comment_end_string), + e(environment.comment_end_string), + block_suffix_re, + ) + ), + (TOKEN_COMMENT, TOKEN_COMMENT_END), + "#pop", + ), + (c("(.)"), (Failure("Missing end of comment tag"),), None), + ], + # blocks + TOKEN_BLOCK_BEGIN: [ + ( + c( + r"(?:\-%s\s*|%s)%s" + % ( + e(environment.block_end_string), + e(environment.block_end_string), + block_suffix_re, + ) + ), + TOKEN_BLOCK_END, + "#pop", + ), + ] + + tag_rules, + # variables + TOKEN_VARIABLE_BEGIN: [ + ( + c( + r"\-%s\s*|%s" + % ( + e(environment.variable_end_string), + e(environment.variable_end_string), + ) + ), + TOKEN_VARIABLE_END, + "#pop", + ) + ] + + tag_rules, + # raw block + TOKEN_RAW_BEGIN: [ + ( + c( + r"(.*?)((?:%s(\-|\+|))\s*endraw\s*(?:\-%s\s*|%s%s))" + % ( + e(environment.block_start_string), + e(environment.block_end_string), + e(environment.block_end_string), + block_suffix_re, + ) + ), + OptionalLStrip(TOKEN_DATA, TOKEN_RAW_END), + "#pop", + ), + (c("(.)"), (Failure("Missing end of raw directive"),), None), + ], + # line statements + TOKEN_LINESTATEMENT_BEGIN: [ + (c(r"\s*(\n|$)"), TOKEN_LINESTATEMENT_END, "#pop") + ] + + tag_rules, + # line comments + TOKEN_LINECOMMENT_BEGIN: [ + ( + c(r"(.*?)()(?=\n|$)"), + (TOKEN_LINECOMMENT, TOKEN_LINECOMMENT_END), + "#pop", + ) + ], + } + + def _normalize_newlines(self, value): + """Called for strings and template data to normalize it to unicode.""" + return newline_re.sub(self.newline_sequence, value) + + def tokenize(self, source, name=None, filename=None, state=None): + """Calls tokeniter + tokenize and wraps it in a token stream.""" + stream = self.tokeniter(source, name, filename, state) + return TokenStream(self.wrap(stream, name, filename), name, filename) + + def wrap(self, stream, name=None, filename=None): + """This is called with the stream as returned by `tokenize` and wraps + every token in a :class:`Token` and converts the value. + """ + for lineno, token, value in stream: + if token in ignored_tokens: + continue + elif token == TOKEN_LINESTATEMENT_BEGIN: + token = TOKEN_BLOCK_BEGIN + elif token == TOKEN_LINESTATEMENT_END: + token = TOKEN_BLOCK_END + # we are not interested in those tokens in the parser + elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END): + continue + elif token == TOKEN_DATA: + value = self._normalize_newlines(value) + elif token == "keyword": + token = value + elif token == TOKEN_NAME: + value = str(value) + if check_ident and not value.isidentifier(): + raise TemplateSyntaxError( + "Invalid character in identifier", lineno, name, filename + ) + elif token == TOKEN_STRING: + # try to unescape string + try: + value = ( + self._normalize_newlines(value[1:-1]) + .encode("ascii", "backslashreplace") + .decode("unicode-escape") + ) + except Exception as e: + msg = str(e).split(":")[-1].strip() + raise TemplateSyntaxError(msg, lineno, name, filename) + elif token == TOKEN_INTEGER: + value = int(value.replace("_", "")) + elif token == TOKEN_FLOAT: + # remove all "_" first to support more Python versions + value = literal_eval(value.replace("_", "")) + elif token == TOKEN_OPERATOR: + token = operators[value] + yield Token(lineno, token, value) + + def tokeniter(self, source, name, filename=None, state=None): + """This method tokenizes the text and returns the tokens in a + generator. Use this method if you just want to tokenize a template. + """ + source = text_type(source) + lines = source.splitlines() + if self.keep_trailing_newline and source: + for newline in ("\r\n", "\r", "\n"): + if source.endswith(newline): + lines.append("") + break + source = "\n".join(lines) + pos = 0 + lineno = 1 + stack = ["root"] + if state is not None and state != "root": + assert state in ("variable", "block"), "invalid state" + stack.append(state + "_begin") + statetokens = self.rules[stack[-1]] + source_length = len(source) + balancing_stack = [] + lstrip_unless_re = self.lstrip_unless_re + newlines_stripped = 0 + line_starting = True + + while 1: + # tokenizer loop + for regex, tokens, new_state in statetokens: + m = regex.match(source, pos) + # if no match we try again with the next rule + if m is None: + continue + + # we only match blocks and variables if braces / parentheses + # are balanced. continue parsing with the lower rule which + # is the operator rule. do this only if the end tags look + # like operators + if balancing_stack and tokens in ( + TOKEN_VARIABLE_END, + TOKEN_BLOCK_END, + TOKEN_LINESTATEMENT_END, + ): + continue + + # tuples support more options + if isinstance(tokens, tuple): + groups = m.groups() + + if isinstance(tokens, OptionalLStrip): + # Rule supports lstrip. Match will look like + # text, block type, whitespace control, type, control, ... + text = groups[0] + + # Skipping the text and first type, every other group is the + # whitespace control for each type. One of the groups will be + # -, +, or empty string instead of None. + strip_sign = next(g for g in groups[2::2] if g is not None) + + if strip_sign == "-": + # Strip all whitespace between the text and the tag. + stripped = text.rstrip() + newlines_stripped = text[len(stripped) :].count("\n") + groups = (stripped,) + groups[1:] + elif ( + # Not marked for preserving whitespace. + strip_sign != "+" + # lstrip is enabled. + and lstrip_unless_re is not None + # Not a variable expression. + and not m.groupdict().get(TOKEN_VARIABLE_BEGIN) + ): + # The start of text between the last newline and the tag. + l_pos = text.rfind("\n") + 1 + if l_pos > 0 or line_starting: + # If there's only whitespace between the newline and the + # tag, strip it. + if not lstrip_unless_re.search(text, l_pos): + groups = (text[:l_pos],) + groups[1:] + + for idx, token in enumerate(tokens): + # failure group + if token.__class__ is Failure: + raise token(lineno, filename) + # bygroup is a bit more complex, in that case we + # yield for the current token the first named + # group that matched + elif token == "#bygroup": + for key, value in iteritems(m.groupdict()): + if value is not None: + yield lineno, key, value + lineno += value.count("\n") + break + else: + raise RuntimeError( + "%r wanted to resolve " + "the token dynamically" + " but no group matched" % regex + ) + # normal group + else: + data = groups[idx] + if data or token not in ignore_if_empty: + yield lineno, token, data + lineno += data.count("\n") + newlines_stripped + newlines_stripped = 0 + + # strings as token just are yielded as it. + else: + data = m.group() + # update brace/parentheses balance + if tokens == TOKEN_OPERATOR: + if data == "{": + balancing_stack.append("}") + elif data == "(": + balancing_stack.append(")") + elif data == "[": + balancing_stack.append("]") + elif data in ("}", ")", "]"): + if not balancing_stack: + raise TemplateSyntaxError( + "unexpected '%s'" % data, lineno, name, filename + ) + expected_op = balancing_stack.pop() + if expected_op != data: + raise TemplateSyntaxError( + "unexpected '%s', " + "expected '%s'" % (data, expected_op), + lineno, + name, + filename, + ) + # yield items + if data or tokens not in ignore_if_empty: + yield lineno, tokens, data + lineno += data.count("\n") + + line_starting = m.group()[-1:] == "\n" + + # fetch new position into new variable so that we can check + # if there is a internal parsing error which would result + # in an infinite loop + pos2 = m.end() + + # handle state changes + if new_state is not None: + # remove the uppermost state + if new_state == "#pop": + stack.pop() + # resolve the new state by group checking + elif new_state == "#bygroup": + for key, value in iteritems(m.groupdict()): + if value is not None: + stack.append(key) + break + else: + raise RuntimeError( + "%r wanted to resolve the " + "new state dynamically but" + " no group matched" % regex + ) + # direct state name given + else: + stack.append(new_state) + statetokens = self.rules[stack[-1]] + # we are still at the same position and no stack change. + # this means a loop without break condition, avoid that and + # raise error + elif pos2 == pos: + raise RuntimeError( + "%r yielded empty string without stack change" % regex + ) + # publish new function and start again + pos = pos2 + break + # if loop terminated without break we haven't found a single match + # either we are at the end of the file or we have a problem + else: + # end of text + if pos >= source_length: + return + # something went wrong + raise TemplateSyntaxError( + "unexpected char %r at %d" % (source[pos], pos), + lineno, + name, + filename, + ) diff --git a/contrib/python/Jinja2/py2/jinja2/loaders.py b/contrib/python/Jinja2/py2/jinja2/loaders.py new file mode 100644 index 0000000000..bad1774ab0 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/loaders.py @@ -0,0 +1,518 @@ +# -*- coding: utf-8 -*- +"""API and implementations for loading templates from different data +sources. +""" +import os +import sys +import weakref +from hashlib import sha1 +from os import path +from types import ModuleType + +from ._compat import abc +from ._compat import fspath +from ._compat import iteritems +from ._compat import string_types +from .exceptions import TemplateNotFound +from .utils import internalcode +from .utils import open_if_exists + + +def split_template_path(template): + """Split a path into segments and perform a sanity check. If it detects + '..' in the path it will raise a `TemplateNotFound` error. + """ + pieces = [] + for piece in template.split("/"): + if ( + path.sep in piece + or (path.altsep and path.altsep in piece) + or piece == path.pardir + ): + raise TemplateNotFound(template) + elif piece and piece != ".": + pieces.append(piece) + return pieces + + +class BaseLoader(object): + """Baseclass for all loaders. Subclass this and override `get_source` to + implement a custom loading mechanism. The environment provides a + `get_template` method that calls the loader's `load` method to get the + :class:`Template` object. + + A very basic example for a loader that looks up templates on the file + system could look like this:: + + from jinja2 import BaseLoader, TemplateNotFound + from os.path import join, exists, getmtime + + class MyLoader(BaseLoader): + + def __init__(self, path): + self.path = path + + def get_source(self, environment, template): + path = join(self.path, template) + if not exists(path): + raise TemplateNotFound(template) + mtime = getmtime(path) + with file(path) as f: + source = f.read().decode('utf-8') + return source, path, lambda: mtime == getmtime(path) + """ + + #: if set to `False` it indicates that the loader cannot provide access + #: to the source of templates. + #: + #: .. versionadded:: 2.4 + has_source_access = True + + def get_source(self, environment, template): + """Get the template source, filename and reload helper for a template. + It's passed the environment and template name and has to return a + tuple in the form ``(source, filename, uptodate)`` or raise a + `TemplateNotFound` error if it can't locate the template. + + The source part of the returned tuple must be the source of the + template as unicode string or a ASCII bytestring. The filename should + be the name of the file on the filesystem if it was loaded from there, + otherwise `None`. The filename is used by python for the tracebacks + if no loader extension is used. + + The last item in the tuple is the `uptodate` function. If auto + reloading is enabled it's always called to check if the template + changed. No arguments are passed so the function must store the + old state somewhere (for example in a closure). If it returns `False` + the template will be reloaded. + """ + if not self.has_source_access: + raise RuntimeError( + "%s cannot provide access to the source" % self.__class__.__name__ + ) + raise TemplateNotFound(template) + + def list_templates(self): + """Iterates over all templates. If the loader does not support that + it should raise a :exc:`TypeError` which is the default behavior. + """ + raise TypeError("this loader cannot iterate over all templates") + + @internalcode + def load(self, environment, name, globals=None): + """Loads a template. This method looks up the template in the cache + or loads one by calling :meth:`get_source`. Subclasses should not + override this method as loaders working on collections of other + loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`) + will not call this method but `get_source` directly. + """ + code = None + if globals is None: + globals = {} + + # first we try to get the source for this template together + # with the filename and the uptodate function. + source, filename, uptodate = self.get_source(environment, name) + + # try to load the code from the bytecode cache if there is a + # bytecode cache configured. + bcc = environment.bytecode_cache + if bcc is not None: + bucket = bcc.get_bucket(environment, name, filename, source) + code = bucket.code + + # if we don't have code so far (not cached, no longer up to + # date) etc. we compile the template + if code is None: + code = environment.compile(source, name, filename) + + # if the bytecode cache is available and the bucket doesn't + # have a code so far, we give the bucket the new code and put + # it back to the bytecode cache. + if bcc is not None and bucket.code is None: + bucket.code = code + bcc.set_bucket(bucket) + + return environment.template_class.from_code( + environment, code, globals, uptodate + ) + + +class FileSystemLoader(BaseLoader): + """Loads templates from the file system. This loader can find templates + in folders on the file system and is the preferred way to load them. + + The loader takes the path to the templates as string, or if multiple + locations are wanted a list of them which is then looked up in the + given order:: + + >>> loader = FileSystemLoader('/path/to/templates') + >>> loader = FileSystemLoader(['/path/to/templates', '/other/path']) + + Per default the template encoding is ``'utf-8'`` which can be changed + by setting the `encoding` parameter to something else. + + To follow symbolic links, set the *followlinks* parameter to ``True``:: + + >>> loader = FileSystemLoader('/path/to/templates', followlinks=True) + + .. versionchanged:: 2.8 + The ``followlinks`` parameter was added. + """ + + def __init__(self, searchpath, encoding="utf-8", followlinks=False): + if not isinstance(searchpath, abc.Iterable) or isinstance( + searchpath, string_types + ): + searchpath = [searchpath] + + # In Python 3.5, os.path.join doesn't support Path. This can be + # simplified to list(searchpath) when Python 3.5 is dropped. + self.searchpath = [fspath(p) for p in searchpath] + + self.encoding = encoding + self.followlinks = followlinks + + def get_source(self, environment, template): + pieces = split_template_path(template) + for searchpath in self.searchpath: + filename = path.join(searchpath, *pieces) + f = open_if_exists(filename) + if f is None: + continue + try: + contents = f.read().decode(self.encoding) + finally: + f.close() + + mtime = path.getmtime(filename) + + def uptodate(): + try: + return path.getmtime(filename) == mtime + except OSError: + return False + + return contents, filename, uptodate + raise TemplateNotFound(template) + + def list_templates(self): + found = set() + for searchpath in self.searchpath: + walk_dir = os.walk(searchpath, followlinks=self.followlinks) + for dirpath, _, filenames in walk_dir: + for filename in filenames: + template = ( + os.path.join(dirpath, filename)[len(searchpath) :] + .strip(os.path.sep) + .replace(os.path.sep, "/") + ) + if template[:2] == "./": + template = template[2:] + if template not in found: + found.add(template) + return sorted(found) + + +class PackageLoader(BaseLoader): + """Load templates from python eggs or packages. It is constructed with + the name of the python package and the path to the templates in that + package:: + + loader = PackageLoader('mypackage', 'views') + + If the package path is not given, ``'templates'`` is assumed. + + Per default the template encoding is ``'utf-8'`` which can be changed + by setting the `encoding` parameter to something else. Due to the nature + of eggs it's only possible to reload templates if the package was loaded + from the file system and not a zip file. + """ + + def __init__(self, package_name, package_path="templates", encoding="utf-8"): + from pkg_resources import DefaultProvider + from pkg_resources import get_provider + from pkg_resources import ResourceManager + + provider = get_provider(package_name) + self.encoding = encoding + self.manager = ResourceManager() + self.filesystem_bound = isinstance(provider, DefaultProvider) + self.provider = provider + self.package_path = package_path + + def get_source(self, environment, template): + pieces = split_template_path(template) + p = "/".join((self.package_path,) + tuple(pieces)) + + if not self.provider.has_resource(p): + raise TemplateNotFound(template) + + filename = uptodate = None + + if self.filesystem_bound: + filename = self.provider.get_resource_filename(self.manager, p) + mtime = path.getmtime(filename) + + def uptodate(): + try: + return path.getmtime(filename) == mtime + except OSError: + return False + + source = self.provider.get_resource_string(self.manager, p) + return source.decode(self.encoding), filename, uptodate + + def list_templates(self): + path = self.package_path + + if path[:2] == "./": + path = path[2:] + elif path == ".": + path = "" + + offset = len(path) + results = [] + + def _walk(path): + for filename in self.provider.resource_listdir(path): + fullname = path + "/" + filename + + if self.provider.resource_isdir(fullname): + _walk(fullname) + else: + results.append(fullname[offset:].lstrip("/")) + + _walk(path) + results.sort() + return results + + +class DictLoader(BaseLoader): + """Loads a template from a python dict. It's passed a dict of unicode + strings bound to template names. This loader is useful for unittesting: + + >>> loader = DictLoader({'index.html': 'source here'}) + + Because auto reloading is rarely useful this is disabled per default. + """ + + def __init__(self, mapping): + self.mapping = mapping + + def get_source(self, environment, template): + if template in self.mapping: + source = self.mapping[template] + return source, None, lambda: source == self.mapping.get(template) + raise TemplateNotFound(template) + + def list_templates(self): + return sorted(self.mapping) + + +class FunctionLoader(BaseLoader): + """A loader that is passed a function which does the loading. The + function receives the name of the template and has to return either + an unicode string with the template source, a tuple in the form ``(source, + filename, uptodatefunc)`` or `None` if the template does not exist. + + >>> def load_template(name): + ... if name == 'index.html': + ... return '...' + ... + >>> loader = FunctionLoader(load_template) + + The `uptodatefunc` is a function that is called if autoreload is enabled + and has to return `True` if the template is still up to date. For more + details have a look at :meth:`BaseLoader.get_source` which has the same + return value. + """ + + def __init__(self, load_func): + self.load_func = load_func + + def get_source(self, environment, template): + rv = self.load_func(template) + if rv is None: + raise TemplateNotFound(template) + elif isinstance(rv, string_types): + return rv, None, None + return rv + + +class PrefixLoader(BaseLoader): + """A loader that is passed a dict of loaders where each loader is bound + to a prefix. The prefix is delimited from the template by a slash per + default, which can be changed by setting the `delimiter` argument to + something else:: + + loader = PrefixLoader({ + 'app1': PackageLoader('mypackage.app1'), + 'app2': PackageLoader('mypackage.app2') + }) + + By loading ``'app1/index.html'`` the file from the app1 package is loaded, + by loading ``'app2/index.html'`` the file from the second. + """ + + def __init__(self, mapping, delimiter="/"): + self.mapping = mapping + self.delimiter = delimiter + + def get_loader(self, template): + try: + prefix, name = template.split(self.delimiter, 1) + loader = self.mapping[prefix] + except (ValueError, KeyError): + raise TemplateNotFound(template) + return loader, name + + def get_source(self, environment, template): + loader, name = self.get_loader(template) + try: + return loader.get_source(environment, name) + except TemplateNotFound: + # re-raise the exception with the correct filename here. + # (the one that includes the prefix) + raise TemplateNotFound(template) + + @internalcode + def load(self, environment, name, globals=None): + loader, local_name = self.get_loader(name) + try: + return loader.load(environment, local_name, globals) + except TemplateNotFound: + # re-raise the exception with the correct filename here. + # (the one that includes the prefix) + raise TemplateNotFound(name) + + def list_templates(self): + result = [] + for prefix, loader in iteritems(self.mapping): + for template in loader.list_templates(): + result.append(prefix + self.delimiter + template) + return result + + +class ChoiceLoader(BaseLoader): + """This loader works like the `PrefixLoader` just that no prefix is + specified. If a template could not be found by one loader the next one + is tried. + + >>> loader = ChoiceLoader([ + ... FileSystemLoader('/path/to/user/templates'), + ... FileSystemLoader('/path/to/system/templates') + ... ]) + + This is useful if you want to allow users to override builtin templates + from a different location. + """ + + def __init__(self, loaders): + self.loaders = loaders + + def get_source(self, environment, template): + for loader in self.loaders: + try: + return loader.get_source(environment, template) + except TemplateNotFound: + pass + raise TemplateNotFound(template) + + @internalcode + def load(self, environment, name, globals=None): + for loader in self.loaders: + try: + return loader.load(environment, name, globals) + except TemplateNotFound: + pass + raise TemplateNotFound(name) + + def list_templates(self): + found = set() + for loader in self.loaders: + found.update(loader.list_templates()) + return sorted(found) + + +class _TemplateModule(ModuleType): + """Like a normal module but with support for weak references""" + + +class ModuleLoader(BaseLoader): + """This loader loads templates from precompiled templates. + + Example usage: + + >>> loader = ChoiceLoader([ + ... ModuleLoader('/path/to/compiled/templates'), + ... FileSystemLoader('/path/to/templates') + ... ]) + + Templates can be precompiled with :meth:`Environment.compile_templates`. + """ + + has_source_access = False + + def __init__(self, path): + package_name = "_jinja2_module_templates_%x" % id(self) + + # create a fake module that looks for the templates in the + # path given. + mod = _TemplateModule(package_name) + + if not isinstance(path, abc.Iterable) or isinstance(path, string_types): + path = [path] + + mod.__path__ = [fspath(p) for p in path] + + sys.modules[package_name] = weakref.proxy( + mod, lambda x: sys.modules.pop(package_name, None) + ) + + # the only strong reference, the sys.modules entry is weak + # so that the garbage collector can remove it once the + # loader that created it goes out of business. + self.module = mod + self.package_name = package_name + + @staticmethod + def get_template_key(name): + return "tmpl_" + sha1(name.encode("utf-8")).hexdigest() + + @staticmethod + def get_module_filename(name): + return ModuleLoader.get_template_key(name) + ".py" + + @internalcode + def load(self, environment, name, globals=None): + key = self.get_template_key(name) + module = "%s.%s" % (self.package_name, key) + mod = getattr(self.module, module, None) + if mod is None: + try: + mod = __import__(module, None, None, ["root"]) + except ImportError: + raise TemplateNotFound(name) + + # remove the entry from sys.modules, we only want the attribute + # on the module object we have stored on the loader. + sys.modules.pop(module, None) + + return environment.template_class.from_module_dict( + environment, mod.__dict__, globals + ) + + +class ResourceLoader(BaseLoader): + def __init__(self, prefix, module_loader): + self.prefix = prefix + self.module_loader = module_loader + + def get_source(self, environment, template): + if self.module_loader is None: + raise TemplateNotFound(template) + try: + return self.module_loader.get_data(os.path.join(self.prefix, template)).decode('utf-8'), None, None + except IOError: + raise TemplateNotFound(template) diff --git a/contrib/python/Jinja2/py2/jinja2/meta.py b/contrib/python/Jinja2/py2/jinja2/meta.py new file mode 100644 index 0000000000..3795aace59 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/meta.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +"""Functions that expose information about templates that might be +interesting for introspection. +""" +from . import nodes +from ._compat import iteritems +from ._compat import string_types +from .compiler import CodeGenerator + + +class TrackingCodeGenerator(CodeGenerator): + """We abuse the code generator for introspection.""" + + def __init__(self, environment): + CodeGenerator.__init__(self, environment, "<introspection>", "<introspection>") + self.undeclared_identifiers = set() + + def write(self, x): + """Don't write.""" + + def enter_frame(self, frame): + """Remember all undeclared identifiers.""" + CodeGenerator.enter_frame(self, frame) + for _, (action, param) in iteritems(frame.symbols.loads): + if action == "resolve" and param not in self.environment.globals: + self.undeclared_identifiers.add(param) + + +def find_undeclared_variables(ast): + """Returns a set of all variables in the AST that will be looked up from + the context at runtime. Because at compile time it's not known which + variables will be used depending on the path the execution takes at + runtime, all variables are returned. + + >>> from jinja2 import Environment, meta + >>> env = Environment() + >>> ast = env.parse('{% set foo = 42 %}{{ bar + foo }}') + >>> meta.find_undeclared_variables(ast) == set(['bar']) + True + + .. admonition:: Implementation + + Internally the code generator is used for finding undeclared variables. + This is good to know because the code generator might raise a + :exc:`TemplateAssertionError` during compilation and as a matter of + fact this function can currently raise that exception as well. + """ + codegen = TrackingCodeGenerator(ast.environment) + codegen.visit(ast) + return codegen.undeclared_identifiers + + +def find_referenced_templates(ast): + """Finds all the referenced templates from the AST. This will return an + iterator over all the hardcoded template extensions, inclusions and + imports. If dynamic inheritance or inclusion is used, `None` will be + yielded. + + >>> from jinja2 import Environment, meta + >>> env = Environment() + >>> ast = env.parse('{% extends "layout.html" %}{% include helper %}') + >>> list(meta.find_referenced_templates(ast)) + ['layout.html', None] + + This function is useful for dependency tracking. For example if you want + to rebuild parts of the website after a layout template has changed. + """ + for node in ast.find_all( + (nodes.Extends, nodes.FromImport, nodes.Import, nodes.Include) + ): + if not isinstance(node.template, nodes.Const): + # a tuple with some non consts in there + if isinstance(node.template, (nodes.Tuple, nodes.List)): + for template_name in node.template.items: + # something const, only yield the strings and ignore + # non-string consts that really just make no sense + if isinstance(template_name, nodes.Const): + if isinstance(template_name.value, string_types): + yield template_name.value + # something dynamic in there + else: + yield None + # something dynamic we don't know about here + else: + yield None + continue + # constant is a basestring, direct template name + if isinstance(node.template.value, string_types): + yield node.template.value + # a tuple or list (latter *should* not happen) made of consts, + # yield the consts that are strings. We could warn here for + # non string values + elif isinstance(node, nodes.Include) and isinstance( + node.template.value, (tuple, list) + ): + for template_name in node.template.value: + if isinstance(template_name, string_types): + yield template_name + # something else we don't care about, we could warn here + else: + yield None diff --git a/contrib/python/Jinja2/py2/jinja2/nativetypes.py b/contrib/python/Jinja2/py2/jinja2/nativetypes.py new file mode 100644 index 0000000000..9f0d39cd98 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/nativetypes.py @@ -0,0 +1,94 @@ +from ast import literal_eval +from itertools import chain +from itertools import islice + +from . import nodes +from ._compat import text_type +from .compiler import CodeGenerator +from .compiler import has_safe_repr +from .environment import Environment +from .environment import Template + + +def native_concat(nodes): + """Return a native Python type from the list of compiled nodes. If + the result is a single node, its value is returned. Otherwise, the + nodes are concatenated as strings. If the result can be parsed with + :func:`ast.literal_eval`, the parsed value is returned. Otherwise, + the string is returned. + + :param nodes: Iterable of nodes to concatenate. + """ + head = list(islice(nodes, 2)) + + if not head: + return None + + if len(head) == 1: + raw = head[0] + else: + raw = u"".join([text_type(v) for v in chain(head, nodes)]) + + try: + return literal_eval(raw) + except (ValueError, SyntaxError, MemoryError): + return raw + + +class NativeCodeGenerator(CodeGenerator): + """A code generator which renders Python types by not adding + ``to_string()`` around output nodes. + """ + + @staticmethod + def _default_finalize(value): + return value + + def _output_const_repr(self, group): + return repr(u"".join([text_type(v) for v in group])) + + def _output_child_to_const(self, node, frame, finalize): + const = node.as_const(frame.eval_ctx) + + if not has_safe_repr(const): + raise nodes.Impossible() + + if isinstance(node, nodes.TemplateData): + return const + + return finalize.const(const) + + def _output_child_pre(self, node, frame, finalize): + if finalize.src is not None: + self.write(finalize.src) + + def _output_child_post(self, node, frame, finalize): + if finalize.src is not None: + self.write(")") + + +class NativeEnvironment(Environment): + """An environment that renders templates to native Python types.""" + + code_generator_class = NativeCodeGenerator + + +class NativeTemplate(Template): + environment_class = NativeEnvironment + + def render(self, *args, **kwargs): + """Render the template to produce a native Python type. If the + result is a single node, its value is returned. Otherwise, the + nodes are concatenated as strings. If the result can be parsed + with :func:`ast.literal_eval`, the parsed value is returned. + Otherwise, the string is returned. + """ + vars = dict(*args, **kwargs) + + try: + return native_concat(self.root_render_func(self.new_context(vars))) + except Exception: + return self.environment.handle_exception() + + +NativeEnvironment.template_class = NativeTemplate # type: ignore diff --git a/contrib/python/Jinja2/py2/jinja2/nodes.py b/contrib/python/Jinja2/py2/jinja2/nodes.py new file mode 100644 index 0000000000..95bd614a14 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/nodes.py @@ -0,0 +1,1088 @@ +# -*- coding: utf-8 -*- +"""AST nodes generated by the parser for the compiler. Also provides +some node tree helper functions used by the parser and compiler in order +to normalize nodes. +""" +import operator +from collections import deque + +from markupsafe import Markup + +from ._compat import izip +from ._compat import PY2 +from ._compat import text_type +from ._compat import with_metaclass + +_binop_to_func = { + "*": operator.mul, + "/": operator.truediv, + "//": operator.floordiv, + "**": operator.pow, + "%": operator.mod, + "+": operator.add, + "-": operator.sub, +} + +_uaop_to_func = {"not": operator.not_, "+": operator.pos, "-": operator.neg} + +_cmpop_to_func = { + "eq": operator.eq, + "ne": operator.ne, + "gt": operator.gt, + "gteq": operator.ge, + "lt": operator.lt, + "lteq": operator.le, + "in": lambda a, b: a in b, + "notin": lambda a, b: a not in b, +} + + +class Impossible(Exception): + """Raised if the node could not perform a requested action.""" + + +class NodeType(type): + """A metaclass for nodes that handles the field and attribute + inheritance. fields and attributes from the parent class are + automatically forwarded to the child.""" + + def __new__(mcs, name, bases, d): + for attr in "fields", "attributes": + storage = [] + storage.extend(getattr(bases[0], attr, ())) + storage.extend(d.get(attr, ())) + assert len(bases) == 1, "multiple inheritance not allowed" + assert len(storage) == len(set(storage)), "layout conflict" + d[attr] = tuple(storage) + d.setdefault("abstract", False) + return type.__new__(mcs, name, bases, d) + + +class EvalContext(object): + """Holds evaluation time information. Custom attributes can be attached + to it in extensions. + """ + + def __init__(self, environment, template_name=None): + self.environment = environment + if callable(environment.autoescape): + self.autoescape = environment.autoescape(template_name) + else: + self.autoescape = environment.autoescape + self.volatile = False + + def save(self): + return self.__dict__.copy() + + def revert(self, old): + self.__dict__.clear() + self.__dict__.update(old) + + +def get_eval_context(node, ctx): + if ctx is None: + if node.environment is None: + raise RuntimeError( + "if no eval context is passed, the " + "node must have an attached " + "environment." + ) + return EvalContext(node.environment) + return ctx + + +class Node(with_metaclass(NodeType, object)): + """Baseclass for all Jinja nodes. There are a number of nodes available + of different types. There are four major types: + + - :class:`Stmt`: statements + - :class:`Expr`: expressions + - :class:`Helper`: helper nodes + - :class:`Template`: the outermost wrapper node + + All nodes have fields and attributes. Fields may be other nodes, lists, + or arbitrary values. Fields are passed to the constructor as regular + positional arguments, attributes as keyword arguments. Each node has + two attributes: `lineno` (the line number of the node) and `environment`. + The `environment` attribute is set at the end of the parsing process for + all nodes automatically. + """ + + fields = () + attributes = ("lineno", "environment") + abstract = True + + def __init__(self, *fields, **attributes): + if self.abstract: + raise TypeError("abstract nodes are not instantiable") + if fields: + if len(fields) != len(self.fields): + if not self.fields: + raise TypeError("%r takes 0 arguments" % self.__class__.__name__) + raise TypeError( + "%r takes 0 or %d argument%s" + % ( + self.__class__.__name__, + len(self.fields), + len(self.fields) != 1 and "s" or "", + ) + ) + for name, arg in izip(self.fields, fields): + setattr(self, name, arg) + for attr in self.attributes: + setattr(self, attr, attributes.pop(attr, None)) + if attributes: + raise TypeError("unknown attribute %r" % next(iter(attributes))) + + def iter_fields(self, exclude=None, only=None): + """This method iterates over all fields that are defined and yields + ``(key, value)`` tuples. Per default all fields are returned, but + it's possible to limit that to some fields by providing the `only` + parameter or to exclude some using the `exclude` parameter. Both + should be sets or tuples of field names. + """ + for name in self.fields: + if ( + (exclude is only is None) + or (exclude is not None and name not in exclude) + or (only is not None and name in only) + ): + try: + yield name, getattr(self, name) + except AttributeError: + pass + + def iter_child_nodes(self, exclude=None, only=None): + """Iterates over all direct child nodes of the node. This iterates + over all fields and yields the values of they are nodes. If the value + of a field is a list all the nodes in that list are returned. + """ + for _, item in self.iter_fields(exclude, only): + if isinstance(item, list): + for n in item: + if isinstance(n, Node): + yield n + elif isinstance(item, Node): + yield item + + def find(self, node_type): + """Find the first node of a given type. If no such node exists the + return value is `None`. + """ + for result in self.find_all(node_type): + return result + + def find_all(self, node_type): + """Find all the nodes of a given type. If the type is a tuple, + the check is performed for any of the tuple items. + """ + for child in self.iter_child_nodes(): + if isinstance(child, node_type): + yield child + for result in child.find_all(node_type): + yield result + + def set_ctx(self, ctx): + """Reset the context of a node and all child nodes. Per default the + parser will all generate nodes that have a 'load' context as it's the + most common one. This method is used in the parser to set assignment + targets and other nodes to a store context. + """ + todo = deque([self]) + while todo: + node = todo.popleft() + if "ctx" in node.fields: + node.ctx = ctx + todo.extend(node.iter_child_nodes()) + return self + + def set_lineno(self, lineno, override=False): + """Set the line numbers of the node and children.""" + todo = deque([self]) + while todo: + node = todo.popleft() + if "lineno" in node.attributes: + if node.lineno is None or override: + node.lineno = lineno + todo.extend(node.iter_child_nodes()) + return self + + def set_environment(self, environment): + """Set the environment for all nodes.""" + todo = deque([self]) + while todo: + node = todo.popleft() + node.environment = environment + todo.extend(node.iter_child_nodes()) + return self + + def __eq__(self, other): + return type(self) is type(other) and tuple(self.iter_fields()) == tuple( + other.iter_fields() + ) + + def __ne__(self, other): + return not self.__eq__(other) + + # Restore Python 2 hashing behavior on Python 3 + __hash__ = object.__hash__ + + def __repr__(self): + return "%s(%s)" % ( + self.__class__.__name__, + ", ".join("%s=%r" % (arg, getattr(self, arg, None)) for arg in self.fields), + ) + + def dump(self): + def _dump(node): + if not isinstance(node, Node): + buf.append(repr(node)) + return + + buf.append("nodes.%s(" % node.__class__.__name__) + if not node.fields: + buf.append(")") + return + for idx, field in enumerate(node.fields): + if idx: + buf.append(", ") + value = getattr(node, field) + if isinstance(value, list): + buf.append("[") + for idx, item in enumerate(value): + if idx: + buf.append(", ") + _dump(item) + buf.append("]") + else: + _dump(value) + buf.append(")") + + buf = [] + _dump(self) + return "".join(buf) + + +class Stmt(Node): + """Base node for all statements.""" + + abstract = True + + +class Helper(Node): + """Nodes that exist in a specific context only.""" + + abstract = True + + +class Template(Node): + """Node that represents a template. This must be the outermost node that + is passed to the compiler. + """ + + fields = ("body",) + + +class Output(Stmt): + """A node that holds multiple expressions which are then printed out. + This is used both for the `print` statement and the regular template data. + """ + + fields = ("nodes",) + + +class Extends(Stmt): + """Represents an extends statement.""" + + fields = ("template",) + + +class For(Stmt): + """The for loop. `target` is the target for the iteration (usually a + :class:`Name` or :class:`Tuple`), `iter` the iterable. `body` is a list + of nodes that are used as loop-body, and `else_` a list of nodes for the + `else` block. If no else node exists it has to be an empty list. + + For filtered nodes an expression can be stored as `test`, otherwise `None`. + """ + + fields = ("target", "iter", "body", "else_", "test", "recursive") + + +class If(Stmt): + """If `test` is true, `body` is rendered, else `else_`.""" + + fields = ("test", "body", "elif_", "else_") + + +class Macro(Stmt): + """A macro definition. `name` is the name of the macro, `args` a list of + arguments and `defaults` a list of defaults if there are any. `body` is + a list of nodes for the macro body. + """ + + fields = ("name", "args", "defaults", "body") + + +class CallBlock(Stmt): + """Like a macro without a name but a call instead. `call` is called with + the unnamed macro as `caller` argument this node holds. + """ + + fields = ("call", "args", "defaults", "body") + + +class FilterBlock(Stmt): + """Node for filter sections.""" + + fields = ("body", "filter") + + +class With(Stmt): + """Specific node for with statements. In older versions of Jinja the + with statement was implemented on the base of the `Scope` node instead. + + .. versionadded:: 2.9.3 + """ + + fields = ("targets", "values", "body") + + +class Block(Stmt): + """A node that represents a block.""" + + fields = ("name", "body", "scoped") + + +class Include(Stmt): + """A node that represents the include tag.""" + + fields = ("template", "with_context", "ignore_missing") + + +class Import(Stmt): + """A node that represents the import tag.""" + + fields = ("template", "target", "with_context") + + +class FromImport(Stmt): + """A node that represents the from import tag. It's important to not + pass unsafe names to the name attribute. The compiler translates the + attribute lookups directly into getattr calls and does *not* use the + subscript callback of the interface. As exported variables may not + start with double underscores (which the parser asserts) this is not a + problem for regular Jinja code, but if this node is used in an extension + extra care must be taken. + + The list of names may contain tuples if aliases are wanted. + """ + + fields = ("template", "names", "with_context") + + +class ExprStmt(Stmt): + """A statement that evaluates an expression and discards the result.""" + + fields = ("node",) + + +class Assign(Stmt): + """Assigns an expression to a target.""" + + fields = ("target", "node") + + +class AssignBlock(Stmt): + """Assigns a block to a target.""" + + fields = ("target", "filter", "body") + + +class Expr(Node): + """Baseclass for all expressions.""" + + abstract = True + + def as_const(self, eval_ctx=None): + """Return the value of the expression as constant or raise + :exc:`Impossible` if this was not possible. + + An :class:`EvalContext` can be provided, if none is given + a default context is created which requires the nodes to have + an attached environment. + + .. versionchanged:: 2.4 + the `eval_ctx` parameter was added. + """ + raise Impossible() + + def can_assign(self): + """Check if it's possible to assign something to this node.""" + return False + + +class BinExpr(Expr): + """Baseclass for all binary expressions.""" + + fields = ("left", "right") + operator = None + abstract = True + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + # intercepted operators cannot be folded at compile time + if ( + self.environment.sandboxed + and self.operator in self.environment.intercepted_binops + ): + raise Impossible() + f = _binop_to_func[self.operator] + try: + return f(self.left.as_const(eval_ctx), self.right.as_const(eval_ctx)) + except Exception: + raise Impossible() + + +class UnaryExpr(Expr): + """Baseclass for all unary expressions.""" + + fields = ("node",) + operator = None + abstract = True + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + # intercepted operators cannot be folded at compile time + if ( + self.environment.sandboxed + and self.operator in self.environment.intercepted_unops + ): + raise Impossible() + f = _uaop_to_func[self.operator] + try: + return f(self.node.as_const(eval_ctx)) + except Exception: + raise Impossible() + + +class Name(Expr): + """Looks up a name or stores a value in a name. + The `ctx` of the node can be one of the following values: + + - `store`: store a value in the name + - `load`: load that name + - `param`: like `store` but if the name was defined as function parameter. + """ + + fields = ("name", "ctx") + + def can_assign(self): + return self.name not in ("true", "false", "none", "True", "False", "None") + + +class NSRef(Expr): + """Reference to a namespace value assignment""" + + fields = ("name", "attr") + + def can_assign(self): + # We don't need any special checks here; NSRef assignments have a + # runtime check to ensure the target is a namespace object which will + # have been checked already as it is created using a normal assignment + # which goes through a `Name` node. + return True + + +class Literal(Expr): + """Baseclass for literals.""" + + abstract = True + + +class Const(Literal): + """All constant values. The parser will return this node for simple + constants such as ``42`` or ``"foo"`` but it can be used to store more + complex values such as lists too. Only constants with a safe + representation (objects where ``eval(repr(x)) == x`` is true). + """ + + fields = ("value",) + + def as_const(self, eval_ctx=None): + rv = self.value + if ( + PY2 + and type(rv) is text_type + and self.environment.policies["compiler.ascii_str"] + ): + try: + rv = rv.encode("ascii") + except UnicodeError: + pass + return rv + + @classmethod + def from_untrusted(cls, value, lineno=None, environment=None): + """Return a const object if the value is representable as + constant value in the generated code, otherwise it will raise + an `Impossible` exception. + """ + from .compiler import has_safe_repr + + if not has_safe_repr(value): + raise Impossible() + return cls(value, lineno=lineno, environment=environment) + + +class TemplateData(Literal): + """A constant template string.""" + + fields = ("data",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + if eval_ctx.volatile: + raise Impossible() + if eval_ctx.autoescape: + return Markup(self.data) + return self.data + + +class Tuple(Literal): + """For loop unpacking and some other things like multiple arguments + for subscripts. Like for :class:`Name` `ctx` specifies if the tuple + is used for loading the names or storing. + """ + + fields = ("items", "ctx") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return tuple(x.as_const(eval_ctx) for x in self.items) + + def can_assign(self): + for item in self.items: + if not item.can_assign(): + return False + return True + + +class List(Literal): + """Any list literal such as ``[1, 2, 3]``""" + + fields = ("items",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return [x.as_const(eval_ctx) for x in self.items] + + +class Dict(Literal): + """Any dict literal such as ``{1: 2, 3: 4}``. The items must be a list of + :class:`Pair` nodes. + """ + + fields = ("items",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return dict(x.as_const(eval_ctx) for x in self.items) + + +class Pair(Helper): + """A key, value pair for dicts.""" + + fields = ("key", "value") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return self.key.as_const(eval_ctx), self.value.as_const(eval_ctx) + + +class Keyword(Helper): + """A key, value pair for keyword arguments where key is a string.""" + + fields = ("key", "value") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return self.key, self.value.as_const(eval_ctx) + + +class CondExpr(Expr): + """A conditional expression (inline if expression). (``{{ + foo if bar else baz }}``) + """ + + fields = ("test", "expr1", "expr2") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + if self.test.as_const(eval_ctx): + return self.expr1.as_const(eval_ctx) + + # if we evaluate to an undefined object, we better do that at runtime + if self.expr2 is None: + raise Impossible() + + return self.expr2.as_const(eval_ctx) + + +def args_as_const(node, eval_ctx): + args = [x.as_const(eval_ctx) for x in node.args] + kwargs = dict(x.as_const(eval_ctx) for x in node.kwargs) + + if node.dyn_args is not None: + try: + args.extend(node.dyn_args.as_const(eval_ctx)) + except Exception: + raise Impossible() + + if node.dyn_kwargs is not None: + try: + kwargs.update(node.dyn_kwargs.as_const(eval_ctx)) + except Exception: + raise Impossible() + + return args, kwargs + + +class Filter(Expr): + """This node applies a filter on an expression. `name` is the name of + the filter, the rest of the fields are the same as for :class:`Call`. + + If the `node` of a filter is `None` the contents of the last buffer are + filtered. Buffers are created by macros and filter blocks. + """ + + fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + + if eval_ctx.volatile or self.node is None: + raise Impossible() + + # we have to be careful here because we call filter_ below. + # if this variable would be called filter, 2to3 would wrap the + # call in a list because it is assuming we are talking about the + # builtin filter function here which no longer returns a list in + # python 3. because of that, do not rename filter_ to filter! + filter_ = self.environment.filters.get(self.name) + + if filter_ is None or getattr(filter_, "contextfilter", False) is True: + raise Impossible() + + # We cannot constant handle async filters, so we need to make sure + # to not go down this path. + if eval_ctx.environment.is_async and getattr( + filter_, "asyncfiltervariant", False + ): + raise Impossible() + + args, kwargs = args_as_const(self, eval_ctx) + args.insert(0, self.node.as_const(eval_ctx)) + + if getattr(filter_, "evalcontextfilter", False) is True: + args.insert(0, eval_ctx) + elif getattr(filter_, "environmentfilter", False) is True: + args.insert(0, self.environment) + + try: + return filter_(*args, **kwargs) + except Exception: + raise Impossible() + + +class Test(Expr): + """Applies a test on an expression. `name` is the name of the test, the + rest of the fields are the same as for :class:`Call`. + """ + + fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs") + + def as_const(self, eval_ctx=None): + test = self.environment.tests.get(self.name) + + if test is None: + raise Impossible() + + eval_ctx = get_eval_context(self, eval_ctx) + args, kwargs = args_as_const(self, eval_ctx) + args.insert(0, self.node.as_const(eval_ctx)) + + try: + return test(*args, **kwargs) + except Exception: + raise Impossible() + + +class Call(Expr): + """Calls an expression. `args` is a list of arguments, `kwargs` a list + of keyword arguments (list of :class:`Keyword` nodes), and `dyn_args` + and `dyn_kwargs` has to be either `None` or a node that is used as + node for dynamic positional (``*args``) or keyword (``**kwargs``) + arguments. + """ + + fields = ("node", "args", "kwargs", "dyn_args", "dyn_kwargs") + + +class Getitem(Expr): + """Get an attribute or item from an expression and prefer the item.""" + + fields = ("node", "arg", "ctx") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + if self.ctx != "load": + raise Impossible() + try: + return self.environment.getitem( + self.node.as_const(eval_ctx), self.arg.as_const(eval_ctx) + ) + except Exception: + raise Impossible() + + def can_assign(self): + return False + + +class Getattr(Expr): + """Get an attribute or item from an expression that is a ascii-only + bytestring and prefer the attribute. + """ + + fields = ("node", "attr", "ctx") + + def as_const(self, eval_ctx=None): + if self.ctx != "load": + raise Impossible() + try: + eval_ctx = get_eval_context(self, eval_ctx) + return self.environment.getattr(self.node.as_const(eval_ctx), self.attr) + except Exception: + raise Impossible() + + def can_assign(self): + return False + + +class Slice(Expr): + """Represents a slice object. This must only be used as argument for + :class:`Subscript`. + """ + + fields = ("start", "stop", "step") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + + def const(obj): + if obj is None: + return None + return obj.as_const(eval_ctx) + + return slice(const(self.start), const(self.stop), const(self.step)) + + +class Concat(Expr): + """Concatenates the list of expressions provided after converting them to + unicode. + """ + + fields = ("nodes",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return "".join(text_type(x.as_const(eval_ctx)) for x in self.nodes) + + +class Compare(Expr): + """Compares an expression with some other expressions. `ops` must be a + list of :class:`Operand`\\s. + """ + + fields = ("expr", "ops") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + result = value = self.expr.as_const(eval_ctx) + + try: + for op in self.ops: + new_value = op.expr.as_const(eval_ctx) + result = _cmpop_to_func[op.op](value, new_value) + + if not result: + return False + + value = new_value + except Exception: + raise Impossible() + + return result + + +class Operand(Helper): + """Holds an operator and an expression.""" + + fields = ("op", "expr") + + +if __debug__: + Operand.__doc__ += "\nThe following operators are available: " + ", ".join( + sorted( + "``%s``" % x + for x in set(_binop_to_func) | set(_uaop_to_func) | set(_cmpop_to_func) + ) + ) + + +class Mul(BinExpr): + """Multiplies the left with the right node.""" + + operator = "*" + + +class Div(BinExpr): + """Divides the left by the right node.""" + + operator = "/" + + +class FloorDiv(BinExpr): + """Divides the left by the right node and truncates conver the + result into an integer by truncating. + """ + + operator = "//" + + +class Add(BinExpr): + """Add the left to the right node.""" + + operator = "+" + + +class Sub(BinExpr): + """Subtract the right from the left node.""" + + operator = "-" + + +class Mod(BinExpr): + """Left modulo right.""" + + operator = "%" + + +class Pow(BinExpr): + """Left to the power of right.""" + + operator = "**" + + +class And(BinExpr): + """Short circuited AND.""" + + operator = "and" + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return self.left.as_const(eval_ctx) and self.right.as_const(eval_ctx) + + +class Or(BinExpr): + """Short circuited OR.""" + + operator = "or" + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return self.left.as_const(eval_ctx) or self.right.as_const(eval_ctx) + + +class Not(UnaryExpr): + """Negate the expression.""" + + operator = "not" + + +class Neg(UnaryExpr): + """Make the expression negative.""" + + operator = "-" + + +class Pos(UnaryExpr): + """Make the expression positive (noop for most expressions)""" + + operator = "+" + + +# Helpers for extensions + + +class EnvironmentAttribute(Expr): + """Loads an attribute from the environment object. This is useful for + extensions that want to call a callback stored on the environment. + """ + + fields = ("name",) + + +class ExtensionAttribute(Expr): + """Returns the attribute of an extension bound to the environment. + The identifier is the identifier of the :class:`Extension`. + + This node is usually constructed by calling the + :meth:`~jinja2.ext.Extension.attr` method on an extension. + """ + + fields = ("identifier", "name") + + +class ImportedName(Expr): + """If created with an import name the import name is returned on node + access. For example ``ImportedName('cgi.escape')`` returns the `escape` + function from the cgi module on evaluation. Imports are optimized by the + compiler so there is no need to assign them to local variables. + """ + + fields = ("importname",) + + +class InternalName(Expr): + """An internal name in the compiler. You cannot create these nodes + yourself but the parser provides a + :meth:`~jinja2.parser.Parser.free_identifier` method that creates + a new identifier for you. This identifier is not available from the + template and is not threated specially by the compiler. + """ + + fields = ("name",) + + def __init__(self): + raise TypeError( + "Can't create internal names. Use the " + "`free_identifier` method on a parser." + ) + + +class MarkSafe(Expr): + """Mark the wrapped expression as safe (wrap it as `Markup`).""" + + fields = ("expr",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return Markup(self.expr.as_const(eval_ctx)) + + +class MarkSafeIfAutoescape(Expr): + """Mark the wrapped expression as safe (wrap it as `Markup`) but + only if autoescaping is active. + + .. versionadded:: 2.5 + """ + + fields = ("expr",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + if eval_ctx.volatile: + raise Impossible() + expr = self.expr.as_const(eval_ctx) + if eval_ctx.autoescape: + return Markup(expr) + return expr + + +class ContextReference(Expr): + """Returns the current template context. It can be used like a + :class:`Name` node, with a ``'load'`` ctx and will return the + current :class:`~jinja2.runtime.Context` object. + + Here an example that assigns the current template name to a + variable named `foo`:: + + Assign(Name('foo', ctx='store'), + Getattr(ContextReference(), 'name')) + + This is basically equivalent to using the + :func:`~jinja2.contextfunction` decorator when using the + high-level API, which causes a reference to the context to be passed + as the first argument to a function. + """ + + +class DerivedContextReference(Expr): + """Return the current template context including locals. Behaves + exactly like :class:`ContextReference`, but includes local + variables, such as from a ``for`` loop. + + .. versionadded:: 2.11 + """ + + +class Continue(Stmt): + """Continue a loop.""" + + +class Break(Stmt): + """Break a loop.""" + + +class Scope(Stmt): + """An artificial scope.""" + + fields = ("body",) + + +class OverlayScope(Stmt): + """An overlay scope for extensions. This is a largely unoptimized scope + that however can be used to introduce completely arbitrary variables into + a sub scope from a dictionary or dictionary like object. The `context` + field has to evaluate to a dictionary object. + + Example usage:: + + OverlayScope(context=self.call_method('get_context'), + body=[...]) + + .. versionadded:: 2.10 + """ + + fields = ("context", "body") + + +class EvalContextModifier(Stmt): + """Modifies the eval context. For each option that should be modified, + a :class:`Keyword` has to be added to the :attr:`options` list. + + Example to change the `autoescape` setting:: + + EvalContextModifier(options=[Keyword('autoescape', Const(True))]) + """ + + fields = ("options",) + + +class ScopedEvalContextModifier(EvalContextModifier): + """Modifies the eval context and reverts it later. Works exactly like + :class:`EvalContextModifier` but will only modify the + :class:`~jinja2.nodes.EvalContext` for nodes in the :attr:`body`. + """ + + fields = ("body",) + + +# make sure nobody creates custom nodes +def _failing_new(*args, **kwargs): + raise TypeError("can't create custom node types") + + +NodeType.__new__ = staticmethod(_failing_new) +del _failing_new diff --git a/contrib/python/Jinja2/py2/jinja2/optimizer.py b/contrib/python/Jinja2/py2/jinja2/optimizer.py new file mode 100644 index 0000000000..7bc78c4524 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/optimizer.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +"""The optimizer tries to constant fold expressions and modify the AST +in place so that it should be faster to evaluate. + +Because the AST does not contain all the scoping information and the +compiler has to find that out, we cannot do all the optimizations we +want. For example, loop unrolling doesn't work because unrolled loops +would have a different scope. The solution would be a second syntax tree +that stored the scoping rules. +""" +from . import nodes +from .visitor import NodeTransformer + + +def optimize(node, environment): + """The context hint can be used to perform an static optimization + based on the context given.""" + optimizer = Optimizer(environment) + return optimizer.visit(node) + + +class Optimizer(NodeTransformer): + def __init__(self, environment): + self.environment = environment + + def generic_visit(self, node, *args, **kwargs): + node = super(Optimizer, self).generic_visit(node, *args, **kwargs) + + # Do constant folding. Some other nodes besides Expr have + # as_const, but folding them causes errors later on. + if isinstance(node, nodes.Expr): + try: + return nodes.Const.from_untrusted( + node.as_const(args[0] if args else None), + lineno=node.lineno, + environment=self.environment, + ) + except nodes.Impossible: + pass + + return node diff --git a/contrib/python/Jinja2/py2/jinja2/parser.py b/contrib/python/Jinja2/py2/jinja2/parser.py new file mode 100644 index 0000000000..d5881066f7 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/parser.py @@ -0,0 +1,939 @@ +# -*- coding: utf-8 -*- +"""Parse tokens from the lexer into nodes for the compiler.""" +from . import nodes +from ._compat import imap +from .exceptions import TemplateAssertionError +from .exceptions import TemplateSyntaxError +from .lexer import describe_token +from .lexer import describe_token_expr + +_statement_keywords = frozenset( + [ + "for", + "if", + "block", + "extends", + "print", + "macro", + "include", + "from", + "import", + "set", + "with", + "autoescape", + ] +) +_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"]) + +_math_nodes = { + "add": nodes.Add, + "sub": nodes.Sub, + "mul": nodes.Mul, + "div": nodes.Div, + "floordiv": nodes.FloorDiv, + "mod": nodes.Mod, +} + + +class Parser(object): + """This is the central parsing class Jinja uses. It's passed to + extensions and can be used to parse expressions or statements. + """ + + def __init__(self, environment, source, name=None, filename=None, state=None): + self.environment = environment + self.stream = environment._tokenize(source, name, filename, state) + self.name = name + self.filename = filename + self.closed = False + self.extensions = {} + for extension in environment.iter_extensions(): + for tag in extension.tags: + self.extensions[tag] = extension.parse + self._last_identifier = 0 + self._tag_stack = [] + self._end_token_stack = [] + + def fail(self, msg, lineno=None, exc=TemplateSyntaxError): + """Convenience method that raises `exc` with the message, passed + line number or last line number as well as the current name and + filename. + """ + if lineno is None: + lineno = self.stream.current.lineno + raise exc(msg, lineno, self.name, self.filename) + + def _fail_ut_eof(self, name, end_token_stack, lineno): + expected = [] + for exprs in end_token_stack: + expected.extend(imap(describe_token_expr, exprs)) + if end_token_stack: + currently_looking = " or ".join( + "'%s'" % describe_token_expr(expr) for expr in end_token_stack[-1] + ) + else: + currently_looking = None + + if name is None: + message = ["Unexpected end of template."] + else: + message = ["Encountered unknown tag '%s'." % name] + + if currently_looking: + if name is not None and name in expected: + message.append( + "You probably made a nesting mistake. Jinja " + "is expecting this tag, but currently looking " + "for %s." % currently_looking + ) + else: + message.append( + "Jinja was looking for the following tags: " + "%s." % currently_looking + ) + + if self._tag_stack: + message.append( + "The innermost block that needs to be " + "closed is '%s'." % self._tag_stack[-1] + ) + + self.fail(" ".join(message), lineno) + + def fail_unknown_tag(self, name, lineno=None): + """Called if the parser encounters an unknown tag. Tries to fail + with a human readable error message that could help to identify + the problem. + """ + return self._fail_ut_eof(name, self._end_token_stack, lineno) + + def fail_eof(self, end_tokens=None, lineno=None): + """Like fail_unknown_tag but for end of template situations.""" + stack = list(self._end_token_stack) + if end_tokens is not None: + stack.append(end_tokens) + return self._fail_ut_eof(None, stack, lineno) + + def is_tuple_end(self, extra_end_rules=None): + """Are we at the end of a tuple?""" + if self.stream.current.type in ("variable_end", "block_end", "rparen"): + return True + elif extra_end_rules is not None: + return self.stream.current.test_any(extra_end_rules) + return False + + def free_identifier(self, lineno=None): + """Return a new free identifier as :class:`~jinja2.nodes.InternalName`.""" + self._last_identifier += 1 + rv = object.__new__(nodes.InternalName) + nodes.Node.__init__(rv, "fi%d" % self._last_identifier, lineno=lineno) + return rv + + def parse_statement(self): + """Parse a single statement.""" + token = self.stream.current + if token.type != "name": + self.fail("tag name expected", token.lineno) + self._tag_stack.append(token.value) + pop_tag = True + try: + if token.value in _statement_keywords: + return getattr(self, "parse_" + self.stream.current.value)() + if token.value == "call": + return self.parse_call_block() + if token.value == "filter": + return self.parse_filter_block() + ext = self.extensions.get(token.value) + if ext is not None: + return ext(self) + + # did not work out, remove the token we pushed by accident + # from the stack so that the unknown tag fail function can + # produce a proper error message. + self._tag_stack.pop() + pop_tag = False + self.fail_unknown_tag(token.value, token.lineno) + finally: + if pop_tag: + self._tag_stack.pop() + + def parse_statements(self, end_tokens, drop_needle=False): + """Parse multiple statements into a list until one of the end tokens + is reached. This is used to parse the body of statements as it also + parses template data if appropriate. The parser checks first if the + current token is a colon and skips it if there is one. Then it checks + for the block end and parses until if one of the `end_tokens` is + reached. Per default the active token in the stream at the end of + the call is the matched end token. If this is not wanted `drop_needle` + can be set to `True` and the end token is removed. + """ + # the first token may be a colon for python compatibility + self.stream.skip_if("colon") + + # in the future it would be possible to add whole code sections + # by adding some sort of end of statement token and parsing those here. + self.stream.expect("block_end") + result = self.subparse(end_tokens) + + # we reached the end of the template too early, the subparser + # does not check for this, so we do that now + if self.stream.current.type == "eof": + self.fail_eof(end_tokens) + + if drop_needle: + next(self.stream) + return result + + def parse_set(self): + """Parse an assign statement.""" + lineno = next(self.stream).lineno + target = self.parse_assign_target(with_namespace=True) + if self.stream.skip_if("assign"): + expr = self.parse_tuple() + return nodes.Assign(target, expr, lineno=lineno) + filter_node = self.parse_filter(None) + body = self.parse_statements(("name:endset",), drop_needle=True) + return nodes.AssignBlock(target, filter_node, body, lineno=lineno) + + def parse_for(self): + """Parse a for loop.""" + lineno = self.stream.expect("name:for").lineno + target = self.parse_assign_target(extra_end_rules=("name:in",)) + self.stream.expect("name:in") + iter = self.parse_tuple( + with_condexpr=False, extra_end_rules=("name:recursive",) + ) + test = None + if self.stream.skip_if("name:if"): + test = self.parse_expression() + recursive = self.stream.skip_if("name:recursive") + body = self.parse_statements(("name:endfor", "name:else")) + if next(self.stream).value == "endfor": + else_ = [] + else: + else_ = self.parse_statements(("name:endfor",), drop_needle=True) + return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno) + + def parse_if(self): + """Parse an if construct.""" + node = result = nodes.If(lineno=self.stream.expect("name:if").lineno) + while 1: + node.test = self.parse_tuple(with_condexpr=False) + node.body = self.parse_statements(("name:elif", "name:else", "name:endif")) + node.elif_ = [] + node.else_ = [] + token = next(self.stream) + if token.test("name:elif"): + node = nodes.If(lineno=self.stream.current.lineno) + result.elif_.append(node) + continue + elif token.test("name:else"): + result.else_ = self.parse_statements(("name:endif",), drop_needle=True) + break + return result + + def parse_with(self): + node = nodes.With(lineno=next(self.stream).lineno) + targets = [] + values = [] + while self.stream.current.type != "block_end": + if targets: + self.stream.expect("comma") + target = self.parse_assign_target() + target.set_ctx("param") + targets.append(target) + self.stream.expect("assign") + values.append(self.parse_expression()) + node.targets = targets + node.values = values + node.body = self.parse_statements(("name:endwith",), drop_needle=True) + return node + + def parse_autoescape(self): + node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno) + node.options = [nodes.Keyword("autoescape", self.parse_expression())] + node.body = self.parse_statements(("name:endautoescape",), drop_needle=True) + return nodes.Scope([node]) + + def parse_block(self): + node = nodes.Block(lineno=next(self.stream).lineno) + node.name = self.stream.expect("name").value + node.scoped = self.stream.skip_if("name:scoped") + + # common problem people encounter when switching from django + # to jinja. we do not support hyphens in block names, so let's + # raise a nicer error message in that case. + if self.stream.current.type == "sub": + self.fail( + "Block names in Jinja have to be valid Python " + "identifiers and may not contain hyphens, use an " + "underscore instead." + ) + + node.body = self.parse_statements(("name:endblock",), drop_needle=True) + self.stream.skip_if("name:" + node.name) + return node + + def parse_extends(self): + node = nodes.Extends(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + return node + + def parse_import_context(self, node, default): + if self.stream.current.test_any( + "name:with", "name:without" + ) and self.stream.look().test("name:context"): + node.with_context = next(self.stream).value == "with" + self.stream.skip() + else: + node.with_context = default + return node + + def parse_include(self): + node = nodes.Include(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + if self.stream.current.test("name:ignore") and self.stream.look().test( + "name:missing" + ): + node.ignore_missing = True + self.stream.skip(2) + else: + node.ignore_missing = False + return self.parse_import_context(node, True) + + def parse_import(self): + node = nodes.Import(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + self.stream.expect("name:as") + node.target = self.parse_assign_target(name_only=True).name + return self.parse_import_context(node, False) + + def parse_from(self): + node = nodes.FromImport(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + self.stream.expect("name:import") + node.names = [] + + def parse_context(): + if self.stream.current.value in ( + "with", + "without", + ) and self.stream.look().test("name:context"): + node.with_context = next(self.stream).value == "with" + self.stream.skip() + return True + return False + + while 1: + if node.names: + self.stream.expect("comma") + if self.stream.current.type == "name": + if parse_context(): + break + target = self.parse_assign_target(name_only=True) + if target.name.startswith("_"): + self.fail( + "names starting with an underline can not be imported", + target.lineno, + exc=TemplateAssertionError, + ) + if self.stream.skip_if("name:as"): + alias = self.parse_assign_target(name_only=True) + node.names.append((target.name, alias.name)) + else: + node.names.append(target.name) + if parse_context() or self.stream.current.type != "comma": + break + else: + self.stream.expect("name") + if not hasattr(node, "with_context"): + node.with_context = False + return node + + def parse_signature(self, node): + node.args = args = [] + node.defaults = defaults = [] + self.stream.expect("lparen") + while self.stream.current.type != "rparen": + if args: + self.stream.expect("comma") + arg = self.parse_assign_target(name_only=True) + arg.set_ctx("param") + if self.stream.skip_if("assign"): + defaults.append(self.parse_expression()) + elif defaults: + self.fail("non-default argument follows default argument") + args.append(arg) + self.stream.expect("rparen") + + def parse_call_block(self): + node = nodes.CallBlock(lineno=next(self.stream).lineno) + if self.stream.current.type == "lparen": + self.parse_signature(node) + else: + node.args = [] + node.defaults = [] + + node.call = self.parse_expression() + if not isinstance(node.call, nodes.Call): + self.fail("expected call", node.lineno) + node.body = self.parse_statements(("name:endcall",), drop_needle=True) + return node + + def parse_filter_block(self): + node = nodes.FilterBlock(lineno=next(self.stream).lineno) + node.filter = self.parse_filter(None, start_inline=True) + node.body = self.parse_statements(("name:endfilter",), drop_needle=True) + return node + + def parse_macro(self): + node = nodes.Macro(lineno=next(self.stream).lineno) + node.name = self.parse_assign_target(name_only=True).name + self.parse_signature(node) + node.body = self.parse_statements(("name:endmacro",), drop_needle=True) + return node + + def parse_print(self): + node = nodes.Output(lineno=next(self.stream).lineno) + node.nodes = [] + while self.stream.current.type != "block_end": + if node.nodes: + self.stream.expect("comma") + node.nodes.append(self.parse_expression()) + return node + + def parse_assign_target( + self, + with_tuple=True, + name_only=False, + extra_end_rules=None, + with_namespace=False, + ): + """Parse an assignment target. As Jinja allows assignments to + tuples, this function can parse all allowed assignment targets. Per + default assignments to tuples are parsed, that can be disable however + by setting `with_tuple` to `False`. If only assignments to names are + wanted `name_only` can be set to `True`. The `extra_end_rules` + parameter is forwarded to the tuple parsing function. If + `with_namespace` is enabled, a namespace assignment may be parsed. + """ + if with_namespace and self.stream.look().type == "dot": + token = self.stream.expect("name") + next(self.stream) # dot + attr = self.stream.expect("name") + target = nodes.NSRef(token.value, attr.value, lineno=token.lineno) + elif name_only: + token = self.stream.expect("name") + target = nodes.Name(token.value, "store", lineno=token.lineno) + else: + if with_tuple: + target = self.parse_tuple( + simplified=True, extra_end_rules=extra_end_rules + ) + else: + target = self.parse_primary() + target.set_ctx("store") + if not target.can_assign(): + self.fail( + "can't assign to %r" % target.__class__.__name__.lower(), target.lineno + ) + return target + + def parse_expression(self, with_condexpr=True): + """Parse an expression. Per default all expressions are parsed, if + the optional `with_condexpr` parameter is set to `False` conditional + expressions are not parsed. + """ + if with_condexpr: + return self.parse_condexpr() + return self.parse_or() + + def parse_condexpr(self): + lineno = self.stream.current.lineno + expr1 = self.parse_or() + while self.stream.skip_if("name:if"): + expr2 = self.parse_or() + if self.stream.skip_if("name:else"): + expr3 = self.parse_condexpr() + else: + expr3 = None + expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno) + lineno = self.stream.current.lineno + return expr1 + + def parse_or(self): + lineno = self.stream.current.lineno + left = self.parse_and() + while self.stream.skip_if("name:or"): + right = self.parse_and() + left = nodes.Or(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_and(self): + lineno = self.stream.current.lineno + left = self.parse_not() + while self.stream.skip_if("name:and"): + right = self.parse_not() + left = nodes.And(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_not(self): + if self.stream.current.test("name:not"): + lineno = next(self.stream).lineno + return nodes.Not(self.parse_not(), lineno=lineno) + return self.parse_compare() + + def parse_compare(self): + lineno = self.stream.current.lineno + expr = self.parse_math1() + ops = [] + while 1: + token_type = self.stream.current.type + if token_type in _compare_operators: + next(self.stream) + ops.append(nodes.Operand(token_type, self.parse_math1())) + elif self.stream.skip_if("name:in"): + ops.append(nodes.Operand("in", self.parse_math1())) + elif self.stream.current.test("name:not") and self.stream.look().test( + "name:in" + ): + self.stream.skip(2) + ops.append(nodes.Operand("notin", self.parse_math1())) + else: + break + lineno = self.stream.current.lineno + if not ops: + return expr + return nodes.Compare(expr, ops, lineno=lineno) + + def parse_math1(self): + lineno = self.stream.current.lineno + left = self.parse_concat() + while self.stream.current.type in ("add", "sub"): + cls = _math_nodes[self.stream.current.type] + next(self.stream) + right = self.parse_concat() + left = cls(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_concat(self): + lineno = self.stream.current.lineno + args = [self.parse_math2()] + while self.stream.current.type == "tilde": + next(self.stream) + args.append(self.parse_math2()) + if len(args) == 1: + return args[0] + return nodes.Concat(args, lineno=lineno) + + def parse_math2(self): + lineno = self.stream.current.lineno + left = self.parse_pow() + while self.stream.current.type in ("mul", "div", "floordiv", "mod"): + cls = _math_nodes[self.stream.current.type] + next(self.stream) + right = self.parse_pow() + left = cls(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_pow(self): + lineno = self.stream.current.lineno + left = self.parse_unary() + while self.stream.current.type == "pow": + next(self.stream) + right = self.parse_unary() + left = nodes.Pow(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_unary(self, with_filter=True): + token_type = self.stream.current.type + lineno = self.stream.current.lineno + if token_type == "sub": + next(self.stream) + node = nodes.Neg(self.parse_unary(False), lineno=lineno) + elif token_type == "add": + next(self.stream) + node = nodes.Pos(self.parse_unary(False), lineno=lineno) + else: + node = self.parse_primary() + node = self.parse_postfix(node) + if with_filter: + node = self.parse_filter_expr(node) + return node + + def parse_primary(self): + token = self.stream.current + if token.type == "name": + if token.value in ("true", "false", "True", "False"): + node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) + elif token.value in ("none", "None"): + node = nodes.Const(None, lineno=token.lineno) + else: + node = nodes.Name(token.value, "load", lineno=token.lineno) + next(self.stream) + elif token.type == "string": + next(self.stream) + buf = [token.value] + lineno = token.lineno + while self.stream.current.type == "string": + buf.append(self.stream.current.value) + next(self.stream) + node = nodes.Const("".join(buf), lineno=lineno) + elif token.type in ("integer", "float"): + next(self.stream) + node = nodes.Const(token.value, lineno=token.lineno) + elif token.type == "lparen": + next(self.stream) + node = self.parse_tuple(explicit_parentheses=True) + self.stream.expect("rparen") + elif token.type == "lbracket": + node = self.parse_list() + elif token.type == "lbrace": + node = self.parse_dict() + else: + self.fail("unexpected '%s'" % describe_token(token), token.lineno) + return node + + def parse_tuple( + self, + simplified=False, + with_condexpr=True, + extra_end_rules=None, + explicit_parentheses=False, + ): + """Works like `parse_expression` but if multiple expressions are + delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created. + This method could also return a regular expression instead of a tuple + if no commas where found. + + The default parsing mode is a full tuple. If `simplified` is `True` + only names and literals are parsed. The `no_condexpr` parameter is + forwarded to :meth:`parse_expression`. + + Because tuples do not require delimiters and may end in a bogus comma + an extra hint is needed that marks the end of a tuple. For example + for loops support tuples between `for` and `in`. In that case the + `extra_end_rules` is set to ``['name:in']``. + + `explicit_parentheses` is true if the parsing was triggered by an + expression in parentheses. This is used to figure out if an empty + tuple is a valid expression or not. + """ + lineno = self.stream.current.lineno + if simplified: + parse = self.parse_primary + elif with_condexpr: + parse = self.parse_expression + else: + + def parse(): + return self.parse_expression(with_condexpr=False) + + args = [] + is_tuple = False + while 1: + if args: + self.stream.expect("comma") + if self.is_tuple_end(extra_end_rules): + break + args.append(parse()) + if self.stream.current.type == "comma": + is_tuple = True + else: + break + lineno = self.stream.current.lineno + + if not is_tuple: + if args: + return args[0] + + # if we don't have explicit parentheses, an empty tuple is + # not a valid expression. This would mean nothing (literally + # nothing) in the spot of an expression would be an empty + # tuple. + if not explicit_parentheses: + self.fail( + "Expected an expression, got '%s'" + % describe_token(self.stream.current) + ) + + return nodes.Tuple(args, "load", lineno=lineno) + + def parse_list(self): + token = self.stream.expect("lbracket") + items = [] + while self.stream.current.type != "rbracket": + if items: + self.stream.expect("comma") + if self.stream.current.type == "rbracket": + break + items.append(self.parse_expression()) + self.stream.expect("rbracket") + return nodes.List(items, lineno=token.lineno) + + def parse_dict(self): + token = self.stream.expect("lbrace") + items = [] + while self.stream.current.type != "rbrace": + if items: + self.stream.expect("comma") + if self.stream.current.type == "rbrace": + break + key = self.parse_expression() + self.stream.expect("colon") + value = self.parse_expression() + items.append(nodes.Pair(key, value, lineno=key.lineno)) + self.stream.expect("rbrace") + return nodes.Dict(items, lineno=token.lineno) + + def parse_postfix(self, node): + while 1: + token_type = self.stream.current.type + if token_type == "dot" or token_type == "lbracket": + node = self.parse_subscript(node) + # calls are valid both after postfix expressions (getattr + # and getitem) as well as filters and tests + elif token_type == "lparen": + node = self.parse_call(node) + else: + break + return node + + def parse_filter_expr(self, node): + while 1: + token_type = self.stream.current.type + if token_type == "pipe": + node = self.parse_filter(node) + elif token_type == "name" and self.stream.current.value == "is": + node = self.parse_test(node) + # calls are valid both after postfix expressions (getattr + # and getitem) as well as filters and tests + elif token_type == "lparen": + node = self.parse_call(node) + else: + break + return node + + def parse_subscript(self, node): + token = next(self.stream) + if token.type == "dot": + attr_token = self.stream.current + next(self.stream) + if attr_token.type == "name": + return nodes.Getattr( + node, attr_token.value, "load", lineno=token.lineno + ) + elif attr_token.type != "integer": + self.fail("expected name or number", attr_token.lineno) + arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) + return nodes.Getitem(node, arg, "load", lineno=token.lineno) + if token.type == "lbracket": + args = [] + while self.stream.current.type != "rbracket": + if args: + self.stream.expect("comma") + args.append(self.parse_subscribed()) + self.stream.expect("rbracket") + if len(args) == 1: + arg = args[0] + else: + arg = nodes.Tuple(args, "load", lineno=token.lineno) + return nodes.Getitem(node, arg, "load", lineno=token.lineno) + self.fail("expected subscript expression", token.lineno) + + def parse_subscribed(self): + lineno = self.stream.current.lineno + + if self.stream.current.type == "colon": + next(self.stream) + args = [None] + else: + node = self.parse_expression() + if self.stream.current.type != "colon": + return node + next(self.stream) + args = [node] + + if self.stream.current.type == "colon": + args.append(None) + elif self.stream.current.type not in ("rbracket", "comma"): + args.append(self.parse_expression()) + else: + args.append(None) + + if self.stream.current.type == "colon": + next(self.stream) + if self.stream.current.type not in ("rbracket", "comma"): + args.append(self.parse_expression()) + else: + args.append(None) + else: + args.append(None) + + return nodes.Slice(lineno=lineno, *args) + + def parse_call(self, node): + token = self.stream.expect("lparen") + args = [] + kwargs = [] + dyn_args = dyn_kwargs = None + require_comma = False + + def ensure(expr): + if not expr: + self.fail("invalid syntax for function call expression", token.lineno) + + while self.stream.current.type != "rparen": + if require_comma: + self.stream.expect("comma") + # support for trailing comma + if self.stream.current.type == "rparen": + break + if self.stream.current.type == "mul": + ensure(dyn_args is None and dyn_kwargs is None) + next(self.stream) + dyn_args = self.parse_expression() + elif self.stream.current.type == "pow": + ensure(dyn_kwargs is None) + next(self.stream) + dyn_kwargs = self.parse_expression() + else: + if ( + self.stream.current.type == "name" + and self.stream.look().type == "assign" + ): + # Parsing a kwarg + ensure(dyn_kwargs is None) + key = self.stream.current.value + self.stream.skip(2) + value = self.parse_expression() + kwargs.append(nodes.Keyword(key, value, lineno=value.lineno)) + else: + # Parsing an arg + ensure(dyn_args is None and dyn_kwargs is None and not kwargs) + args.append(self.parse_expression()) + + require_comma = True + self.stream.expect("rparen") + + if node is None: + return args, kwargs, dyn_args, dyn_kwargs + return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno) + + def parse_filter(self, node, start_inline=False): + while self.stream.current.type == "pipe" or start_inline: + if not start_inline: + next(self.stream) + token = self.stream.expect("name") + name = token.value + while self.stream.current.type == "dot": + next(self.stream) + name += "." + self.stream.expect("name").value + if self.stream.current.type == "lparen": + args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None) + else: + args = [] + kwargs = [] + dyn_args = dyn_kwargs = None + node = nodes.Filter( + node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno + ) + start_inline = False + return node + + def parse_test(self, node): + token = next(self.stream) + if self.stream.current.test("name:not"): + next(self.stream) + negated = True + else: + negated = False + name = self.stream.expect("name").value + while self.stream.current.type == "dot": + next(self.stream) + name += "." + self.stream.expect("name").value + dyn_args = dyn_kwargs = None + kwargs = [] + if self.stream.current.type == "lparen": + args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None) + elif self.stream.current.type in ( + "name", + "string", + "integer", + "float", + "lparen", + "lbracket", + "lbrace", + ) and not self.stream.current.test_any("name:else", "name:or", "name:and"): + if self.stream.current.test("name:is"): + self.fail("You cannot chain multiple tests with is") + arg_node = self.parse_primary() + arg_node = self.parse_postfix(arg_node) + args = [arg_node] + else: + args = [] + node = nodes.Test( + node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno + ) + if negated: + node = nodes.Not(node, lineno=token.lineno) + return node + + def subparse(self, end_tokens=None): + body = [] + data_buffer = [] + add_data = data_buffer.append + + if end_tokens is not None: + self._end_token_stack.append(end_tokens) + + def flush_data(): + if data_buffer: + lineno = data_buffer[0].lineno + body.append(nodes.Output(data_buffer[:], lineno=lineno)) + del data_buffer[:] + + try: + while self.stream: + token = self.stream.current + if token.type == "data": + if token.value: + add_data(nodes.TemplateData(token.value, lineno=token.lineno)) + next(self.stream) + elif token.type == "variable_begin": + next(self.stream) + add_data(self.parse_tuple(with_condexpr=True)) + self.stream.expect("variable_end") + elif token.type == "block_begin": + flush_data() + next(self.stream) + if end_tokens is not None and self.stream.current.test_any( + *end_tokens + ): + return body + rv = self.parse_statement() + if isinstance(rv, list): + body.extend(rv) + else: + body.append(rv) + self.stream.expect("block_end") + else: + raise AssertionError("internal parsing error") + + flush_data() + finally: + if end_tokens is not None: + self._end_token_stack.pop() + + return body + + def parse(self): + """Parse the whole template into a `Template` node.""" + result = nodes.Template(self.subparse(), lineno=1) + result.set_environment(self.environment) + return result diff --git a/contrib/python/Jinja2/py2/jinja2/runtime.py b/contrib/python/Jinja2/py2/jinja2/runtime.py new file mode 100644 index 0000000000..3ad7968624 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/runtime.py @@ -0,0 +1,1011 @@ +# -*- coding: utf-8 -*- +"""The runtime functions and state used by compiled templates.""" +import sys +from itertools import chain +from types import MethodType + +from markupsafe import escape # noqa: F401 +from markupsafe import Markup +from markupsafe import soft_unicode + +from ._compat import abc +from ._compat import imap +from ._compat import implements_iterator +from ._compat import implements_to_string +from ._compat import iteritems +from ._compat import PY2 +from ._compat import string_types +from ._compat import text_type +from ._compat import with_metaclass +from .exceptions import TemplateNotFound # noqa: F401 +from .exceptions import TemplateRuntimeError # noqa: F401 +from .exceptions import UndefinedError +from .nodes import EvalContext +from .utils import concat +from .utils import evalcontextfunction +from .utils import internalcode +from .utils import missing +from .utils import Namespace # noqa: F401 +from .utils import object_type_repr + +# these variables are exported to the template runtime +exported = [ + "LoopContext", + "TemplateReference", + "Macro", + "Markup", + "TemplateRuntimeError", + "missing", + "concat", + "escape", + "markup_join", + "unicode_join", + "to_string", + "identity", + "TemplateNotFound", + "Namespace", + "Undefined", +] + +#: the name of the function that is used to convert something into +#: a string. We can just use the text type here. +to_string = text_type + + +def identity(x): + """Returns its argument. Useful for certain things in the + environment. + """ + return x + + +def markup_join(seq): + """Concatenation that escapes if necessary and converts to unicode.""" + buf = [] + iterator = imap(soft_unicode, seq) + for arg in iterator: + buf.append(arg) + if hasattr(arg, "__html__"): + return Markup(u"").join(chain(buf, iterator)) + return concat(buf) + + +def unicode_join(seq): + """Simple args to unicode conversion and concatenation.""" + return concat(imap(text_type, seq)) + + +def new_context( + environment, + template_name, + blocks, + vars=None, + shared=None, + globals=None, + locals=None, +): + """Internal helper for context creation.""" + if vars is None: + vars = {} + if shared: + parent = vars + else: + parent = dict(globals or (), **vars) + if locals: + # if the parent is shared a copy should be created because + # we don't want to modify the dict passed + if shared: + parent = dict(parent) + for key, value in iteritems(locals): + if value is not missing: + parent[key] = value + return environment.context_class(environment, parent, template_name, blocks) + + +class TemplateReference(object): + """The `self` in templates.""" + + def __init__(self, context): + self.__context = context + + def __getitem__(self, name): + blocks = self.__context.blocks[name] + return BlockReference(name, self.__context, blocks, 0) + + def __repr__(self): + return "<%s %r>" % (self.__class__.__name__, self.__context.name) + + +def _get_func(x): + return getattr(x, "__func__", x) + + +class ContextMeta(type): + def __new__(mcs, name, bases, d): + rv = type.__new__(mcs, name, bases, d) + if bases == (): + return rv + + resolve = _get_func(rv.resolve) + default_resolve = _get_func(Context.resolve) + resolve_or_missing = _get_func(rv.resolve_or_missing) + default_resolve_or_missing = _get_func(Context.resolve_or_missing) + + # If we have a changed resolve but no changed default or missing + # resolve we invert the call logic. + if ( + resolve is not default_resolve + and resolve_or_missing is default_resolve_or_missing + ): + rv._legacy_resolve_mode = True + elif ( + resolve is default_resolve + and resolve_or_missing is default_resolve_or_missing + ): + rv._fast_resolve_mode = True + + return rv + + +def resolve_or_missing(context, key, missing=missing): + if key in context.vars: + return context.vars[key] + if key in context.parent: + return context.parent[key] + return missing + + +class Context(with_metaclass(ContextMeta)): + """The template context holds the variables of a template. It stores the + values passed to the template and also the names the template exports. + Creating instances is neither supported nor useful as it's created + automatically at various stages of the template evaluation and should not + be created by hand. + + The context is immutable. Modifications on :attr:`parent` **must not** + happen and modifications on :attr:`vars` are allowed from generated + template code only. Template filters and global functions marked as + :func:`contextfunction`\\s get the active context passed as first argument + and are allowed to access the context read-only. + + The template context supports read only dict operations (`get`, + `keys`, `values`, `items`, `iterkeys`, `itervalues`, `iteritems`, + `__getitem__`, `__contains__`). Additionally there is a :meth:`resolve` + method that doesn't fail with a `KeyError` but returns an + :class:`Undefined` object for missing variables. + """ + + # XXX: we want to eventually make this be a deprecation warning and + # remove it. + _legacy_resolve_mode = False + _fast_resolve_mode = False + + def __init__(self, environment, parent, name, blocks): + self.parent = parent + self.vars = {} + self.environment = environment + self.eval_ctx = EvalContext(self.environment, name) + self.exported_vars = set() + self.name = name + + # create the initial mapping of blocks. Whenever template inheritance + # takes place the runtime will update this mapping with the new blocks + # from the template. + self.blocks = dict((k, [v]) for k, v in iteritems(blocks)) + + # In case we detect the fast resolve mode we can set up an alias + # here that bypasses the legacy code logic. + if self._fast_resolve_mode: + self.resolve_or_missing = MethodType(resolve_or_missing, self) + + def super(self, name, current): + """Render a parent block.""" + try: + blocks = self.blocks[name] + index = blocks.index(current) + 1 + blocks[index] + except LookupError: + return self.environment.undefined( + "there is no parent block called %r." % name, name="super" + ) + return BlockReference(name, self, blocks, index) + + def get(self, key, default=None): + """Returns an item from the template context, if it doesn't exist + `default` is returned. + """ + try: + return self[key] + except KeyError: + return default + + def resolve(self, key): + """Looks up a variable like `__getitem__` or `get` but returns an + :class:`Undefined` object with the name of the name looked up. + """ + if self._legacy_resolve_mode: + rv = resolve_or_missing(self, key) + else: + rv = self.resolve_or_missing(key) + if rv is missing: + return self.environment.undefined(name=key) + return rv + + def resolve_or_missing(self, key): + """Resolves a variable like :meth:`resolve` but returns the + special `missing` value if it cannot be found. + """ + if self._legacy_resolve_mode: + rv = self.resolve(key) + if isinstance(rv, Undefined): + rv = missing + return rv + return resolve_or_missing(self, key) + + def get_exported(self): + """Get a new dict with the exported variables.""" + return dict((k, self.vars[k]) for k in self.exported_vars) + + def get_all(self): + """Return the complete context as dict including the exported + variables. For optimizations reasons this might not return an + actual copy so be careful with using it. + """ + if not self.vars: + return self.parent + if not self.parent: + return self.vars + return dict(self.parent, **self.vars) + + @internalcode + def call(__self, __obj, *args, **kwargs): # noqa: B902 + """Call the callable with the arguments and keyword arguments + provided but inject the active context or environment as first + argument if the callable is a :func:`contextfunction` or + :func:`environmentfunction`. + """ + if __debug__: + __traceback_hide__ = True # noqa + + # Allow callable classes to take a context + if hasattr(__obj, "__call__"): # noqa: B004 + fn = __obj.__call__ + for fn_type in ( + "contextfunction", + "evalcontextfunction", + "environmentfunction", + ): + if hasattr(fn, fn_type): + __obj = fn + break + + if callable(__obj): + if getattr(__obj, "contextfunction", False) is True: + args = (__self,) + args + elif getattr(__obj, "evalcontextfunction", False) is True: + args = (__self.eval_ctx,) + args + elif getattr(__obj, "environmentfunction", False) is True: + args = (__self.environment,) + args + try: + return __obj(*args, **kwargs) + except StopIteration: + return __self.environment.undefined( + "value was undefined because " + "a callable raised a " + "StopIteration exception" + ) + + def derived(self, locals=None): + """Internal helper function to create a derived context. This is + used in situations where the system needs a new context in the same + template that is independent. + """ + context = new_context( + self.environment, self.name, {}, self.get_all(), True, None, locals + ) + context.eval_ctx = self.eval_ctx + context.blocks.update((k, list(v)) for k, v in iteritems(self.blocks)) + return context + + def _all(meth): # noqa: B902 + def proxy(self): + return getattr(self.get_all(), meth)() + + proxy.__doc__ = getattr(dict, meth).__doc__ + proxy.__name__ = meth + return proxy + + keys = _all("keys") + values = _all("values") + items = _all("items") + + # not available on python 3 + if PY2: + iterkeys = _all("iterkeys") + itervalues = _all("itervalues") + iteritems = _all("iteritems") + del _all + + def __contains__(self, name): + return name in self.vars or name in self.parent + + def __getitem__(self, key): + """Lookup a variable or raise `KeyError` if the variable is + undefined. + """ + item = self.resolve_or_missing(key) + if item is missing: + raise KeyError(key) + return item + + def __repr__(self): + return "<%s %s of %r>" % ( + self.__class__.__name__, + repr(self.get_all()), + self.name, + ) + + +abc.Mapping.register(Context) + + +class BlockReference(object): + """One block on a template reference.""" + + def __init__(self, name, context, stack, depth): + self.name = name + self._context = context + self._stack = stack + self._depth = depth + + @property + def super(self): + """Super the block.""" + if self._depth + 1 >= len(self._stack): + return self._context.environment.undefined( + "there is no parent block called %r." % self.name, name="super" + ) + return BlockReference(self.name, self._context, self._stack, self._depth + 1) + + @internalcode + def __call__(self): + rv = concat(self._stack[self._depth](self._context)) + if self._context.eval_ctx.autoescape: + rv = Markup(rv) + return rv + + +@implements_iterator +class LoopContext: + """A wrapper iterable for dynamic ``for`` loops, with information + about the loop and iteration. + """ + + #: Current iteration of the loop, starting at 0. + index0 = -1 + + _length = None + _after = missing + _current = missing + _before = missing + _last_changed_value = missing + + def __init__(self, iterable, undefined, recurse=None, depth0=0): + """ + :param iterable: Iterable to wrap. + :param undefined: :class:`Undefined` class to use for next and + previous items. + :param recurse: The function to render the loop body when the + loop is marked recursive. + :param depth0: Incremented when looping recursively. + """ + self._iterable = iterable + self._iterator = self._to_iterator(iterable) + self._undefined = undefined + self._recurse = recurse + #: How many levels deep a recursive loop currently is, starting at 0. + self.depth0 = depth0 + + @staticmethod + def _to_iterator(iterable): + return iter(iterable) + + @property + def length(self): + """Length of the iterable. + + If the iterable is a generator or otherwise does not have a + size, it is eagerly evaluated to get a size. + """ + if self._length is not None: + return self._length + + try: + self._length = len(self._iterable) + except TypeError: + iterable = list(self._iterator) + self._iterator = self._to_iterator(iterable) + self._length = len(iterable) + self.index + (self._after is not missing) + + return self._length + + def __len__(self): + return self.length + + @property + def depth(self): + """How many levels deep a recursive loop currently is, starting at 1.""" + return self.depth0 + 1 + + @property + def index(self): + """Current iteration of the loop, starting at 1.""" + return self.index0 + 1 + + @property + def revindex0(self): + """Number of iterations from the end of the loop, ending at 0. + + Requires calculating :attr:`length`. + """ + return self.length - self.index + + @property + def revindex(self): + """Number of iterations from the end of the loop, ending at 1. + + Requires calculating :attr:`length`. + """ + return self.length - self.index0 + + @property + def first(self): + """Whether this is the first iteration of the loop.""" + return self.index0 == 0 + + def _peek_next(self): + """Return the next element in the iterable, or :data:`missing` + if the iterable is exhausted. Only peeks one item ahead, caching + the result in :attr:`_last` for use in subsequent checks. The + cache is reset when :meth:`__next__` is called. + """ + if self._after is not missing: + return self._after + + self._after = next(self._iterator, missing) + return self._after + + @property + def last(self): + """Whether this is the last iteration of the loop. + + Causes the iterable to advance early. See + :func:`itertools.groupby` for issues this can cause. + The :func:`groupby` filter avoids that issue. + """ + return self._peek_next() is missing + + @property + def previtem(self): + """The item in the previous iteration. Undefined during the + first iteration. + """ + if self.first: + return self._undefined("there is no previous item") + + return self._before + + @property + def nextitem(self): + """The item in the next iteration. Undefined during the last + iteration. + + Causes the iterable to advance early. See + :func:`itertools.groupby` for issues this can cause. + The :func:`groupby` filter avoids that issue. + """ + rv = self._peek_next() + + if rv is missing: + return self._undefined("there is no next item") + + return rv + + def cycle(self, *args): + """Return a value from the given args, cycling through based on + the current :attr:`index0`. + + :param args: One or more values to cycle through. + """ + if not args: + raise TypeError("no items for cycling given") + + return args[self.index0 % len(args)] + + def changed(self, *value): + """Return ``True`` if previously called with a different value + (including when called for the first time). + + :param value: One or more values to compare to the last call. + """ + if self._last_changed_value != value: + self._last_changed_value = value + return True + + return False + + def __iter__(self): + return self + + def __next__(self): + if self._after is not missing: + rv = self._after + self._after = missing + else: + rv = next(self._iterator) + + self.index0 += 1 + self._before = self._current + self._current = rv + return rv, self + + @internalcode + def __call__(self, iterable): + """When iterating over nested data, render the body of the loop + recursively with the given inner iterable data. + + The loop must have the ``recursive`` marker for this to work. + """ + if self._recurse is None: + raise TypeError( + "The loop must have the 'recursive' marker to be called recursively." + ) + + return self._recurse(iterable, self._recurse, depth=self.depth) + + def __repr__(self): + return "<%s %d/%d>" % (self.__class__.__name__, self.index, self.length) + + +class Macro(object): + """Wraps a macro function.""" + + def __init__( + self, + environment, + func, + name, + arguments, + catch_kwargs, + catch_varargs, + caller, + default_autoescape=None, + ): + self._environment = environment + self._func = func + self._argument_count = len(arguments) + self.name = name + self.arguments = arguments + self.catch_kwargs = catch_kwargs + self.catch_varargs = catch_varargs + self.caller = caller + self.explicit_caller = "caller" in arguments + if default_autoescape is None: + default_autoescape = environment.autoescape + self._default_autoescape = default_autoescape + + @internalcode + @evalcontextfunction + def __call__(self, *args, **kwargs): + # This requires a bit of explanation, In the past we used to + # decide largely based on compile-time information if a macro is + # safe or unsafe. While there was a volatile mode it was largely + # unused for deciding on escaping. This turns out to be + # problematic for macros because whether a macro is safe depends not + # on the escape mode when it was defined, but rather when it was used. + # + # Because however we export macros from the module system and + # there are historic callers that do not pass an eval context (and + # will continue to not pass one), we need to perform an instance + # check here. + # + # This is considered safe because an eval context is not a valid + # argument to callables otherwise anyway. Worst case here is + # that if no eval context is passed we fall back to the compile + # time autoescape flag. + if args and isinstance(args[0], EvalContext): + autoescape = args[0].autoescape + args = args[1:] + else: + autoescape = self._default_autoescape + + # try to consume the positional arguments + arguments = list(args[: self._argument_count]) + off = len(arguments) + + # For information why this is necessary refer to the handling + # of caller in the `macro_body` handler in the compiler. + found_caller = False + + # if the number of arguments consumed is not the number of + # arguments expected we start filling in keyword arguments + # and defaults. + if off != self._argument_count: + for name in self.arguments[len(arguments) :]: + try: + value = kwargs.pop(name) + except KeyError: + value = missing + if name == "caller": + found_caller = True + arguments.append(value) + else: + found_caller = self.explicit_caller + + # it's important that the order of these arguments does not change + # if not also changed in the compiler's `function_scoping` method. + # the order is caller, keyword arguments, positional arguments! + if self.caller and not found_caller: + caller = kwargs.pop("caller", None) + if caller is None: + caller = self._environment.undefined("No caller defined", name="caller") + arguments.append(caller) + + if self.catch_kwargs: + arguments.append(kwargs) + elif kwargs: + if "caller" in kwargs: + raise TypeError( + "macro %r was invoked with two values for " + "the special caller argument. This is " + "most likely a bug." % self.name + ) + raise TypeError( + "macro %r takes no keyword argument %r" + % (self.name, next(iter(kwargs))) + ) + if self.catch_varargs: + arguments.append(args[self._argument_count :]) + elif len(args) > self._argument_count: + raise TypeError( + "macro %r takes not more than %d argument(s)" + % (self.name, len(self.arguments)) + ) + + return self._invoke(arguments, autoescape) + + def _invoke(self, arguments, autoescape): + """This method is being swapped out by the async implementation.""" + rv = self._func(*arguments) + if autoescape: + rv = Markup(rv) + return rv + + def __repr__(self): + return "<%s %s>" % ( + self.__class__.__name__, + self.name is None and "anonymous" or repr(self.name), + ) + + +@implements_to_string +class Undefined(object): + """The default undefined type. This undefined type can be printed and + iterated over, but every other access will raise an :exc:`UndefinedError`: + + >>> foo = Undefined(name='foo') + >>> str(foo) + '' + >>> not foo + True + >>> foo + 42 + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + """ + + __slots__ = ( + "_undefined_hint", + "_undefined_obj", + "_undefined_name", + "_undefined_exception", + ) + + def __init__(self, hint=None, obj=missing, name=None, exc=UndefinedError): + self._undefined_hint = hint + self._undefined_obj = obj + self._undefined_name = name + self._undefined_exception = exc + + @property + def _undefined_message(self): + """Build a message about the undefined value based on how it was + accessed. + """ + if self._undefined_hint: + return self._undefined_hint + + if self._undefined_obj is missing: + return "%r is undefined" % self._undefined_name + + if not isinstance(self._undefined_name, string_types): + return "%s has no element %r" % ( + object_type_repr(self._undefined_obj), + self._undefined_name, + ) + + return "%r has no attribute %r" % ( + object_type_repr(self._undefined_obj), + self._undefined_name, + ) + + @internalcode + def _fail_with_undefined_error(self, *args, **kwargs): + """Raise an :exc:`UndefinedError` when operations are performed + on the undefined value. + """ + raise self._undefined_exception(self._undefined_message) + + @internalcode + def __getattr__(self, name): + if name[:2] == "__": + raise AttributeError(name) + return self._fail_with_undefined_error() + + __add__ = ( + __radd__ + ) = ( + __mul__ + ) = ( + __rmul__ + ) = ( + __div__ + ) = ( + __rdiv__ + ) = ( + __truediv__ + ) = ( + __rtruediv__ + ) = ( + __floordiv__ + ) = ( + __rfloordiv__ + ) = ( + __mod__ + ) = ( + __rmod__ + ) = ( + __pos__ + ) = ( + __neg__ + ) = ( + __call__ + ) = ( + __getitem__ + ) = ( + __lt__ + ) = ( + __le__ + ) = ( + __gt__ + ) = ( + __ge__ + ) = ( + __int__ + ) = ( + __float__ + ) = ( + __complex__ + ) = __pow__ = __rpow__ = __sub__ = __rsub__ = _fail_with_undefined_error + + def __eq__(self, other): + return type(self) is type(other) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return id(type(self)) + + def __str__(self): + return u"" + + def __len__(self): + return 0 + + def __iter__(self): + if 0: + yield None + + def __nonzero__(self): + return False + + __bool__ = __nonzero__ + + def __repr__(self): + return "Undefined" + + +def make_logging_undefined(logger=None, base=None): + """Given a logger object this returns a new undefined class that will + log certain failures. It will log iterations and printing. If no + logger is given a default logger is created. + + Example:: + + logger = logging.getLogger(__name__) + LoggingUndefined = make_logging_undefined( + logger=logger, + base=Undefined + ) + + .. versionadded:: 2.8 + + :param logger: the logger to use. If not provided, a default logger + is created. + :param base: the base class to add logging functionality to. This + defaults to :class:`Undefined`. + """ + if logger is None: + import logging + + logger = logging.getLogger(__name__) + logger.addHandler(logging.StreamHandler(sys.stderr)) + if base is None: + base = Undefined + + def _log_message(undef): + if undef._undefined_hint is None: + if undef._undefined_obj is missing: + hint = "%s is undefined" % undef._undefined_name + elif not isinstance(undef._undefined_name, string_types): + hint = "%s has no element %s" % ( + object_type_repr(undef._undefined_obj), + undef._undefined_name, + ) + else: + hint = "%s has no attribute %s" % ( + object_type_repr(undef._undefined_obj), + undef._undefined_name, + ) + else: + hint = undef._undefined_hint + logger.warning("Template variable warning: %s", hint) + + class LoggingUndefined(base): + def _fail_with_undefined_error(self, *args, **kwargs): + try: + return base._fail_with_undefined_error(self, *args, **kwargs) + except self._undefined_exception as e: + logger.error("Template variable error: %s", str(e)) + raise e + + def __str__(self): + rv = base.__str__(self) + _log_message(self) + return rv + + def __iter__(self): + rv = base.__iter__(self) + _log_message(self) + return rv + + if PY2: + + def __nonzero__(self): + rv = base.__nonzero__(self) + _log_message(self) + return rv + + def __unicode__(self): + rv = base.__unicode__(self) + _log_message(self) + return rv + + else: + + def __bool__(self): + rv = base.__bool__(self) + _log_message(self) + return rv + + return LoggingUndefined + + +# No @implements_to_string decorator here because __str__ +# is not overwritten from Undefined in this class. +# This would cause a recursion error in Python 2. +class ChainableUndefined(Undefined): + """An undefined that is chainable, where both ``__getattr__`` and + ``__getitem__`` return itself rather than raising an + :exc:`UndefinedError`. + + >>> foo = ChainableUndefined(name='foo') + >>> str(foo.bar['baz']) + '' + >>> foo.bar['baz'] + 42 + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + + .. versionadded:: 2.11.0 + """ + + __slots__ = () + + def __html__(self): + return self.__str__() + + def __getattr__(self, _): + return self + + __getitem__ = __getattr__ + + +@implements_to_string +class DebugUndefined(Undefined): + """An undefined that returns the debug info when printed. + + >>> foo = DebugUndefined(name='foo') + >>> str(foo) + '{{ foo }}' + >>> not foo + True + >>> foo + 42 + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + """ + + __slots__ = () + + def __str__(self): + if self._undefined_hint is None: + if self._undefined_obj is missing: + return u"{{ %s }}" % self._undefined_name + return "{{ no such element: %s[%r] }}" % ( + object_type_repr(self._undefined_obj), + self._undefined_name, + ) + return u"{{ undefined value printed: %s }}" % self._undefined_hint + + +@implements_to_string +class StrictUndefined(Undefined): + """An undefined that barks on print and iteration as well as boolean + tests and all kinds of comparisons. In other words: you can do nothing + with it except checking if it's defined using the `defined` test. + + >>> foo = StrictUndefined(name='foo') + >>> str(foo) + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + >>> not foo + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + >>> foo + 42 + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + """ + + __slots__ = () + __iter__ = ( + __str__ + ) = ( + __len__ + ) = ( + __nonzero__ + ) = __eq__ = __ne__ = __bool__ = __hash__ = Undefined._fail_with_undefined_error + + +# remove remaining slots attributes, after the metaclass did the magic they +# are unneeded and irritating as they contain wrong data for the subclasses. +del ( + Undefined.__slots__, + ChainableUndefined.__slots__, + DebugUndefined.__slots__, + StrictUndefined.__slots__, +) diff --git a/contrib/python/Jinja2/py2/jinja2/sandbox.py b/contrib/python/Jinja2/py2/jinja2/sandbox.py new file mode 100644 index 0000000000..cfd7993aee --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/sandbox.py @@ -0,0 +1,510 @@ +# -*- coding: utf-8 -*- +"""A sandbox layer that ensures unsafe operations cannot be performed. +Useful when the template itself comes from an untrusted source. +""" +import operator +import types +import warnings +from collections import deque +from string import Formatter + +from markupsafe import EscapeFormatter +from markupsafe import Markup + +from ._compat import abc +from ._compat import PY2 +from ._compat import range_type +from ._compat import string_types +from .environment import Environment +from .exceptions import SecurityError + +#: maximum number of items a range may produce +MAX_RANGE = 100000 + +#: attributes of function objects that are considered unsafe. +if PY2: + UNSAFE_FUNCTION_ATTRIBUTES = { + "func_closure", + "func_code", + "func_dict", + "func_defaults", + "func_globals", + } +else: + # On versions > python 2 the special attributes on functions are gone, + # but they remain on methods and generators for whatever reason. + UNSAFE_FUNCTION_ATTRIBUTES = set() + +#: unsafe method attributes. function attributes are unsafe for methods too +UNSAFE_METHOD_ATTRIBUTES = {"im_class", "im_func", "im_self"} + +#: unsafe generator attributes. +UNSAFE_GENERATOR_ATTRIBUTES = {"gi_frame", "gi_code"} + +#: unsafe attributes on coroutines +UNSAFE_COROUTINE_ATTRIBUTES = {"cr_frame", "cr_code"} + +#: unsafe attributes on async generators +UNSAFE_ASYNC_GENERATOR_ATTRIBUTES = {"ag_code", "ag_frame"} + +# make sure we don't warn in python 2.6 about stuff we don't care about +warnings.filterwarnings( + "ignore", "the sets module", DeprecationWarning, module=__name__ +) + +_mutable_set_types = (set,) +_mutable_mapping_types = (dict,) +_mutable_sequence_types = (list,) + +# on python 2.x we can register the user collection types +try: + from UserDict import UserDict, DictMixin + from UserList import UserList + + _mutable_mapping_types += (UserDict, DictMixin) + _mutable_set_types += (UserList,) +except ImportError: + pass + +# if sets is still available, register the mutable set from there as well +try: + from sets import Set + + _mutable_set_types += (Set,) +except ImportError: + pass + +#: register Python 2.6 abstract base classes +_mutable_set_types += (abc.MutableSet,) +_mutable_mapping_types += (abc.MutableMapping,) +_mutable_sequence_types += (abc.MutableSequence,) + +_mutable_spec = ( + ( + _mutable_set_types, + frozenset( + [ + "add", + "clear", + "difference_update", + "discard", + "pop", + "remove", + "symmetric_difference_update", + "update", + ] + ), + ), + ( + _mutable_mapping_types, + frozenset(["clear", "pop", "popitem", "setdefault", "update"]), + ), + ( + _mutable_sequence_types, + frozenset(["append", "reverse", "insert", "sort", "extend", "remove"]), + ), + ( + deque, + frozenset( + [ + "append", + "appendleft", + "clear", + "extend", + "extendleft", + "pop", + "popleft", + "remove", + "rotate", + ] + ), + ), +) + + +class _MagicFormatMapping(abc.Mapping): + """This class implements a dummy wrapper to fix a bug in the Python + standard library for string formatting. + + See https://bugs.python.org/issue13598 for information about why + this is necessary. + """ + + def __init__(self, args, kwargs): + self._args = args + self._kwargs = kwargs + self._last_index = 0 + + def __getitem__(self, key): + if key == "": + idx = self._last_index + self._last_index += 1 + try: + return self._args[idx] + except LookupError: + pass + key = str(idx) + return self._kwargs[key] + + def __iter__(self): + return iter(self._kwargs) + + def __len__(self): + return len(self._kwargs) + + +def inspect_format_method(callable): + if not isinstance( + callable, (types.MethodType, types.BuiltinMethodType) + ) or callable.__name__ not in ("format", "format_map"): + return None + obj = callable.__self__ + if isinstance(obj, string_types): + return obj + + +def safe_range(*args): + """A range that can't generate ranges with a length of more than + MAX_RANGE items. + """ + rng = range_type(*args) + + if len(rng) > MAX_RANGE: + raise OverflowError( + "Range too big. The sandbox blocks ranges larger than" + " MAX_RANGE (%d)." % MAX_RANGE + ) + + return rng + + +def unsafe(f): + """Marks a function or method as unsafe. + + :: + + @unsafe + def delete(self): + pass + """ + f.unsafe_callable = True + return f + + +def is_internal_attribute(obj, attr): + """Test if the attribute given is an internal python attribute. For + example this function returns `True` for the `func_code` attribute of + python objects. This is useful if the environment method + :meth:`~SandboxedEnvironment.is_safe_attribute` is overridden. + + >>> from jinja2.sandbox import is_internal_attribute + >>> is_internal_attribute(str, "mro") + True + >>> is_internal_attribute(str, "upper") + False + """ + if isinstance(obj, types.FunctionType): + if attr in UNSAFE_FUNCTION_ATTRIBUTES: + return True + elif isinstance(obj, types.MethodType): + if attr in UNSAFE_FUNCTION_ATTRIBUTES or attr in UNSAFE_METHOD_ATTRIBUTES: + return True + elif isinstance(obj, type): + if attr == "mro": + return True + elif isinstance(obj, (types.CodeType, types.TracebackType, types.FrameType)): + return True + elif isinstance(obj, types.GeneratorType): + if attr in UNSAFE_GENERATOR_ATTRIBUTES: + return True + elif hasattr(types, "CoroutineType") and isinstance(obj, types.CoroutineType): + if attr in UNSAFE_COROUTINE_ATTRIBUTES: + return True + elif hasattr(types, "AsyncGeneratorType") and isinstance( + obj, types.AsyncGeneratorType + ): + if attr in UNSAFE_ASYNC_GENERATOR_ATTRIBUTES: + return True + return attr.startswith("__") + + +def modifies_known_mutable(obj, attr): + """This function checks if an attribute on a builtin mutable object + (list, dict, set or deque) would modify it if called. It also supports + the "user"-versions of the objects (`sets.Set`, `UserDict.*` etc.) and + with Python 2.6 onwards the abstract base classes `MutableSet`, + `MutableMapping`, and `MutableSequence`. + + >>> modifies_known_mutable({}, "clear") + True + >>> modifies_known_mutable({}, "keys") + False + >>> modifies_known_mutable([], "append") + True + >>> modifies_known_mutable([], "index") + False + + If called with an unsupported object (such as unicode) `False` is + returned. + + >>> modifies_known_mutable("foo", "upper") + False + """ + for typespec, unsafe in _mutable_spec: + if isinstance(obj, typespec): + return attr in unsafe + return False + + +class SandboxedEnvironment(Environment): + """The sandboxed environment. It works like the regular environment but + tells the compiler to generate sandboxed code. Additionally subclasses of + this environment may override the methods that tell the runtime what + attributes or functions are safe to access. + + If the template tries to access insecure code a :exc:`SecurityError` is + raised. However also other exceptions may occur during the rendering so + the caller has to ensure that all exceptions are caught. + """ + + sandboxed = True + + #: default callback table for the binary operators. A copy of this is + #: available on each instance of a sandboxed environment as + #: :attr:`binop_table` + default_binop_table = { + "+": operator.add, + "-": operator.sub, + "*": operator.mul, + "/": operator.truediv, + "//": operator.floordiv, + "**": operator.pow, + "%": operator.mod, + } + + #: default callback table for the unary operators. A copy of this is + #: available on each instance of a sandboxed environment as + #: :attr:`unop_table` + default_unop_table = {"+": operator.pos, "-": operator.neg} + + #: a set of binary operators that should be intercepted. Each operator + #: that is added to this set (empty by default) is delegated to the + #: :meth:`call_binop` method that will perform the operator. The default + #: operator callback is specified by :attr:`binop_table`. + #: + #: The following binary operators are interceptable: + #: ``//``, ``%``, ``+``, ``*``, ``-``, ``/``, and ``**`` + #: + #: The default operation form the operator table corresponds to the + #: builtin function. Intercepted calls are always slower than the native + #: operator call, so make sure only to intercept the ones you are + #: interested in. + #: + #: .. versionadded:: 2.6 + intercepted_binops = frozenset() + + #: a set of unary operators that should be intercepted. Each operator + #: that is added to this set (empty by default) is delegated to the + #: :meth:`call_unop` method that will perform the operator. The default + #: operator callback is specified by :attr:`unop_table`. + #: + #: The following unary operators are interceptable: ``+``, ``-`` + #: + #: The default operation form the operator table corresponds to the + #: builtin function. Intercepted calls are always slower than the native + #: operator call, so make sure only to intercept the ones you are + #: interested in. + #: + #: .. versionadded:: 2.6 + intercepted_unops = frozenset() + + def intercept_unop(self, operator): + """Called during template compilation with the name of a unary + operator to check if it should be intercepted at runtime. If this + method returns `True`, :meth:`call_unop` is executed for this unary + operator. The default implementation of :meth:`call_unop` will use + the :attr:`unop_table` dictionary to perform the operator with the + same logic as the builtin one. + + The following unary operators are interceptable: ``+`` and ``-`` + + Intercepted calls are always slower than the native operator call, + so make sure only to intercept the ones you are interested in. + + .. versionadded:: 2.6 + """ + return False + + def __init__(self, *args, **kwargs): + Environment.__init__(self, *args, **kwargs) + self.globals["range"] = safe_range + self.binop_table = self.default_binop_table.copy() + self.unop_table = self.default_unop_table.copy() + + def is_safe_attribute(self, obj, attr, value): + """The sandboxed environment will call this method to check if the + attribute of an object is safe to access. Per default all attributes + starting with an underscore are considered private as well as the + special attributes of internal python objects as returned by the + :func:`is_internal_attribute` function. + """ + return not (attr.startswith("_") or is_internal_attribute(obj, attr)) + + def is_safe_callable(self, obj): + """Check if an object is safely callable. Per default a function is + considered safe unless the `unsafe_callable` attribute exists and is + True. Override this method to alter the behavior, but this won't + affect the `unsafe` decorator from this module. + """ + return not ( + getattr(obj, "unsafe_callable", False) or getattr(obj, "alters_data", False) + ) + + def call_binop(self, context, operator, left, right): + """For intercepted binary operator calls (:meth:`intercepted_binops`) + this function is executed instead of the builtin operator. This can + be used to fine tune the behavior of certain operators. + + .. versionadded:: 2.6 + """ + return self.binop_table[operator](left, right) + + def call_unop(self, context, operator, arg): + """For intercepted unary operator calls (:meth:`intercepted_unops`) + this function is executed instead of the builtin operator. This can + be used to fine tune the behavior of certain operators. + + .. versionadded:: 2.6 + """ + return self.unop_table[operator](arg) + + def getitem(self, obj, argument): + """Subscribe an object from sandboxed code.""" + try: + return obj[argument] + except (TypeError, LookupError): + if isinstance(argument, string_types): + try: + attr = str(argument) + except Exception: + pass + else: + try: + value = getattr(obj, attr) + except AttributeError: + pass + else: + if self.is_safe_attribute(obj, argument, value): + return value + return self.unsafe_undefined(obj, argument) + return self.undefined(obj=obj, name=argument) + + def getattr(self, obj, attribute): + """Subscribe an object from sandboxed code and prefer the + attribute. The attribute passed *must* be a bytestring. + """ + try: + value = getattr(obj, attribute) + except AttributeError: + try: + return obj[attribute] + except (TypeError, LookupError): + pass + else: + if self.is_safe_attribute(obj, attribute, value): + return value + return self.unsafe_undefined(obj, attribute) + return self.undefined(obj=obj, name=attribute) + + def unsafe_undefined(self, obj, attribute): + """Return an undefined object for unsafe attributes.""" + return self.undefined( + "access to attribute %r of %r " + "object is unsafe." % (attribute, obj.__class__.__name__), + name=attribute, + obj=obj, + exc=SecurityError, + ) + + def format_string(self, s, args, kwargs, format_func=None): + """If a format call is detected, then this is routed through this + method so that our safety sandbox can be used for it. + """ + if isinstance(s, Markup): + formatter = SandboxedEscapeFormatter(self, s.escape) + else: + formatter = SandboxedFormatter(self) + + if format_func is not None and format_func.__name__ == "format_map": + if len(args) != 1 or kwargs: + raise TypeError( + "format_map() takes exactly one argument %d given" + % (len(args) + (kwargs is not None)) + ) + + kwargs = args[0] + args = None + + kwargs = _MagicFormatMapping(args, kwargs) + rv = formatter.vformat(s, args, kwargs) + return type(s)(rv) + + def call(__self, __context, __obj, *args, **kwargs): # noqa: B902 + """Call an object from sandboxed code.""" + fmt = inspect_format_method(__obj) + if fmt is not None: + return __self.format_string(fmt, args, kwargs, __obj) + + # the double prefixes are to avoid double keyword argument + # errors when proxying the call. + if not __self.is_safe_callable(__obj): + raise SecurityError("%r is not safely callable" % (__obj,)) + return __context.call(__obj, *args, **kwargs) + + +class ImmutableSandboxedEnvironment(SandboxedEnvironment): + """Works exactly like the regular `SandboxedEnvironment` but does not + permit modifications on the builtin mutable objects `list`, `set`, and + `dict` by using the :func:`modifies_known_mutable` function. + """ + + def is_safe_attribute(self, obj, attr, value): + if not SandboxedEnvironment.is_safe_attribute(self, obj, attr, value): + return False + return not modifies_known_mutable(obj, attr) + + +# This really is not a public API apparently. +try: + from _string import formatter_field_name_split +except ImportError: + + def formatter_field_name_split(field_name): + return field_name._formatter_field_name_split() + + +class SandboxedFormatterMixin(object): + def __init__(self, env): + self._env = env + + def get_field(self, field_name, args, kwargs): + first, rest = formatter_field_name_split(field_name) + obj = self.get_value(first, args, kwargs) + for is_attr, i in rest: + if is_attr: + obj = self._env.getattr(obj, i) + else: + obj = self._env.getitem(obj, i) + return obj, first + + +class SandboxedFormatter(SandboxedFormatterMixin, Formatter): + def __init__(self, env): + SandboxedFormatterMixin.__init__(self, env) + Formatter.__init__(self) + + +class SandboxedEscapeFormatter(SandboxedFormatterMixin, EscapeFormatter): + def __init__(self, env, escape): + SandboxedFormatterMixin.__init__(self, env) + EscapeFormatter.__init__(self, escape) diff --git a/contrib/python/Jinja2/py2/jinja2/tests.py b/contrib/python/Jinja2/py2/jinja2/tests.py new file mode 100644 index 0000000000..fabd4ce51b --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/tests.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- +"""Built-in template tests used with the ``is`` operator.""" +import decimal +import operator +import re + +from ._compat import abc +from ._compat import integer_types +from ._compat import string_types +from ._compat import text_type +from .runtime import Undefined + +number_re = re.compile(r"^-?\d+(\.\d+)?$") +regex_type = type(number_re) +test_callable = callable + + +def test_odd(value): + """Return true if the variable is odd.""" + return value % 2 == 1 + + +def test_even(value): + """Return true if the variable is even.""" + return value % 2 == 0 + + +def test_divisibleby(value, num): + """Check if a variable is divisible by a number.""" + return value % num == 0 + + +def test_defined(value): + """Return true if the variable is defined: + + .. sourcecode:: jinja + + {% if variable is defined %} + value of variable: {{ variable }} + {% else %} + variable is not defined + {% endif %} + + See the :func:`default` filter for a simple way to set undefined + variables. + """ + return not isinstance(value, Undefined) + + +def test_undefined(value): + """Like :func:`defined` but the other way round.""" + return isinstance(value, Undefined) + + +def test_none(value): + """Return true if the variable is none.""" + return value is None + + +def test_boolean(value): + """Return true if the object is a boolean value. + + .. versionadded:: 2.11 + """ + return value is True or value is False + + +def test_false(value): + """Return true if the object is False. + + .. versionadded:: 2.11 + """ + return value is False + + +def test_true(value): + """Return true if the object is True. + + .. versionadded:: 2.11 + """ + return value is True + + +# NOTE: The existing 'number' test matches booleans and floats +def test_integer(value): + """Return true if the object is an integer. + + .. versionadded:: 2.11 + """ + return isinstance(value, integer_types) and value is not True and value is not False + + +# NOTE: The existing 'number' test matches booleans and integers +def test_float(value): + """Return true if the object is a float. + + .. versionadded:: 2.11 + """ + return isinstance(value, float) + + +def test_lower(value): + """Return true if the variable is lowercased.""" + return text_type(value).islower() + + +def test_upper(value): + """Return true if the variable is uppercased.""" + return text_type(value).isupper() + + +def test_string(value): + """Return true if the object is a string.""" + return isinstance(value, string_types) + + +def test_mapping(value): + """Return true if the object is a mapping (dict etc.). + + .. versionadded:: 2.6 + """ + return isinstance(value, abc.Mapping) + + +def test_number(value): + """Return true if the variable is a number.""" + return isinstance(value, integer_types + (float, complex, decimal.Decimal)) + + +def test_sequence(value): + """Return true if the variable is a sequence. Sequences are variables + that are iterable. + """ + try: + len(value) + value.__getitem__ + except Exception: + return False + return True + + +def test_sameas(value, other): + """Check if an object points to the same memory address than another + object: + + .. sourcecode:: jinja + + {% if foo.attribute is sameas false %} + the foo attribute really is the `False` singleton + {% endif %} + """ + return value is other + + +def test_iterable(value): + """Check if it's possible to iterate over an object.""" + try: + iter(value) + except TypeError: + return False + return True + + +def test_escaped(value): + """Check if the value is escaped.""" + return hasattr(value, "__html__") + + +def test_in(value, seq): + """Check if value is in seq. + + .. versionadded:: 2.10 + """ + return value in seq + + +TESTS = { + "odd": test_odd, + "even": test_even, + "divisibleby": test_divisibleby, + "defined": test_defined, + "undefined": test_undefined, + "none": test_none, + "boolean": test_boolean, + "false": test_false, + "true": test_true, + "integer": test_integer, + "float": test_float, + "lower": test_lower, + "upper": test_upper, + "string": test_string, + "mapping": test_mapping, + "number": test_number, + "sequence": test_sequence, + "iterable": test_iterable, + "callable": test_callable, + "sameas": test_sameas, + "escaped": test_escaped, + "in": test_in, + "==": operator.eq, + "eq": operator.eq, + "equalto": operator.eq, + "!=": operator.ne, + "ne": operator.ne, + ">": operator.gt, + "gt": operator.gt, + "greaterthan": operator.gt, + "ge": operator.ge, + ">=": operator.ge, + "<": operator.lt, + "lt": operator.lt, + "lessthan": operator.lt, + "<=": operator.le, + "le": operator.le, +} diff --git a/contrib/python/Jinja2/py2/jinja2/utils.py b/contrib/python/Jinja2/py2/jinja2/utils.py new file mode 100644 index 0000000000..6afca81055 --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/utils.py @@ -0,0 +1,737 @@ +# -*- coding: utf-8 -*- +import json +import os +import re +import warnings +from collections import deque +from random import choice +from random import randrange +from string import ascii_letters as _letters +from string import digits as _digits +from threading import Lock + +from markupsafe import escape +from markupsafe import Markup + +from ._compat import abc +from ._compat import string_types +from ._compat import text_type +from ._compat import url_quote + +# special singleton representing missing values for the runtime +missing = type("MissingType", (), {"__repr__": lambda x: "missing"})() + +# internal code +internal_code = set() + +concat = u"".join + +_slash_escape = "\\/" not in json.dumps("/") + + +def contextfunction(f): + """This decorator can be used to mark a function or method context callable. + A context callable is passed the active :class:`Context` as first argument when + called from the template. This is useful if a function wants to get access + to the context or functions provided on the context object. For example + a function that returns a sorted list of template variables the current + template exports could look like this:: + + @contextfunction + def get_exported_names(context): + return sorted(context.exported_vars) + """ + f.contextfunction = True + return f + + +def evalcontextfunction(f): + """This decorator can be used to mark a function or method as an eval + context callable. This is similar to the :func:`contextfunction` + but instead of passing the context, an evaluation context object is + passed. For more information about the eval context, see + :ref:`eval-context`. + + .. versionadded:: 2.4 + """ + f.evalcontextfunction = True + return f + + +def environmentfunction(f): + """This decorator can be used to mark a function or method as environment + callable. This decorator works exactly like the :func:`contextfunction` + decorator just that the first argument is the active :class:`Environment` + and not context. + """ + f.environmentfunction = True + return f + + +def internalcode(f): + """Marks the function as internally used""" + internal_code.add(f.__code__) + return f + + +def is_undefined(obj): + """Check if the object passed is undefined. This does nothing more than + performing an instance check against :class:`Undefined` but looks nicer. + This can be used for custom filters or tests that want to react to + undefined variables. For example a custom default filter can look like + this:: + + def default(var, default=''): + if is_undefined(var): + return default + return var + """ + from .runtime import Undefined + + return isinstance(obj, Undefined) + + +def consume(iterable): + """Consumes an iterable without doing anything with it.""" + for _ in iterable: + pass + + +def clear_caches(): + """Jinja keeps internal caches for environments and lexers. These are + used so that Jinja doesn't have to recreate environments and lexers all + the time. Normally you don't have to care about that but if you are + measuring memory consumption you may want to clean the caches. + """ + from .environment import _spontaneous_environments + from .lexer import _lexer_cache + + _spontaneous_environments.clear() + _lexer_cache.clear() + + +def import_string(import_name, silent=False): + """Imports an object based on a string. This is useful if you want to + use import paths as endpoints or something similar. An import path can + be specified either in dotted notation (``xml.sax.saxutils.escape``) + or with a colon as object delimiter (``xml.sax.saxutils:escape``). + + If the `silent` is True the return value will be `None` if the import + fails. + + :return: imported object + """ + try: + if ":" in import_name: + module, obj = import_name.split(":", 1) + elif "." in import_name: + module, _, obj = import_name.rpartition(".") + else: + return __import__(import_name) + return getattr(__import__(module, None, None, [obj]), obj) + except (ImportError, AttributeError): + if not silent: + raise + + +def open_if_exists(filename, mode="rb"): + """Returns a file descriptor for the filename if that file exists, + otherwise ``None``. + """ + if not os.path.isfile(filename): + return None + + return open(filename, mode) + + +def object_type_repr(obj): + """Returns the name of the object's type. For some recognized + singletons the name of the object is returned instead. (For + example for `None` and `Ellipsis`). + """ + if obj is None: + return "None" + elif obj is Ellipsis: + return "Ellipsis" + + cls = type(obj) + + # __builtin__ in 2.x, builtins in 3.x + if cls.__module__ in ("__builtin__", "builtins"): + name = cls.__name__ + else: + name = cls.__module__ + "." + cls.__name__ + + return "%s object" % name + + +def pformat(obj, verbose=False): + """Prettyprint an object. Either use the `pretty` library or the + builtin `pprint`. + """ + try: + from pretty import pretty + + return pretty(obj, verbose=verbose) + except ImportError: + from pprint import pformat + + return pformat(obj) + + +def urlize(text, trim_url_limit=None, rel=None, target=None): + """Converts any URLs in text into clickable links. Works on http://, + https:// and www. links. Links can have trailing punctuation (periods, + commas, close-parens) and leading punctuation (opening parens) and + it'll still do the right thing. + + If trim_url_limit is not None, the URLs in link text will be limited + to trim_url_limit characters. + + If nofollow is True, the URLs in link text will get a rel="nofollow" + attribute. + + If target is not None, a target attribute will be added to the link. + """ + trim_url = ( + lambda x, limit=trim_url_limit: limit is not None + and (x[:limit] + (len(x) >= limit and "..." or "")) + or x + ) + words = re.split(r"(\s+)", text_type(escape(text))) + rel_attr = rel and ' rel="%s"' % text_type(escape(rel)) or "" + target_attr = target and ' target="%s"' % escape(target) or "" + + for i, word in enumerate(words): + head, middle, tail = "", word, "" + match = re.match(r"^([(<]|<)+", middle) + + if match: + head = match.group() + middle = middle[match.end() :] + + # Unlike lead, which is anchored to the start of the string, + # need to check that the string ends with any of the characters + # before trying to match all of them, to avoid backtracking. + if middle.endswith((")", ">", ".", ",", "\n", ">")): + match = re.search(r"([)>.,\n]|>)+$", middle) + + if match: + tail = match.group() + middle = middle[: match.start()] + + if middle.startswith("www.") or ( + "@" not in middle + and not middle.startswith("http://") + and not middle.startswith("https://") + and len(middle) > 0 + and middle[0] in _letters + _digits + and ( + middle.endswith(".org") + or middle.endswith(".net") + or middle.endswith(".com") + ) + ): + middle = '<a href="http://%s"%s%s>%s</a>' % ( + middle, + rel_attr, + target_attr, + trim_url(middle), + ) + + if middle.startswith("http://") or middle.startswith("https://"): + middle = '<a href="%s"%s%s>%s</a>' % ( + middle, + rel_attr, + target_attr, + trim_url(middle), + ) + + if ( + "@" in middle + and not middle.startswith("www.") + and ":" not in middle + and re.match(r"^\S+@\w[\w.-]*\.\w+$", middle) + ): + middle = '<a href="mailto:%s">%s</a>' % (middle, middle) + + words[i] = head + middle + tail + + return u"".join(words) + + +def generate_lorem_ipsum(n=5, html=True, min=20, max=100): + """Generate some lorem ipsum for the template.""" + from .constants import LOREM_IPSUM_WORDS + + words = LOREM_IPSUM_WORDS.split() + result = [] + + for _ in range(n): + next_capitalized = True + last_comma = last_fullstop = 0 + word = None + last = None + p = [] + + # each paragraph contains out of 20 to 100 words. + for idx, _ in enumerate(range(randrange(min, max))): + while True: + word = choice(words) + if word != last: + last = word + break + if next_capitalized: + word = word.capitalize() + next_capitalized = False + # add commas + if idx - randrange(3, 8) > last_comma: + last_comma = idx + last_fullstop += 2 + word += "," + # add end of sentences + if idx - randrange(10, 20) > last_fullstop: + last_comma = last_fullstop = idx + word += "." + next_capitalized = True + p.append(word) + + # ensure that the paragraph ends with a dot. + p = u" ".join(p) + if p.endswith(","): + p = p[:-1] + "." + elif not p.endswith("."): + p += "." + result.append(p) + + if not html: + return u"\n\n".join(result) + return Markup(u"\n".join(u"<p>%s</p>" % escape(x) for x in result)) + + +def unicode_urlencode(obj, charset="utf-8", for_qs=False): + """Quote a string for use in a URL using the given charset. + + This function is misnamed, it is a wrapper around + :func:`urllib.parse.quote`. + + :param obj: String or bytes to quote. Other types are converted to + string then encoded to bytes using the given charset. + :param charset: Encode text to bytes using this charset. + :param for_qs: Quote "/" and use "+" for spaces. + """ + if not isinstance(obj, string_types): + obj = text_type(obj) + + if isinstance(obj, text_type): + obj = obj.encode(charset) + + safe = b"" if for_qs else b"/" + rv = url_quote(obj, safe) + + if not isinstance(rv, text_type): + rv = rv.decode("utf-8") + + if for_qs: + rv = rv.replace("%20", "+") + + return rv + + +class LRUCache(object): + """A simple LRU Cache implementation.""" + + # this is fast for small capacities (something below 1000) but doesn't + # scale. But as long as it's only used as storage for templates this + # won't do any harm. + + def __init__(self, capacity): + self.capacity = capacity + self._mapping = {} + self._queue = deque() + self._postinit() + + def _postinit(self): + # alias all queue methods for faster lookup + self._popleft = self._queue.popleft + self._pop = self._queue.pop + self._remove = self._queue.remove + self._wlock = Lock() + self._append = self._queue.append + + def __getstate__(self): + return { + "capacity": self.capacity, + "_mapping": self._mapping, + "_queue": self._queue, + } + + def __setstate__(self, d): + self.__dict__.update(d) + self._postinit() + + def __getnewargs__(self): + return (self.capacity,) + + def copy(self): + """Return a shallow copy of the instance.""" + rv = self.__class__(self.capacity) + rv._mapping.update(self._mapping) + rv._queue.extend(self._queue) + return rv + + def get(self, key, default=None): + """Return an item from the cache dict or `default`""" + try: + return self[key] + except KeyError: + return default + + def setdefault(self, key, default=None): + """Set `default` if the key is not in the cache otherwise + leave unchanged. Return the value of this key. + """ + try: + return self[key] + except KeyError: + self[key] = default + return default + + def clear(self): + """Clear the cache.""" + self._wlock.acquire() + try: + self._mapping.clear() + self._queue.clear() + finally: + self._wlock.release() + + def __contains__(self, key): + """Check if a key exists in this cache.""" + return key in self._mapping + + def __len__(self): + """Return the current size of the cache.""" + return len(self._mapping) + + def __repr__(self): + return "<%s %r>" % (self.__class__.__name__, self._mapping) + + def __getitem__(self, key): + """Get an item from the cache. Moves the item up so that it has the + highest priority then. + + Raise a `KeyError` if it does not exist. + """ + self._wlock.acquire() + try: + rv = self._mapping[key] + if self._queue[-1] != key: + try: + self._remove(key) + except ValueError: + # if something removed the key from the container + # when we read, ignore the ValueError that we would + # get otherwise. + pass + self._append(key) + return rv + finally: + self._wlock.release() + + def __setitem__(self, key, value): + """Sets the value for an item. Moves the item up so that it + has the highest priority then. + """ + self._wlock.acquire() + try: + if key in self._mapping: + self._remove(key) + elif len(self._mapping) == self.capacity: + del self._mapping[self._popleft()] + self._append(key) + self._mapping[key] = value + finally: + self._wlock.release() + + def __delitem__(self, key): + """Remove an item from the cache dict. + Raise a `KeyError` if it does not exist. + """ + self._wlock.acquire() + try: + del self._mapping[key] + try: + self._remove(key) + except ValueError: + pass + finally: + self._wlock.release() + + def items(self): + """Return a list of items.""" + result = [(key, self._mapping[key]) for key in list(self._queue)] + result.reverse() + return result + + def iteritems(self): + """Iterate over all items.""" + warnings.warn( + "'iteritems()' will be removed in version 3.0. Use" + " 'iter(cache.items())' instead.", + DeprecationWarning, + stacklevel=2, + ) + return iter(self.items()) + + def values(self): + """Return a list of all values.""" + return [x[1] for x in self.items()] + + def itervalue(self): + """Iterate over all values.""" + warnings.warn( + "'itervalue()' will be removed in version 3.0. Use" + " 'iter(cache.values())' instead.", + DeprecationWarning, + stacklevel=2, + ) + return iter(self.values()) + + def itervalues(self): + """Iterate over all values.""" + warnings.warn( + "'itervalues()' will be removed in version 3.0. Use" + " 'iter(cache.values())' instead.", + DeprecationWarning, + stacklevel=2, + ) + return iter(self.values()) + + def keys(self): + """Return a list of all keys ordered by most recent usage.""" + return list(self) + + def iterkeys(self): + """Iterate over all keys in the cache dict, ordered by + the most recent usage. + """ + warnings.warn( + "'iterkeys()' will be removed in version 3.0. Use" + " 'iter(cache.keys())' instead.", + DeprecationWarning, + stacklevel=2, + ) + return iter(self) + + def __iter__(self): + return reversed(tuple(self._queue)) + + def __reversed__(self): + """Iterate over the keys in the cache dict, oldest items + coming first. + """ + return iter(tuple(self._queue)) + + __copy__ = copy + + +abc.MutableMapping.register(LRUCache) + + +def select_autoescape( + enabled_extensions=("html", "htm", "xml"), + disabled_extensions=(), + default_for_string=True, + default=False, +): + """Intelligently sets the initial value of autoescaping based on the + filename of the template. This is the recommended way to configure + autoescaping if you do not want to write a custom function yourself. + + If you want to enable it for all templates created from strings or + for all templates with `.html` and `.xml` extensions:: + + from jinja2 import Environment, select_autoescape + env = Environment(autoescape=select_autoescape( + enabled_extensions=('html', 'xml'), + default_for_string=True, + )) + + Example configuration to turn it on at all times except if the template + ends with `.txt`:: + + from jinja2 import Environment, select_autoescape + env = Environment(autoescape=select_autoescape( + disabled_extensions=('txt',), + default_for_string=True, + default=True, + )) + + The `enabled_extensions` is an iterable of all the extensions that + autoescaping should be enabled for. Likewise `disabled_extensions` is + a list of all templates it should be disabled for. If a template is + loaded from a string then the default from `default_for_string` is used. + If nothing matches then the initial value of autoescaping is set to the + value of `default`. + + For security reasons this function operates case insensitive. + + .. versionadded:: 2.9 + """ + enabled_patterns = tuple("." + x.lstrip(".").lower() for x in enabled_extensions) + disabled_patterns = tuple("." + x.lstrip(".").lower() for x in disabled_extensions) + + def autoescape(template_name): + if template_name is None: + return default_for_string + template_name = template_name.lower() + if template_name.endswith(enabled_patterns): + return True + if template_name.endswith(disabled_patterns): + return False + return default + + return autoescape + + +def htmlsafe_json_dumps(obj, dumper=None, **kwargs): + """Works exactly like :func:`dumps` but is safe for use in ``<script>`` + tags. It accepts the same arguments and returns a JSON string. Note that + this is available in templates through the ``|tojson`` filter which will + also mark the result as safe. Due to how this function escapes certain + characters this is safe even if used outside of ``<script>`` tags. + + The following characters are escaped in strings: + + - ``<`` + - ``>`` + - ``&`` + - ``'`` + + This makes it safe to embed such strings in any place in HTML with the + notable exception of double quoted attributes. In that case single + quote your attributes or HTML escape it in addition. + """ + if dumper is None: + dumper = json.dumps + rv = ( + dumper(obj, **kwargs) + .replace(u"<", u"\\u003c") + .replace(u">", u"\\u003e") + .replace(u"&", u"\\u0026") + .replace(u"'", u"\\u0027") + ) + return Markup(rv) + + +class Cycler(object): + """Cycle through values by yield them one at a time, then restarting + once the end is reached. Available as ``cycler`` in templates. + + Similar to ``loop.cycle``, but can be used outside loops or across + multiple loops. For example, render a list of folders and files in a + list, alternating giving them "odd" and "even" classes. + + .. code-block:: html+jinja + + {% set row_class = cycler("odd", "even") %} + <ul class="browser"> + {% for folder in folders %} + <li class="folder {{ row_class.next() }}">{{ folder }} + {% endfor %} + {% for file in files %} + <li class="file {{ row_class.next() }}">{{ file }} + {% endfor %} + </ul> + + :param items: Each positional argument will be yielded in the order + given for each cycle. + + .. versionadded:: 2.1 + """ + + def __init__(self, *items): + if not items: + raise RuntimeError("at least one item has to be provided") + self.items = items + self.pos = 0 + + def reset(self): + """Resets the current item to the first item.""" + self.pos = 0 + + @property + def current(self): + """Return the current item. Equivalent to the item that will be + returned next time :meth:`next` is called. + """ + return self.items[self.pos] + + def next(self): + """Return the current item, then advance :attr:`current` to the + next item. + """ + rv = self.current + self.pos = (self.pos + 1) % len(self.items) + return rv + + __next__ = next + + +class Joiner(object): + """A joining helper for templates.""" + + def __init__(self, sep=u", "): + self.sep = sep + self.used = False + + def __call__(self): + if not self.used: + self.used = True + return u"" + return self.sep + + +class Namespace(object): + """A namespace object that can hold arbitrary attributes. It may be + initialized from a dictionary or with keyword arguments.""" + + def __init__(*args, **kwargs): # noqa: B902 + self, args = args[0], args[1:] + self.__attrs = dict(*args, **kwargs) + + def __getattribute__(self, name): + # __class__ is needed for the awaitable check in async mode + if name in {"_Namespace__attrs", "__class__"}: + return object.__getattribute__(self, name) + try: + return self.__attrs[name] + except KeyError: + raise AttributeError(name) + + def __setitem__(self, name, value): + self.__attrs[name] = value + + def __repr__(self): + return "<Namespace %r>" % self.__attrs + + +# does this python version support async for in and async generators? +try: + exec("async def _():\n async for _ in ():\n yield _") + have_async_gen = True +except SyntaxError: + have_async_gen = False + + +def soft_unicode(s): + from markupsafe import soft_unicode + + warnings.warn( + "'jinja2.utils.soft_unicode' will be removed in version 3.0." + " Use 'markupsafe.soft_unicode' instead.", + DeprecationWarning, + stacklevel=2, + ) + return soft_unicode(s) diff --git a/contrib/python/Jinja2/py2/jinja2/visitor.py b/contrib/python/Jinja2/py2/jinja2/visitor.py new file mode 100644 index 0000000000..d1365bf10e --- /dev/null +++ b/contrib/python/Jinja2/py2/jinja2/visitor.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +"""API for traversing the AST nodes. Implemented by the compiler and +meta introspection. +""" +from .nodes import Node + + +class NodeVisitor(object): + """Walks the abstract syntax tree and call visitor functions for every + node found. The visitor functions may return values which will be + forwarded by the `visit` method. + + Per default the visitor functions for the nodes are ``'visit_'`` + + class name of the node. So a `TryFinally` node visit function would + be `visit_TryFinally`. This behavior can be changed by overriding + the `get_visitor` function. If no visitor function exists for a node + (return value `None`) the `generic_visit` visitor is used instead. + """ + + def get_visitor(self, node): + """Return the visitor function for this node or `None` if no visitor + exists for this node. In that case the generic visit function is + used instead. + """ + method = "visit_" + node.__class__.__name__ + return getattr(self, method, None) + + def visit(self, node, *args, **kwargs): + """Visit a node.""" + f = self.get_visitor(node) + if f is not None: + return f(node, *args, **kwargs) + return self.generic_visit(node, *args, **kwargs) + + def generic_visit(self, node, *args, **kwargs): + """Called if no explicit visitor function exists for a node.""" + for node in node.iter_child_nodes(): + self.visit(node, *args, **kwargs) + + +class NodeTransformer(NodeVisitor): + """Walks the abstract syntax tree and allows modifications of nodes. + + The `NodeTransformer` will walk the AST and use the return value of the + visitor functions to replace or remove the old node. If the return + value of the visitor function is `None` the node will be removed + from the previous location otherwise it's replaced with the return + value. The return value may be the original node in which case no + replacement takes place. + """ + + def generic_visit(self, node, *args, **kwargs): + for field, old_value in node.iter_fields(): + if isinstance(old_value, list): + new_values = [] + for value in old_value: + if isinstance(value, Node): + value = self.visit(value, *args, **kwargs) + if value is None: + continue + elif not isinstance(value, Node): + new_values.extend(value) + continue + new_values.append(value) + old_value[:] = new_values + elif isinstance(old_value, Node): + new_node = self.visit(old_value, *args, **kwargs) + if new_node is None: + delattr(node, field) + else: + setattr(node, field, new_node) + return node + + def visit_list(self, node, *args, **kwargs): + """As transformers may return lists in some places this method + can be used to enforce a list as return value. + """ + rv = self.visit(node, *args, **kwargs) + if not isinstance(rv, list): + rv = [rv] + return rv diff --git a/contrib/python/Jinja2/py2/tests/conftest.py b/contrib/python/Jinja2/py2/tests/conftest.py new file mode 100644 index 0000000000..c34e6c53c6 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/conftest.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +import os + +import pytest + +from jinja2 import Environment +from jinja2 import loaders +from jinja2.utils import have_async_gen + + +def pytest_ignore_collect(path): + if "async" in path.basename and not have_async_gen: + return True + return False + + +@pytest.fixture +def env(): + """returns a new environment.""" + return Environment() + + +@pytest.fixture +def dict_loader(): + """returns DictLoader""" + return loaders.DictLoader({"justdict.html": "FOO"}) + + +@pytest.fixture +def package_loader(): + """returns PackageLoader initialized from templates""" + return loaders.PackageLoader("res", "templates") + + +@pytest.fixture +def filesystem_loader(): + """returns FileSystemLoader initialized to res/templates directory""" + try: + import yatest.common + here = yatest.common.test_source_path() + except ImportError: + here = os.path.dirname(os.path.abspath(__file__)) + return loaders.FileSystemLoader(here + "/res/templates") + + +@pytest.fixture +def function_loader(): + """returns a FunctionLoader""" + return loaders.FunctionLoader({"justfunction.html": "FOO"}.get) + + +@pytest.fixture +def choice_loader(dict_loader, package_loader): + """returns a ChoiceLoader""" + return loaders.ChoiceLoader([dict_loader, package_loader]) + + +@pytest.fixture +def prefix_loader(filesystem_loader, dict_loader): + """returns a PrefixLoader""" + return loaders.PrefixLoader({"a": filesystem_loader, "b": dict_loader}) diff --git a/contrib/python/Jinja2/py2/tests/res/__init__.py b/contrib/python/Jinja2/py2/tests/res/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/res/__init__.py diff --git a/contrib/python/Jinja2/py2/tests/res/templates/broken.html b/contrib/python/Jinja2/py2/tests/res/templates/broken.html new file mode 100644 index 0000000000..77669fae57 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/res/templates/broken.html @@ -0,0 +1,3 @@ +Before +{{ fail() }} +After diff --git a/contrib/python/Jinja2/py2/tests/res/templates/foo/test.html b/contrib/python/Jinja2/py2/tests/res/templates/foo/test.html new file mode 100644 index 0000000000..b7d6715e2d --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/res/templates/foo/test.html @@ -0,0 +1 @@ +FOO diff --git a/contrib/python/Jinja2/py2/tests/res/templates/mojibake.txt b/contrib/python/Jinja2/py2/tests/res/templates/mojibake.txt new file mode 100644 index 0000000000..4b94aa63dc --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/res/templates/mojibake.txt @@ -0,0 +1 @@ +文字化け diff --git a/contrib/python/Jinja2/py2/tests/res/templates/syntaxerror.html b/contrib/python/Jinja2/py2/tests/res/templates/syntaxerror.html new file mode 100644 index 0000000000..f21b817939 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/res/templates/syntaxerror.html @@ -0,0 +1,4 @@ +Foo +{% for item in broken %} + ... +{% endif %} diff --git a/contrib/python/Jinja2/py2/tests/res/templates/test.html b/contrib/python/Jinja2/py2/tests/res/templates/test.html new file mode 100644 index 0000000000..ba578e48b1 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/res/templates/test.html @@ -0,0 +1 @@ +BAR diff --git a/contrib/python/Jinja2/py2/tests/res/templates2/foo b/contrib/python/Jinja2/py2/tests/res/templates2/foo new file mode 100644 index 0000000000..1c4ad3e496 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/res/templates2/foo @@ -0,0 +1,2 @@ +Looks like the start of templates/foo/test.html +Tested by test_filesystem_loader_overlapping_names diff --git a/contrib/python/Jinja2/py2/tests/test_api.py b/contrib/python/Jinja2/py2/tests/test_api.py new file mode 100644 index 0000000000..7a1cae866b --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_api.py @@ -0,0 +1,437 @@ +# -*- coding: utf-8 -*- +import os +import shutil +import tempfile + +import pytest + +from jinja2 import ChainableUndefined +from jinja2 import DebugUndefined +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import is_undefined +from jinja2 import make_logging_undefined +from jinja2 import meta +from jinja2 import StrictUndefined +from jinja2 import Template +from jinja2 import TemplatesNotFound +from jinja2 import Undefined +from jinja2 import UndefinedError +from jinja2.compiler import CodeGenerator +from jinja2.runtime import Context +from jinja2.utils import contextfunction +from jinja2.utils import Cycler +from jinja2.utils import environmentfunction +from jinja2.utils import evalcontextfunction + + +class TestExtendedAPI(object): + def test_item_and_attribute(self, env): + from jinja2.sandbox import SandboxedEnvironment + + for env in Environment(), SandboxedEnvironment(): + # the |list is necessary for python3 + tmpl = env.from_string("{{ foo.items()|list }}") + assert tmpl.render(foo={"items": 42}) == "[('items', 42)]" + tmpl = env.from_string('{{ foo|attr("items")()|list }}') + assert tmpl.render(foo={"items": 42}) == "[('items', 42)]" + tmpl = env.from_string('{{ foo["items"] }}') + assert tmpl.render(foo={"items": 42}) == "42" + + def test_finalize(self): + e = Environment(finalize=lambda v: "" if v is None else v) + t = e.from_string("{% for item in seq %}|{{ item }}{% endfor %}") + assert t.render(seq=(None, 1, "foo")) == "||1|foo" + + def test_finalize_constant_expression(self): + e = Environment(finalize=lambda v: "" if v is None else v) + t = e.from_string("<{{ none }}>") + assert t.render() == "<>" + + def test_no_finalize_template_data(self): + e = Environment(finalize=lambda v: type(v).__name__) + t = e.from_string("<{{ value }}>") + # If template data was finalized, it would print "strintstr". + assert t.render(value=123) == "<int>" + + def test_context_finalize(self): + @contextfunction + def finalize(context, value): + return value * context["scale"] + + e = Environment(finalize=finalize) + t = e.from_string("{{ value }}") + assert t.render(value=5, scale=3) == "15" + + def test_eval_finalize(self): + @evalcontextfunction + def finalize(eval_ctx, value): + return str(eval_ctx.autoescape) + value + + e = Environment(finalize=finalize, autoescape=True) + t = e.from_string("{{ value }}") + assert t.render(value="<script>") == "True<script>" + + def test_env_autoescape(self): + @environmentfunction + def finalize(env, value): + return " ".join( + (env.variable_start_string, repr(value), env.variable_end_string) + ) + + e = Environment(finalize=finalize) + t = e.from_string("{{ value }}") + assert t.render(value="hello") == "{{ 'hello' }}" + + def test_cycler(self, env): + items = 1, 2, 3 + c = Cycler(*items) + for item in items + items: + assert c.current == item + assert next(c) == item + next(c) + assert c.current == 2 + c.reset() + assert c.current == 1 + + def test_expressions(self, env): + expr = env.compile_expression("foo") + assert expr() is None + assert expr(foo=42) == 42 + expr2 = env.compile_expression("foo", undefined_to_none=False) + assert is_undefined(expr2()) + + expr = env.compile_expression("42 + foo") + assert expr(foo=42) == 84 + + def test_template_passthrough(self, env): + t = Template("Content") + assert env.get_template(t) is t + assert env.select_template([t]) is t + assert env.get_or_select_template([t]) is t + assert env.get_or_select_template(t) is t + + def test_get_template_undefined(self, env): + """Passing Undefined to get/select_template raises an + UndefinedError or shows the undefined message in the list. + """ + env.loader = DictLoader({}) + t = Undefined(name="no_name_1") + + with pytest.raises(UndefinedError): + env.get_template(t) + + with pytest.raises(UndefinedError): + env.get_or_select_template(t) + + with pytest.raises(UndefinedError): + env.select_template(t) + + with pytest.raises(TemplatesNotFound) as exc_info: + env.select_template([t, "no_name_2"]) + + exc_message = str(exc_info.value) + assert "'no_name_1' is undefined" in exc_message + assert "no_name_2" in exc_message + + def test_autoescape_autoselect(self, env): + def select_autoescape(name): + if name is None or "." not in name: + return False + return name.endswith(".html") + + env = Environment( + autoescape=select_autoescape, + loader=DictLoader({"test.txt": "{{ foo }}", "test.html": "{{ foo }}"}), + ) + t = env.get_template("test.txt") + assert t.render(foo="<foo>") == "<foo>" + t = env.get_template("test.html") + assert t.render(foo="<foo>") == "<foo>" + t = env.from_string("{{ foo }}") + assert t.render(foo="<foo>") == "<foo>" + + def test_sandbox_max_range(self, env): + from jinja2.sandbox import SandboxedEnvironment, MAX_RANGE + + env = SandboxedEnvironment() + t = env.from_string("{% for item in range(total) %}{{ item }}{% endfor %}") + + with pytest.raises(OverflowError): + t.render(total=MAX_RANGE + 1) + + +class TestMeta(object): + def test_find_undeclared_variables(self, env): + ast = env.parse("{% set foo = 42 %}{{ bar + foo }}") + x = meta.find_undeclared_variables(ast) + assert x == set(["bar"]) + + ast = env.parse( + "{% set foo = 42 %}{{ bar + foo }}" + "{% macro meh(x) %}{{ x }}{% endmacro %}" + "{% for item in seq %}{{ muh(item) + meh(seq) }}" + "{% endfor %}" + ) + x = meta.find_undeclared_variables(ast) + assert x == set(["bar", "seq", "muh"]) + + ast = env.parse("{% for x in range(5) %}{{ x }}{% endfor %}{{ foo }}") + x = meta.find_undeclared_variables(ast) + assert x == set(["foo"]) + + def test_find_refererenced_templates(self, env): + ast = env.parse('{% extends "layout.html" %}{% include helper %}') + i = meta.find_referenced_templates(ast) + assert next(i) == "layout.html" + assert next(i) is None + assert list(i) == [] + + ast = env.parse( + '{% extends "layout.html" %}' + '{% from "test.html" import a, b as c %}' + '{% import "meh.html" as meh %}' + '{% include "muh.html" %}' + ) + i = meta.find_referenced_templates(ast) + assert list(i) == ["layout.html", "test.html", "meh.html", "muh.html"] + + def test_find_included_templates(self, env): + ast = env.parse('{% include ["foo.html", "bar.html"] %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ["foo.html", "bar.html"] + + ast = env.parse('{% include ("foo.html", "bar.html") %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ["foo.html", "bar.html"] + + ast = env.parse('{% include ["foo.html", "bar.html", foo] %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ["foo.html", "bar.html", None] + + ast = env.parse('{% include ("foo.html", "bar.html", foo) %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ["foo.html", "bar.html", None] + + +class TestStreaming(object): + def test_basic_streaming(self, env): + t = env.from_string( + "<ul>{% for item in seq %}<li>{{ loop.index }} - {{ item }}</li>" + "{%- endfor %}</ul>" + ) + stream = t.stream(seq=list(range(3))) + assert next(stream) == "<ul>" + assert "".join(stream) == "<li>1 - 0</li><li>2 - 1</li><li>3 - 2</li></ul>" + + def test_buffered_streaming(self, env): + tmpl = env.from_string( + "<ul>{% for item in seq %}<li>{{ loop.index }} - {{ item }}</li>" + "{%- endfor %}</ul>" + ) + stream = tmpl.stream(seq=list(range(3))) + stream.enable_buffering(size=3) + assert next(stream) == u"<ul><li>1" + assert next(stream) == u" - 0</li>" + + def test_streaming_behavior(self, env): + tmpl = env.from_string("") + stream = tmpl.stream() + assert not stream.buffered + stream.enable_buffering(20) + assert stream.buffered + stream.disable_buffering() + assert not stream.buffered + + def test_dump_stream(self, env): + tmp = tempfile.mkdtemp() + try: + tmpl = env.from_string(u"\u2713") + stream = tmpl.stream() + stream.dump(os.path.join(tmp, "dump.txt"), "utf-8") + with open(os.path.join(tmp, "dump.txt"), "rb") as f: + assert f.read() == b"\xe2\x9c\x93" + finally: + shutil.rmtree(tmp) + + +class TestUndefined(object): + def test_stopiteration_is_undefined(self): + def test(): + raise StopIteration() + + t = Template("A{{ test() }}B") + assert t.render(test=test) == "AB" + t = Template("A{{ test().missingattribute }}B") + pytest.raises(UndefinedError, t.render, test=test) + + def test_undefined_and_special_attributes(self): + with pytest.raises(AttributeError): + Undefined("Foo").__dict__ + + def test_undefined_attribute_error(self): + # Django's LazyObject turns the __class__ attribute into a + # property that resolves the wrapped function. If that wrapped + # function raises an AttributeError, printing the repr of the + # object in the undefined message would cause a RecursionError. + class Error(object): + @property + def __class__(self): + raise AttributeError() + + u = Undefined(obj=Error(), name="hello") + + with pytest.raises(UndefinedError): + getattr(u, "recursion", None) + + def test_logging_undefined(self): + _messages = [] + + class DebugLogger(object): + def warning(self, msg, *args): + _messages.append("W:" + msg % args) + + def error(self, msg, *args): + _messages.append("E:" + msg % args) + + logging_undefined = make_logging_undefined(DebugLogger()) + env = Environment(undefined=logging_undefined) + assert env.from_string("{{ missing }}").render() == u"" + pytest.raises(UndefinedError, env.from_string("{{ missing.attribute }}").render) + assert env.from_string("{{ missing|list }}").render() == "[]" + assert env.from_string("{{ missing is not defined }}").render() == "True" + assert env.from_string("{{ foo.missing }}").render(foo=42) == "" + assert env.from_string("{{ not missing }}").render() == "True" + assert _messages == [ + "W:Template variable warning: missing is undefined", + "E:Template variable error: 'missing' is undefined", + "W:Template variable warning: missing is undefined", + "W:Template variable warning: int object has no attribute missing", + "W:Template variable warning: missing is undefined", + ] + + def test_default_undefined(self): + env = Environment(undefined=Undefined) + assert env.from_string("{{ missing }}").render() == u"" + pytest.raises(UndefinedError, env.from_string("{{ missing.attribute }}").render) + assert env.from_string("{{ missing|list }}").render() == "[]" + assert env.from_string("{{ missing is not defined }}").render() == "True" + assert env.from_string("{{ foo.missing }}").render(foo=42) == "" + assert env.from_string("{{ not missing }}").render() == "True" + pytest.raises(UndefinedError, env.from_string("{{ missing - 1}}").render) + und1 = Undefined(name="x") + und2 = Undefined(name="y") + assert und1 == und2 + assert und1 != 42 + assert hash(und1) == hash(und2) == hash(Undefined()) + with pytest.raises(AttributeError): + getattr(Undefined, "__slots__") # noqa: B009 + + def test_chainable_undefined(self): + env = Environment(undefined=ChainableUndefined) + # The following tests are copied from test_default_undefined + assert env.from_string("{{ missing }}").render() == u"" + assert env.from_string("{{ missing|list }}").render() == "[]" + assert env.from_string("{{ missing is not defined }}").render() == "True" + assert env.from_string("{{ foo.missing }}").render(foo=42) == "" + assert env.from_string("{{ not missing }}").render() == "True" + pytest.raises(UndefinedError, env.from_string("{{ missing - 1}}").render) + with pytest.raises(AttributeError): + getattr(ChainableUndefined, "__slots__") # noqa: B009 + + # The following tests ensure subclass functionality works as expected + assert env.from_string('{{ missing.bar["baz"] }}').render() == u"" + assert ( + env.from_string('{{ foo.bar["baz"]._undefined_name }}').render() == u"foo" + ) + assert ( + env.from_string('{{ foo.bar["baz"]._undefined_name }}').render(foo=42) + == u"bar" + ) + assert ( + env.from_string('{{ foo.bar["baz"]._undefined_name }}').render( + foo={"bar": 42} + ) + == u"baz" + ) + + def test_debug_undefined(self): + env = Environment(undefined=DebugUndefined) + assert env.from_string("{{ missing }}").render() == "{{ missing }}" + pytest.raises(UndefinedError, env.from_string("{{ missing.attribute }}").render) + assert env.from_string("{{ missing|list }}").render() == "[]" + assert env.from_string("{{ missing is not defined }}").render() == "True" + assert ( + env.from_string("{{ foo.missing }}").render(foo=42) + == u"{{ no such element: int object['missing'] }}" + ) + assert env.from_string("{{ not missing }}").render() == "True" + undefined_hint = "this is testing undefined hint of DebugUndefined" + assert ( + str(DebugUndefined(hint=undefined_hint)) + == u"{{ undefined value printed: %s }}" % undefined_hint + ) + with pytest.raises(AttributeError): + getattr(DebugUndefined, "__slots__") # noqa: B009 + + def test_strict_undefined(self): + env = Environment(undefined=StrictUndefined) + pytest.raises(UndefinedError, env.from_string("{{ missing }}").render) + pytest.raises(UndefinedError, env.from_string("{{ missing.attribute }}").render) + pytest.raises(UndefinedError, env.from_string("{{ missing|list }}").render) + assert env.from_string("{{ missing is not defined }}").render() == "True" + pytest.raises( + UndefinedError, env.from_string("{{ foo.missing }}").render, foo=42 + ) + pytest.raises(UndefinedError, env.from_string("{{ not missing }}").render) + assert ( + env.from_string('{{ missing|default("default", true) }}').render() + == "default" + ) + with pytest.raises(AttributeError): + getattr(StrictUndefined, "__slots__") # noqa: B009 + assert env.from_string('{{ "foo" if false }}').render() == "" + + def test_indexing_gives_undefined(self): + t = Template("{{ var[42].foo }}") + pytest.raises(UndefinedError, t.render, var=0) + + def test_none_gives_proper_error(self): + with pytest.raises(UndefinedError, match="'None' has no attribute 'split'"): + Environment().getattr(None, "split")() + + def test_object_repr(self): + with pytest.raises( + UndefinedError, match="'int object' has no attribute 'upper'" + ): + Undefined(obj=42, name="upper")() + + +class TestLowLevel(object): + def test_custom_code_generator(self): + class CustomCodeGenerator(CodeGenerator): + def visit_Const(self, node, frame=None): + # This method is pure nonsense, but works fine for testing... + if node.value == "foo": + self.write(repr("bar")) + else: + super(CustomCodeGenerator, self).visit_Const(node, frame) + + class CustomEnvironment(Environment): + code_generator_class = CustomCodeGenerator + + env = CustomEnvironment() + tmpl = env.from_string('{% set foo = "foo" %}{{ foo }}') + assert tmpl.render() == "bar" + + def test_custom_context(self): + class CustomContext(Context): + def resolve_or_missing(self, key): + return "resolve-" + key + + class CustomEnvironment(Environment): + context_class = CustomContext + + env = CustomEnvironment() + tmpl = env.from_string("{{ foo }}") + assert tmpl.render() == "resolve-foo" diff --git a/contrib/python/Jinja2/py2/tests/test_async.py b/contrib/python/Jinja2/py2/tests/test_async.py new file mode 100644 index 0000000000..2b9974e35d --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_async.py @@ -0,0 +1,591 @@ +import asyncio + +import pytest + +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import Template +from jinja2.asyncsupport import auto_aiter +from jinja2.exceptions import TemplateNotFound +from jinja2.exceptions import TemplatesNotFound +from jinja2.exceptions import UndefinedError + + +def run(coro): + loop = asyncio.get_event_loop() + return loop.run_until_complete(coro) + + +def test_basic_async(): + t = Template( + "{% for item in [1, 2, 3] %}[{{ item }}]{% endfor %}", enable_async=True + ) + + async def func(): + return await t.render_async() + + rv = run(func()) + assert rv == "[1][2][3]" + + +def test_await_on_calls(): + t = Template("{{ async_func() + normal_func() }}", enable_async=True) + + async def async_func(): + return 42 + + def normal_func(): + return 23 + + async def func(): + return await t.render_async(async_func=async_func, normal_func=normal_func) + + rv = run(func()) + assert rv == "65" + + +def test_await_on_calls_normal_render(): + t = Template("{{ async_func() + normal_func() }}", enable_async=True) + + async def async_func(): + return 42 + + def normal_func(): + return 23 + + rv = t.render(async_func=async_func, normal_func=normal_func) + + assert rv == "65" + + +def test_await_and_macros(): + t = Template( + "{% macro foo(x) %}[{{ x }}][{{ async_func() }}]{% endmacro %}{{ foo(42) }}", + enable_async=True, + ) + + async def async_func(): + return 42 + + async def func(): + return await t.render_async(async_func=async_func) + + rv = run(func()) + assert rv == "[42][42]" + + +def test_async_blocks(): + t = Template( + "{% block foo %}<Test>{% endblock %}{{ self.foo() }}", + enable_async=True, + autoescape=True, + ) + + async def func(): + return await t.render_async() + + rv = run(func()) + assert rv == "<Test><Test>" + + +def test_async_generate(): + t = Template("{% for x in [1, 2, 3] %}{{ x }}{% endfor %}", enable_async=True) + rv = list(t.generate()) + assert rv == ["1", "2", "3"] + + +def test_async_iteration_in_templates(): + t = Template("{% for x in rng %}{{ x }}{% endfor %}", enable_async=True) + + async def async_iterator(): + for item in [1, 2, 3]: + yield item + + rv = list(t.generate(rng=async_iterator())) + assert rv == ["1", "2", "3"] + + +def test_async_iteration_in_templates_extended(): + t = Template( + "{% for x in rng %}{{ loop.index0 }}/{{ x }}{% endfor %}", enable_async=True + ) + stream = t.generate(rng=auto_aiter(range(1, 4))) + assert next(stream) == "0" + assert "".join(stream) == "/11/22/3" + + +@pytest.fixture +def test_env_async(): + env = Environment( + loader=DictLoader( + dict( + module="{% macro test() %}[{{ foo }}|{{ bar }}]{% endmacro %}", + header="[{{ foo }}|{{ 23 }}]", + o_printer="({{ o }})", + ) + ), + enable_async=True, + ) + env.globals["bar"] = 23 + return env + + +class TestAsyncImports(object): + def test_context_imports(self, test_env_async): + t = test_env_async.from_string('{% import "module" as m %}{{ m.test() }}') + assert t.render(foo=42) == "[|23]" + t = test_env_async.from_string( + '{% import "module" as m without context %}{{ m.test() }}' + ) + assert t.render(foo=42) == "[|23]" + t = test_env_async.from_string( + '{% import "module" as m with context %}{{ m.test() }}' + ) + assert t.render(foo=42) == "[42|23]" + t = test_env_async.from_string('{% from "module" import test %}{{ test() }}') + assert t.render(foo=42) == "[|23]" + t = test_env_async.from_string( + '{% from "module" import test without context %}{{ test() }}' + ) + assert t.render(foo=42) == "[|23]" + t = test_env_async.from_string( + '{% from "module" import test with context %}{{ test() }}' + ) + assert t.render(foo=42) == "[42|23]" + + def test_trailing_comma(self, test_env_async): + test_env_async.from_string('{% from "foo" import bar, baz with context %}') + test_env_async.from_string('{% from "foo" import bar, baz, with context %}') + test_env_async.from_string('{% from "foo" import bar, with context %}') + test_env_async.from_string('{% from "foo" import bar, with, context %}') + test_env_async.from_string('{% from "foo" import bar, with with context %}') + + def test_exports(self, test_env_async): + m = run( + test_env_async.from_string( + """ + {% macro toplevel() %}...{% endmacro %} + {% macro __private() %}...{% endmacro %} + {% set variable = 42 %} + {% for item in [1] %} + {% macro notthere() %}{% endmacro %} + {% endfor %} + """ + )._get_default_module_async() + ) + assert run(m.toplevel()) == "..." + assert not hasattr(m, "__missing") + assert m.variable == 42 + assert not hasattr(m, "notthere") + + +class TestAsyncIncludes(object): + def test_context_include(self, test_env_async): + t = test_env_async.from_string('{% include "header" %}') + assert t.render(foo=42) == "[42|23]" + t = test_env_async.from_string('{% include "header" with context %}') + assert t.render(foo=42) == "[42|23]" + t = test_env_async.from_string('{% include "header" without context %}') + assert t.render(foo=42) == "[|23]" + + def test_choice_includes(self, test_env_async): + t = test_env_async.from_string('{% include ["missing", "header"] %}') + assert t.render(foo=42) == "[42|23]" + + t = test_env_async.from_string( + '{% include ["missing", "missing2"] ignore missing %}' + ) + assert t.render(foo=42) == "" + + t = test_env_async.from_string('{% include ["missing", "missing2"] %}') + pytest.raises(TemplateNotFound, t.render) + with pytest.raises(TemplatesNotFound) as e: + t.render() + + assert e.value.templates == ["missing", "missing2"] + assert e.value.name == "missing2" + + def test_includes(t, **ctx): + ctx["foo"] = 42 + assert t.render(ctx) == "[42|23]" + + t = test_env_async.from_string('{% include ["missing", "header"] %}') + test_includes(t) + t = test_env_async.from_string("{% include x %}") + test_includes(t, x=["missing", "header"]) + t = test_env_async.from_string('{% include [x, "header"] %}') + test_includes(t, x="missing") + t = test_env_async.from_string("{% include x %}") + test_includes(t, x="header") + t = test_env_async.from_string("{% include x %}") + test_includes(t, x="header") + t = test_env_async.from_string("{% include [x] %}") + test_includes(t, x="header") + + def test_include_ignoring_missing(self, test_env_async): + t = test_env_async.from_string('{% include "missing" %}') + pytest.raises(TemplateNotFound, t.render) + for extra in "", "with context", "without context": + t = test_env_async.from_string( + '{% include "missing" ignore missing ' + extra + " %}" + ) + assert t.render() == "" + + def test_context_include_with_overrides(self, test_env_async): + env = Environment( + loader=DictLoader( + dict( + main="{% for item in [1, 2, 3] %}{% include 'item' %}{% endfor %}", + item="{{ item }}", + ) + ) + ) + assert env.get_template("main").render() == "123" + + def test_unoptimized_scopes(self, test_env_async): + t = test_env_async.from_string( + """ + {% macro outer(o) %} + {% macro inner() %} + {% include "o_printer" %} + {% endmacro %} + {{ inner() }} + {% endmacro %} + {{ outer("FOO") }} + """ + ) + assert t.render().strip() == "(FOO)" + + def test_unoptimized_scopes_autoescape(self): + env = Environment( + loader=DictLoader(dict(o_printer="({{ o }})",)), + autoescape=True, + enable_async=True, + ) + t = env.from_string( + """ + {% macro outer(o) %} + {% macro inner() %} + {% include "o_printer" %} + {% endmacro %} + {{ inner() }} + {% endmacro %} + {{ outer("FOO") }} + """ + ) + assert t.render().strip() == "(FOO)" + + +class TestAsyncForLoop(object): + def test_simple(self, test_env_async): + tmpl = test_env_async.from_string("{% for item in seq %}{{ item }}{% endfor %}") + assert tmpl.render(seq=list(range(10))) == "0123456789" + + def test_else(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for item in seq %}XXX{% else %}...{% endfor %}" + ) + assert tmpl.render() == "..." + + def test_empty_blocks(self, test_env_async): + tmpl = test_env_async.from_string( + "<{% for item in seq %}{% else %}{% endfor %}>" + ) + assert tmpl.render() == "<>" + + @pytest.mark.parametrize( + "transform", [lambda x: x, iter, reversed, lambda x: (i for i in x), auto_aiter] + ) + def test_context_vars(self, test_env_async, transform): + t = test_env_async.from_string( + "{% for item in seq %}{{ loop.index }}|{{ loop.index0 }}" + "|{{ loop.revindex }}|{{ loop.revindex0 }}|{{ loop.first }}" + "|{{ loop.last }}|{{ loop.length }}\n{% endfor %}" + ) + out = t.render(seq=transform([42, 24])) + assert out == "1|0|2|1|True|False|2\n2|1|1|0|False|True|2\n" + + def test_cycling(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in seq %}{{ + loop.cycle('<1>', '<2>') }}{% endfor %}{% + for item in seq %}{{ loop.cycle(*through) }}{% endfor %}""" + ) + output = tmpl.render(seq=list(range(4)), through=("<1>", "<2>")) + assert output == "<1><2>" * 4 + + def test_lookaround(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in seq -%} + {{ loop.previtem|default('x') }}-{{ item }}-{{ + loop.nextitem|default('x') }}| + {%- endfor %}""" + ) + output = tmpl.render(seq=list(range(4))) + assert output == "x-0-1|0-1-2|1-2-3|2-3-x|" + + def test_changed(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in seq -%} + {{ loop.changed(item) }}, + {%- endfor %}""" + ) + output = tmpl.render(seq=[None, None, 1, 2, 2, 3, 4, 4, 4]) + assert output == "True,False,True,True,False,True,True,False,False," + + def test_scope(self, test_env_async): + tmpl = test_env_async.from_string("{% for item in seq %}{% endfor %}{{ item }}") + output = tmpl.render(seq=list(range(10))) + assert not output + + def test_varlen(self, test_env_async): + def inner(): + for item in range(5): + yield item + + tmpl = test_env_async.from_string( + "{% for item in iter %}{{ item }}{% endfor %}" + ) + output = tmpl.render(iter=inner()) + assert output == "01234" + + def test_noniter(self, test_env_async): + tmpl = test_env_async.from_string("{% for item in none %}...{% endfor %}") + pytest.raises(TypeError, tmpl.render) + + def test_recursive(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in seq recursive -%} + [{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[1<[1][2]>][2<[1][2]>][3<[a]>]" + ) + + def test_recursive_lookaround(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in seq recursive -%} + [{{ loop.previtem.a if loop.previtem is defined else 'x' }}.{{ + item.a }}.{{ loop.nextitem.a if loop.nextitem is defined else 'x' + }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[x.1.2<[x.1.2][1.2.x]>][1.2.3<[x.1.2][1.2.x]>][2.3.x<[x.a.x]>]" + ) + + def test_recursive_depth0(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for item in seq recursive %}[{{ loop.depth0 }}:{{ item.a }}" + "{% if item.b %}<{{ loop(item.b) }}>{% endif %}]{% endfor %}" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[0:1<[1:1][1:2]>][0:2<[1:1][1:2]>][0:3<[1:a]>]" + ) + + def test_recursive_depth(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for item in seq recursive %}[{{ loop.depth }}:{{ item.a }}" + "{% if item.b %}<{{ loop(item.b) }}>{% endif %}]{% endfor %}" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[1:1<[2:1][2:2]>][1:2<[2:1][2:2]>][1:3<[2:a]>]" + ) + + def test_looploop(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for row in table %} + {%- set rowloop = loop -%} + {% for cell in row -%} + [{{ rowloop.index }}|{{ loop.index }}] + {%- endfor %} + {%- endfor %}""" + ) + assert tmpl.render(table=["ab", "cd"]) == "[1|1][1|2][2|1][2|2]" + + def test_reversed_bug(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for i in items %}{{ i }}" + "{% if not loop.last %}" + ",{% endif %}{% endfor %}" + ) + assert tmpl.render(items=reversed([3, 2, 1])) == "1,2,3" + + def test_loop_errors(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in [1] if loop.index + == 0 %}...{% endfor %}""" + ) + pytest.raises(UndefinedError, tmpl.render) + tmpl = test_env_async.from_string( + """{% for item in [] %}...{% else + %}{{ loop }}{% endfor %}""" + ) + assert tmpl.render() == "" + + def test_loop_filter(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for item in range(10) if item is even %}[{{ item }}]{% endfor %}" + ) + assert tmpl.render() == "[0][2][4][6][8]" + tmpl = test_env_async.from_string( + """ + {%- for item in range(10) if item is even %}[{{ + loop.index }}:{{ item }}]{% endfor %}""" + ) + assert tmpl.render() == "[1:0][2:2][3:4][4:6][5:8]" + + def test_scoped_special_var(self, test_env_async): + t = test_env_async.from_string( + "{% for s in seq %}[{{ loop.first }}{% for c in s %}" + "|{{ loop.first }}{% endfor %}]{% endfor %}" + ) + assert t.render(seq=("ab", "cd")) == "[True|True|False][False|True|False]" + + def test_scoped_loop_var(self, test_env_async): + t = test_env_async.from_string( + "{% for x in seq %}{{ loop.first }}" + "{% for y in seq %}{% endfor %}{% endfor %}" + ) + assert t.render(seq="ab") == "TrueFalse" + t = test_env_async.from_string( + "{% for x in seq %}{% for y in seq %}" + "{{ loop.first }}{% endfor %}{% endfor %}" + ) + assert t.render(seq="ab") == "TrueFalseTrueFalse" + + def test_recursive_empty_loop_iter(self, test_env_async): + t = test_env_async.from_string( + """ + {%- for item in foo recursive -%}{%- endfor -%} + """ + ) + assert t.render(dict(foo=[])) == "" + + def test_call_in_loop(self, test_env_async): + t = test_env_async.from_string( + """ + {%- macro do_something() -%} + [{{ caller() }}] + {%- endmacro %} + + {%- for i in [1, 2, 3] %} + {%- call do_something() -%} + {{ i }} + {%- endcall %} + {%- endfor -%} + """ + ) + assert t.render() == "[1][2][3]" + + def test_scoping_bug(self, test_env_async): + t = test_env_async.from_string( + """ + {%- for item in foo %}...{{ item }}...{% endfor %} + {%- macro item(a) %}...{{ a }}...{% endmacro %} + {{- item(2) -}} + """ + ) + assert t.render(foo=(1,)) == "...1......2..." + + def test_unpacking(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for a, b, c in [[1, 2, 3]] %}{{ a }}|{{ b }}|{{ c }}{% endfor %}" + ) + assert tmpl.render() == "1|2|3" + + def test_recursive_loop_filter(self, test_env_async): + t = test_env_async.from_string( + """ + <?xml version="1.0" encoding="UTF-8"?> + <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> + {%- for page in [site.root] if page.url != this recursive %} + <url><loc>{{ page.url }}</loc></url> + {{- loop(page.children) }} + {%- endfor %} + </urlset> + """ + ) + sm = t.render( + this="/foo", + site={"root": {"url": "/", "children": [{"url": "/foo"}, {"url": "/bar"}]}}, + ) + lines = [x.strip() for x in sm.splitlines() if x.strip()] + assert lines == [ + '<?xml version="1.0" encoding="UTF-8"?>', + '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">', + "<url><loc>/</loc></url>", + "<url><loc>/bar</loc></url>", + "</urlset>", + ] + + def test_nonrecursive_loop_filter(self, test_env_async): + t = test_env_async.from_string( + """ + <?xml version="1.0" encoding="UTF-8"?> + <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> + {%- for page in items if page.url != this %} + <url><loc>{{ page.url }}</loc></url> + {%- endfor %} + </urlset> + """ + ) + sm = t.render( + this="/foo", items=[{"url": "/"}, {"url": "/foo"}, {"url": "/bar"}] + ) + lines = [x.strip() for x in sm.splitlines() if x.strip()] + assert lines == [ + '<?xml version="1.0" encoding="UTF-8"?>', + '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">', + "<url><loc>/</loc></url>", + "<url><loc>/bar</loc></url>", + "</urlset>", + ] + + def test_bare_async(self, test_env_async): + t = test_env_async.from_string('{% extends "header" %}') + assert t.render(foo=42) == "[42|23]" + + def test_awaitable_property_slicing(self, test_env_async): + t = test_env_async.from_string("{% for x in a.b[:1] %}{{ x }}{% endfor %}") + assert t.render(a=dict(b=[1, 2, 3])) == "1" + + +def test_namespace_awaitable(test_env_async): + async def _test(): + t = test_env_async.from_string( + '{% set ns = namespace(foo="Bar") %}{{ ns.foo }}' + ) + actual = await t.render_async() + assert actual == "Bar" + + run(_test()) diff --git a/contrib/python/Jinja2/py2/tests/test_asyncfilters.py b/contrib/python/Jinja2/py2/tests/test_asyncfilters.py new file mode 100644 index 0000000000..7c737c835e --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_asyncfilters.py @@ -0,0 +1,224 @@ +from collections import namedtuple + +import pytest + +from jinja2 import Environment +from jinja2.utils import Markup + + +async def make_aiter(iter): + for item in iter: + yield item + + +def mark_dualiter(parameter, factory): + def decorator(f): + return pytest.mark.parametrize( + parameter, [lambda: factory(), lambda: make_aiter(factory())] + )(f) + + return decorator + + +@pytest.fixture +def env_async(): + return Environment(enable_async=True) + + +@mark_dualiter("foo", lambda: range(10)) +def test_first(env_async, foo): + tmpl = env_async.from_string("{{ foo()|first }}") + out = tmpl.render(foo=foo) + assert out == "0" + + +@mark_dualiter( + "items", + lambda: [ + {"foo": 1, "bar": 2}, + {"foo": 2, "bar": 3}, + {"foo": 1, "bar": 1}, + {"foo": 3, "bar": 4}, + ], +) +def test_groupby(env_async, items): + tmpl = env_async.from_string( + """ + {%- for grouper, list in items()|groupby('foo') -%} + {{ grouper }}{% for x in list %}: {{ x.foo }}, {{ x.bar }}{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render(items=items).split("|") == [ + "1: 1, 2: 1, 1", + "2: 2, 3", + "3: 3, 4", + "", + ] + + +@mark_dualiter("items", lambda: [("a", 1), ("a", 2), ("b", 1)]) +def test_groupby_tuple_index(env_async, items): + tmpl = env_async.from_string( + """ + {%- for grouper, list in items()|groupby(0) -%} + {{ grouper }}{% for x in list %}:{{ x.1 }}{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render(items=items) == "a:1:2|b:1|" + + +def make_articles(): + Date = namedtuple("Date", "day,month,year") + Article = namedtuple("Article", "title,date") + return [ + Article("aha", Date(1, 1, 1970)), + Article("interesting", Date(2, 1, 1970)), + Article("really?", Date(3, 1, 1970)), + Article("totally not", Date(1, 1, 1971)), + ] + + +@mark_dualiter("articles", make_articles) +def test_groupby_multidot(env_async, articles): + tmpl = env_async.from_string( + """ + {%- for year, list in articles()|groupby('date.year') -%} + {{ year }}{% for x in list %}[{{ x.title }}]{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render(articles=articles).split("|") == [ + "1970[aha][interesting][really?]", + "1971[totally not]", + "", + ] + + +@mark_dualiter("int_items", lambda: [1, 2, 3]) +def test_join_env_int(env_async, int_items): + tmpl = env_async.from_string('{{ items()|join("|") }}') + out = tmpl.render(items=int_items) + assert out == "1|2|3" + + +@mark_dualiter("string_items", lambda: ["<foo>", Markup("<span>foo</span>")]) +def test_join_string_list(string_items): + env2 = Environment(autoescape=True, enable_async=True) + tmpl = env2.from_string('{{ ["<foo>", "<span>foo</span>"|safe]|join }}') + assert tmpl.render(items=string_items) == "<foo><span>foo</span>" + + +def make_users(): + User = namedtuple("User", "username") + return map(User, ["foo", "bar"]) + + +@mark_dualiter("users", make_users) +def test_join_attribute(env_async, users): + tmpl = env_async.from_string("""{{ users()|join(', ', 'username') }}""") + assert tmpl.render(users=users) == "foo, bar" + + +@mark_dualiter("items", lambda: [1, 2, 3, 4, 5]) +def test_simple_reject(env_async, items): + tmpl = env_async.from_string('{{ items()|reject("odd")|join("|") }}') + assert tmpl.render(items=items) == "2|4" + + +@mark_dualiter("items", lambda: [None, False, 0, 1, 2, 3, 4, 5]) +def test_bool_reject(env_async, items): + tmpl = env_async.from_string('{{ items()|reject|join("|") }}') + assert tmpl.render(items=items) == "None|False|0" + + +@mark_dualiter("items", lambda: [1, 2, 3, 4, 5]) +def test_simple_select(env_async, items): + tmpl = env_async.from_string('{{ items()|select("odd")|join("|") }}') + assert tmpl.render(items=items) == "1|3|5" + + +@mark_dualiter("items", lambda: [None, False, 0, 1, 2, 3, 4, 5]) +def test_bool_select(env_async, items): + tmpl = env_async.from_string('{{ items()|select|join("|") }}') + assert tmpl.render(items=items) == "1|2|3|4|5" + + +def make_users(): + User = namedtuple("User", "name,is_active") + return [ + User("john", True), + User("jane", True), + User("mike", False), + ] + + +@mark_dualiter("users", make_users) +def test_simple_select_attr(env_async, users): + tmpl = env_async.from_string( + '{{ users()|selectattr("is_active")|map(attribute="name")|join("|") }}' + ) + assert tmpl.render(users=users) == "john|jane" + + +@mark_dualiter("items", lambda: list("123")) +def test_simple_map(env_async, items): + tmpl = env_async.from_string('{{ items()|map("int")|sum }}') + assert tmpl.render(items=items) == "6" + + +def test_map_sum(env_async): # async map + async filter + tmpl = env_async.from_string('{{ [[1,2], [3], [4,5,6]]|map("sum")|list }}') + assert tmpl.render() == "[3, 3, 15]" + + +@mark_dualiter("users", make_users) +def test_attribute_map(env_async, users): + tmpl = env_async.from_string('{{ users()|map(attribute="name")|join("|") }}') + assert tmpl.render(users=users) == "john|jane|mike" + + +def test_empty_map(env_async): + tmpl = env_async.from_string('{{ none|map("upper")|list }}') + assert tmpl.render() == "[]" + + +@mark_dualiter("items", lambda: [1, 2, 3, 4, 5, 6]) +def test_sum(env_async, items): + tmpl = env_async.from_string("""{{ items()|sum }}""") + assert tmpl.render(items=items) == "21" + + +@mark_dualiter("items", lambda: [{"value": 23}, {"value": 1}, {"value": 18}]) +def test_sum_attributes(env_async, items): + tmpl = env_async.from_string("""{{ items()|sum('value') }}""") + assert tmpl.render(items=items) + + +def test_sum_attributes_nested(env_async): + tmpl = env_async.from_string("""{{ values|sum('real.value') }}""") + assert ( + tmpl.render( + values=[ + {"real": {"value": 23}}, + {"real": {"value": 1}}, + {"real": {"value": 18}}, + ] + ) + == "42" + ) + + +def test_sum_attributes_tuple(env_async): + tmpl = env_async.from_string("""{{ values.items()|sum('1') }}""") + assert tmpl.render(values={"foo": 23, "bar": 1, "baz": 18}) == "42" + + +@mark_dualiter("items", lambda: range(10)) +def test_slice(env_async, items): + tmpl = env_async.from_string( + "{{ items()|slice(3)|list }}|{{ items()|slice(3, 'X')|list }}" + ) + out = tmpl.render(items=items) + assert out == ( + "[[0, 1, 2, 3], [4, 5, 6], [7, 8, 9]]|" + "[[0, 1, 2, 3], [4, 5, 6, 'X'], [7, 8, 9, 'X']]" + ) diff --git a/contrib/python/Jinja2/py2/tests/test_bytecode_cache.py b/contrib/python/Jinja2/py2/tests/test_bytecode_cache.py new file mode 100644 index 0000000000..c7882b1a2d --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_bytecode_cache.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +import pytest + +from jinja2 import Environment +from jinja2.bccache import Bucket +from jinja2.bccache import FileSystemBytecodeCache +from jinja2.bccache import MemcachedBytecodeCache +from jinja2.exceptions import TemplateNotFound + + +@pytest.fixture +def env(package_loader, tmp_path): + bytecode_cache = FileSystemBytecodeCache(str(tmp_path)) + return Environment(loader=package_loader, bytecode_cache=bytecode_cache) + + +class TestByteCodeCache(object): + def test_simple(self, env): + tmpl = env.get_template("test.html") + assert tmpl.render().strip() == "BAR" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + +class MockMemcached(object): + class Error(Exception): + pass + + key = None + value = None + timeout = None + + def get(self, key): + return self.value + + def set(self, key, value, timeout=None): + self.key = key + self.value = value + self.timeout = timeout + + def get_side_effect(self, key): + raise self.Error() + + def set_side_effect(self, *args): + raise self.Error() + + +class TestMemcachedBytecodeCache(object): + def test_dump_load(self): + memcached = MockMemcached() + m = MemcachedBytecodeCache(memcached) + + b = Bucket(None, "key", "") + b.code = "code" + m.dump_bytecode(b) + assert memcached.key == "jinja2/bytecode/key" + + b = Bucket(None, "key", "") + m.load_bytecode(b) + assert b.code == "code" + + def test_exception(self): + memcached = MockMemcached() + memcached.get = memcached.get_side_effect + memcached.set = memcached.set_side_effect + m = MemcachedBytecodeCache(memcached) + b = Bucket(None, "key", "") + b.code = "code" + + m.dump_bytecode(b) + m.load_bytecode(b) + + m.ignore_memcache_errors = False + + with pytest.raises(MockMemcached.Error): + m.dump_bytecode(b) + + with pytest.raises(MockMemcached.Error): + m.load_bytecode(b) diff --git a/contrib/python/Jinja2/py2/tests/test_core_tags.py b/contrib/python/Jinja2/py2/tests/test_core_tags.py new file mode 100644 index 0000000000..1bd96c41e9 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_core_tags.py @@ -0,0 +1,602 @@ +# -*- coding: utf-8 -*- +import pytest + +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import TemplateRuntimeError +from jinja2 import TemplateSyntaxError +from jinja2 import UndefinedError + + +@pytest.fixture +def env_trim(): + return Environment(trim_blocks=True) + + +class TestForLoop(object): + def test_simple(self, env): + tmpl = env.from_string("{% for item in seq %}{{ item }}{% endfor %}") + assert tmpl.render(seq=list(range(10))) == "0123456789" + + def test_else(self, env): + tmpl = env.from_string("{% for item in seq %}XXX{% else %}...{% endfor %}") + assert tmpl.render() == "..." + + def test_else_scoping_item(self, env): + tmpl = env.from_string("{% for item in [] %}{% else %}{{ item }}{% endfor %}") + assert tmpl.render(item=42) == "42" + + def test_empty_blocks(self, env): + tmpl = env.from_string("<{% for item in seq %}{% else %}{% endfor %}>") + assert tmpl.render() == "<>" + + def test_context_vars(self, env): + slist = [42, 24] + for seq in [slist, iter(slist), reversed(slist), (_ for _ in slist)]: + tmpl = env.from_string( + """{% for item in seq -%} + {{ loop.index }}|{{ loop.index0 }}|{{ loop.revindex }}|{{ + loop.revindex0 }}|{{ loop.first }}|{{ loop.last }}|{{ + loop.length }}###{% endfor %}""" + ) + one, two, _ = tmpl.render(seq=seq).split("###") + ( + one_index, + one_index0, + one_revindex, + one_revindex0, + one_first, + one_last, + one_length, + ) = one.split("|") + ( + two_index, + two_index0, + two_revindex, + two_revindex0, + two_first, + two_last, + two_length, + ) = two.split("|") + + assert int(one_index) == 1 and int(two_index) == 2 + assert int(one_index0) == 0 and int(two_index0) == 1 + assert int(one_revindex) == 2 and int(two_revindex) == 1 + assert int(one_revindex0) == 1 and int(two_revindex0) == 0 + assert one_first == "True" and two_first == "False" + assert one_last == "False" and two_last == "True" + assert one_length == two_length == "2" + + def test_cycling(self, env): + tmpl = env.from_string( + """{% for item in seq %}{{ + loop.cycle('<1>', '<2>') }}{% endfor %}{% + for item in seq %}{{ loop.cycle(*through) }}{% endfor %}""" + ) + output = tmpl.render(seq=list(range(4)), through=("<1>", "<2>")) + assert output == "<1><2>" * 4 + + def test_lookaround(self, env): + tmpl = env.from_string( + """{% for item in seq -%} + {{ loop.previtem|default('x') }}-{{ item }}-{{ + loop.nextitem|default('x') }}| + {%- endfor %}""" + ) + output = tmpl.render(seq=list(range(4))) + assert output == "x-0-1|0-1-2|1-2-3|2-3-x|" + + def test_changed(self, env): + tmpl = env.from_string( + """{% for item in seq -%} + {{ loop.changed(item) }}, + {%- endfor %}""" + ) + output = tmpl.render(seq=[None, None, 1, 2, 2, 3, 4, 4, 4]) + assert output == "True,False,True,True,False,True,True,False,False," + + def test_scope(self, env): + tmpl = env.from_string("{% for item in seq %}{% endfor %}{{ item }}") + output = tmpl.render(seq=list(range(10))) + assert not output + + def test_varlen(self, env): + def inner(): + for item in range(5): + yield item + + tmpl = env.from_string("{% for item in iter %}{{ item }}{% endfor %}") + output = tmpl.render(iter=inner()) + assert output == "01234" + + def test_noniter(self, env): + tmpl = env.from_string("{% for item in none %}...{% endfor %}") + pytest.raises(TypeError, tmpl.render) + + def test_recursive(self, env): + tmpl = env.from_string( + """{% for item in seq recursive -%} + [{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[1<[1][2]>][2<[1][2]>][3<[a]>]" + ) + + def test_recursive_lookaround(self, env): + tmpl = env.from_string( + """{% for item in seq recursive -%} + [{{ loop.previtem.a if loop.previtem is defined else 'x' }}.{{ + item.a }}.{{ loop.nextitem.a if loop.nextitem is defined else 'x' + }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[x.1.2<[x.1.2][1.2.x]>][1.2.3<[x.1.2][1.2.x]>][2.3.x<[x.a.x]>]" + ) + + def test_recursive_depth0(self, env): + tmpl = env.from_string( + """{% for item in seq recursive -%} + [{{ loop.depth0 }}:{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[0:1<[1:1][1:2]>][0:2<[1:1][1:2]>][0:3<[1:a]>]" + ) + + def test_recursive_depth(self, env): + tmpl = env.from_string( + """{% for item in seq recursive -%} + [{{ loop.depth }}:{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[1:1<[2:1][2:2]>][1:2<[2:1][2:2]>][1:3<[2:a]>]" + ) + + def test_looploop(self, env): + tmpl = env.from_string( + """{% for row in table %} + {%- set rowloop = loop -%} + {% for cell in row -%} + [{{ rowloop.index }}|{{ loop.index }}] + {%- endfor %} + {%- endfor %}""" + ) + assert tmpl.render(table=["ab", "cd"]) == "[1|1][1|2][2|1][2|2]" + + def test_reversed_bug(self, env): + tmpl = env.from_string( + "{% for i in items %}{{ i }}" + "{% if not loop.last %}" + ",{% endif %}{% endfor %}" + ) + assert tmpl.render(items=reversed([3, 2, 1])) == "1,2,3" + + def test_loop_errors(self, env): + tmpl = env.from_string( + """{% for item in [1] if loop.index + == 0 %}...{% endfor %}""" + ) + pytest.raises(UndefinedError, tmpl.render) + tmpl = env.from_string( + """{% for item in [] %}...{% else + %}{{ loop }}{% endfor %}""" + ) + assert tmpl.render() == "" + + def test_loop_filter(self, env): + tmpl = env.from_string( + "{% for item in range(10) if item is even %}[{{ item }}]{% endfor %}" + ) + assert tmpl.render() == "[0][2][4][6][8]" + tmpl = env.from_string( + """ + {%- for item in range(10) if item is even %}[{{ + loop.index }}:{{ item }}]{% endfor %}""" + ) + assert tmpl.render() == "[1:0][2:2][3:4][4:6][5:8]" + + def test_loop_unassignable(self, env): + pytest.raises( + TemplateSyntaxError, env.from_string, "{% for loop in seq %}...{% endfor %}" + ) + + def test_scoped_special_var(self, env): + t = env.from_string( + "{% for s in seq %}[{{ loop.first }}{% for c in s %}" + "|{{ loop.first }}{% endfor %}]{% endfor %}" + ) + assert t.render(seq=("ab", "cd")) == "[True|True|False][False|True|False]" + + def test_scoped_loop_var(self, env): + t = env.from_string( + "{% for x in seq %}{{ loop.first }}" + "{% for y in seq %}{% endfor %}{% endfor %}" + ) + assert t.render(seq="ab") == "TrueFalse" + t = env.from_string( + "{% for x in seq %}{% for y in seq %}" + "{{ loop.first }}{% endfor %}{% endfor %}" + ) + assert t.render(seq="ab") == "TrueFalseTrueFalse" + + def test_recursive_empty_loop_iter(self, env): + t = env.from_string( + """ + {%- for item in foo recursive -%}{%- endfor -%} + """ + ) + assert t.render(dict(foo=[])) == "" + + def test_call_in_loop(self, env): + t = env.from_string( + """ + {%- macro do_something() -%} + [{{ caller() }}] + {%- endmacro %} + + {%- for i in [1, 2, 3] %} + {%- call do_something() -%} + {{ i }} + {%- endcall %} + {%- endfor -%} + """ + ) + assert t.render() == "[1][2][3]" + + def test_scoping_bug(self, env): + t = env.from_string( + """ + {%- for item in foo %}...{{ item }}...{% endfor %} + {%- macro item(a) %}...{{ a }}...{% endmacro %} + {{- item(2) -}} + """ + ) + assert t.render(foo=(1,)) == "...1......2..." + + def test_unpacking(self, env): + tmpl = env.from_string( + "{% for a, b, c in [[1, 2, 3]] %}{{ a }}|{{ b }}|{{ c }}{% endfor %}" + ) + assert tmpl.render() == "1|2|3" + + def test_intended_scoping_with_set(self, env): + tmpl = env.from_string( + "{% for item in seq %}{{ x }}{% set x = item %}{{ x }}{% endfor %}" + ) + assert tmpl.render(x=0, seq=[1, 2, 3]) == "010203" + + tmpl = env.from_string( + "{% set x = 9 %}{% for item in seq %}{{ x }}" + "{% set x = item %}{{ x }}{% endfor %}" + ) + assert tmpl.render(x=0, seq=[1, 2, 3]) == "919293" + + +class TestIfCondition(object): + def test_simple(self, env): + tmpl = env.from_string("""{% if true %}...{% endif %}""") + assert tmpl.render() == "..." + + def test_elif(self, env): + tmpl = env.from_string( + """{% if false %}XXX{% elif true + %}...{% else %}XXX{% endif %}""" + ) + assert tmpl.render() == "..." + + def test_elif_deep(self, env): + elifs = "\n".join("{{% elif a == {0} %}}{0}".format(i) for i in range(1, 1000)) + tmpl = env.from_string( + "{{% if a == 0 %}}0{0}{{% else %}}x{{% endif %}}".format(elifs) + ) + for x in (0, 10, 999): + assert tmpl.render(a=x).strip() == str(x) + assert tmpl.render(a=1000).strip() == "x" + + def test_else(self, env): + tmpl = env.from_string("{% if false %}XXX{% else %}...{% endif %}") + assert tmpl.render() == "..." + + def test_empty(self, env): + tmpl = env.from_string("[{% if true %}{% else %}{% endif %}]") + assert tmpl.render() == "[]" + + def test_complete(self, env): + tmpl = env.from_string( + "{% if a %}A{% elif b %}B{% elif c == d %}C{% else %}D{% endif %}" + ) + assert tmpl.render(a=0, b=False, c=42, d=42.0) == "C" + + def test_no_scope(self, env): + tmpl = env.from_string("{% if a %}{% set foo = 1 %}{% endif %}{{ foo }}") + assert tmpl.render(a=True) == "1" + tmpl = env.from_string("{% if true %}{% set foo = 1 %}{% endif %}{{ foo }}") + assert tmpl.render() == "1" + + +class TestMacros(object): + def test_simple(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro say_hello(name) %}Hello {{ name }}!{% endmacro %} +{{ say_hello('Peter') }}""" + ) + assert tmpl.render() == "Hello Peter!" + + def test_scoping(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro level1(data1) %} +{% macro level2(data2) %}{{ data1 }}|{{ data2 }}{% endmacro %} +{{ level2('bar') }}{% endmacro %} +{{ level1('foo') }}""" + ) + assert tmpl.render() == "foo|bar" + + def test_arguments(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro m(a, b, c='c', d='d') %}{{ a }}|{{ b }}|{{ c }}|{{ d }}{% endmacro %} +{{ m() }}|{{ m('a') }}|{{ m('a', 'b') }}|{{ m(1, 2, 3) }}""" + ) + assert tmpl.render() == "||c|d|a||c|d|a|b|c|d|1|2|3|d" + + def test_arguments_defaults_nonsense(self, env_trim): + pytest.raises( + TemplateSyntaxError, + env_trim.from_string, + """\ +{% macro m(a, b=1, c) %}a={{ a }}, b={{ b }}, c={{ c }}{% endmacro %}""", + ) + + def test_caller_defaults_nonsense(self, env_trim): + pytest.raises( + TemplateSyntaxError, + env_trim.from_string, + """\ +{% macro a() %}{{ caller() }}{% endmacro %} +{% call(x, y=1, z) a() %}{% endcall %}""", + ) + + def test_varargs(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro test() %}{{ varargs|join('|') }}{% endmacro %}\ +{{ test(1, 2, 3) }}""" + ) + assert tmpl.render() == "1|2|3" + + def test_simple_call(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro test() %}[[{{ caller() }}]]{% endmacro %}\ +{% call test() %}data{% endcall %}""" + ) + assert tmpl.render() == "[[data]]" + + def test_complex_call(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro test() %}[[{{ caller('data') }}]]{% endmacro %}\ +{% call(data) test() %}{{ data }}{% endcall %}""" + ) + assert tmpl.render() == "[[data]]" + + def test_caller_undefined(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% set caller = 42 %}\ +{% macro test() %}{{ caller is not defined }}{% endmacro %}\ +{{ test() }}""" + ) + assert tmpl.render() == "True" + + def test_include(self, env_trim): + env_trim = Environment( + loader=DictLoader( + {"include": "{% macro test(foo) %}[{{ foo }}]{% endmacro %}"} + ) + ) + tmpl = env_trim.from_string('{% from "include" import test %}{{ test("foo") }}') + assert tmpl.render() == "[foo]" + + def test_macro_api(self, env_trim): + tmpl = env_trim.from_string( + "{% macro foo(a, b) %}{% endmacro %}" + "{% macro bar() %}{{ varargs }}{{ kwargs }}{% endmacro %}" + "{% macro baz() %}{{ caller() }}{% endmacro %}" + ) + assert tmpl.module.foo.arguments == ("a", "b") + assert tmpl.module.foo.name == "foo" + assert not tmpl.module.foo.caller + assert not tmpl.module.foo.catch_kwargs + assert not tmpl.module.foo.catch_varargs + assert tmpl.module.bar.arguments == () + assert not tmpl.module.bar.caller + assert tmpl.module.bar.catch_kwargs + assert tmpl.module.bar.catch_varargs + assert tmpl.module.baz.caller + + def test_callself(self, env_trim): + tmpl = env_trim.from_string( + "{% macro foo(x) %}{{ x }}{% if x > 1 %}|" + "{{ foo(x - 1) }}{% endif %}{% endmacro %}" + "{{ foo(5) }}" + ) + assert tmpl.render() == "5|4|3|2|1" + + def test_macro_defaults_self_ref(self, env): + tmpl = env.from_string( + """ + {%- set x = 42 %} + {%- macro m(a, b=x, x=23) %}{{ a }}|{{ b }}|{{ x }}{% endmacro -%} + """ + ) + assert tmpl.module.m(1) == "1||23" + assert tmpl.module.m(1, 2) == "1|2|23" + assert tmpl.module.m(1, 2, 3) == "1|2|3" + assert tmpl.module.m(1, x=7) == "1|7|7" + + +class TestSet(object): + def test_normal(self, env_trim): + tmpl = env_trim.from_string("{% set foo = 1 %}{{ foo }}") + assert tmpl.render() == "1" + assert tmpl.module.foo == 1 + + def test_block(self, env_trim): + tmpl = env_trim.from_string("{% set foo %}42{% endset %}{{ foo }}") + assert tmpl.render() == "42" + assert tmpl.module.foo == u"42" + + def test_block_escaping(self): + env = Environment(autoescape=True) + tmpl = env.from_string( + "{% set foo %}<em>{{ test }}</em>{% endset %}foo: {{ foo }}" + ) + assert tmpl.render(test="<unsafe>") == "foo: <em><unsafe></em>" + + def test_set_invalid(self, env_trim): + pytest.raises( + TemplateSyntaxError, env_trim.from_string, "{% set foo['bar'] = 1 %}" + ) + tmpl = env_trim.from_string("{% set foo.bar = 1 %}") + exc_info = pytest.raises(TemplateRuntimeError, tmpl.render, foo={}) + assert "non-namespace object" in exc_info.value.message + + def test_namespace_redefined(self, env_trim): + tmpl = env_trim.from_string("{% set ns = namespace() %}{% set ns.bar = 'hi' %}") + exc_info = pytest.raises(TemplateRuntimeError, tmpl.render, namespace=dict) + assert "non-namespace object" in exc_info.value.message + + def test_namespace(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace() %}{% set ns.bar = '42' %}{{ ns.bar }}" + ) + assert tmpl.render() == "42" + + def test_namespace_block(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace() %}{% set ns.bar %}42{% endset %}{{ ns.bar }}" + ) + assert tmpl.render() == "42" + + def test_init_namespace(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace(d, self=37) %}" + "{% set ns.b = 42 %}" + "{{ ns.a }}|{{ ns.self }}|{{ ns.b }}" + ) + assert tmpl.render(d={"a": 13}) == "13|37|42" + + def test_namespace_loop(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace(found=false) %}" + "{% for x in range(4) %}" + "{% if x == v %}" + "{% set ns.found = true %}" + "{% endif %}" + "{% endfor %}" + "{{ ns.found }}" + ) + assert tmpl.render(v=3) == "True" + assert tmpl.render(v=4) == "False" + + def test_namespace_macro(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace() %}" + "{% set ns.a = 13 %}" + "{% macro magic(x) %}" + "{% set x.b = 37 %}" + "{% endmacro %}" + "{{ magic(ns) }}" + "{{ ns.a }}|{{ ns.b }}" + ) + assert tmpl.render() == "13|37" + + def test_block_escaping_filtered(self): + env = Environment(autoescape=True) + tmpl = env.from_string( + "{% set foo | trim %}<em>{{ test }}</em> {% endset %}foo: {{ foo }}" + ) + assert tmpl.render(test="<unsafe>") == "foo: <em><unsafe></em>" + + def test_block_filtered(self, env_trim): + tmpl = env_trim.from_string( + "{% set foo | trim | length | string %} 42 {% endset %}{{ foo }}" + ) + assert tmpl.render() == "2" + assert tmpl.module.foo == u"2" + + def test_block_filtered_set(self, env_trim): + def _myfilter(val, arg): + assert arg == " xxx " + return val + + env_trim.filters["myfilter"] = _myfilter + tmpl = env_trim.from_string( + '{% set a = " xxx " %}' + "{% set foo | myfilter(a) | trim | length | string %}" + ' {% set b = " yy " %} 42 {{ a }}{{ b }} ' + "{% endset %}" + "{{ foo }}" + ) + assert tmpl.render() == "11" + assert tmpl.module.foo == u"11" + + +class TestWith(object): + def test_with(self, env): + tmpl = env.from_string( + """\ + {% with a=42, b=23 -%} + {{ a }} = {{ b }} + {% endwith -%} + {{ a }} = {{ b }}\ + """ + ) + assert [x.strip() for x in tmpl.render(a=1, b=2).splitlines()] == [ + "42 = 23", + "1 = 2", + ] + + def test_with_argument_scoping(self, env): + tmpl = env.from_string( + """\ + {%- with a=1, b=2, c=b, d=e, e=5 -%} + {{ a }}|{{ b }}|{{ c }}|{{ d }}|{{ e }} + {%- endwith -%} + """ + ) + assert tmpl.render(b=3, e=4) == "1|2|3|4|5" diff --git a/contrib/python/Jinja2/py2/tests/test_debug.py b/contrib/python/Jinja2/py2/tests/test_debug.py new file mode 100644 index 0000000000..284b9e91a5 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_debug.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +import pickle +import re +from traceback import format_exception + +import pytest + +from jinja2 import ChoiceLoader +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import TemplateSyntaxError + + +@pytest.fixture +def fs_env(filesystem_loader): + """returns a new environment.""" + return Environment(loader=filesystem_loader) + + +class TestDebug(object): + def assert_traceback_matches(self, callback, expected_tb): + with pytest.raises(Exception) as exc_info: + callback() + + tb = format_exception(exc_info.type, exc_info.value, exc_info.tb) + m = re.search(expected_tb.strip(), "".join(tb)) + assert m is not None, "Traceback did not match:\n\n%s\nexpected:\n%s" % ( + "".join(tb), + expected_tb, + ) + + def test_runtime_error(self, fs_env): + def test(): + tmpl.render(fail=lambda: 1 / 0) + + tmpl = fs_env.get_template("broken.html") + self.assert_traceback_matches( + test, + r""" + File ".*?broken.html", line 2, in (top-level template code|<module>) + \{\{ fail\(\) \}\} + File ".*debug?.pyc?", line \d+, in <lambda> + tmpl\.render\(fail=lambda: 1 / 0\) +ZeroDivisionError: (int(eger)? )?division (or modulo )?by zero +""", + ) + + def test_syntax_error(self, fs_env): + # The trailing .*? is for PyPy 2 and 3, which don't seem to + # clear the exception's original traceback, leaving the syntax + # error in the middle of other compiler frames. + self.assert_traceback_matches( + lambda: fs_env.get_template("syntaxerror.html"), + """(?sm) + File ".*?syntaxerror.html", line 4, in (template|<module>) + \\{% endif %\\}.*? +(jinja2\\.exceptions\\.)?TemplateSyntaxError: Encountered unknown tag 'endif'. Jinja \ +was looking for the following tags: 'endfor' or 'else'. The innermost block that needs \ +to be closed is 'for'. + """, + ) + + def test_regular_syntax_error(self, fs_env): + def test(): + raise TemplateSyntaxError("wtf", 42) + + self.assert_traceback_matches( + test, + r""" + File ".*debug.pyc?", line \d+, in test + raise TemplateSyntaxError\("wtf", 42\) +(jinja2\.exceptions\.)?TemplateSyntaxError: wtf + line 42""", + ) + + def test_pickleable_syntax_error(self, fs_env): + original = TemplateSyntaxError("bad template", 42, "test", "test.txt") + unpickled = pickle.loads(pickle.dumps(original)) + assert str(original) == str(unpickled) + assert original.name == unpickled.name + + def test_include_syntax_error_source(self, filesystem_loader): + e = Environment( + loader=ChoiceLoader( + [ + filesystem_loader, + DictLoader({"inc": "a\n{% include 'syntaxerror.html' %}\nb"}), + ] + ) + ) + t = e.get_template("inc") + + with pytest.raises(TemplateSyntaxError) as exc_info: + t.render() + + assert exc_info.value.source is not None + + def test_local_extraction(self): + from jinja2.debug import get_template_locals + from jinja2.runtime import missing + + locals = get_template_locals( + { + "l_0_foo": 42, + "l_1_foo": 23, + "l_2_foo": 13, + "l_0_bar": 99, + "l_1_bar": missing, + "l_0_baz": missing, + } + ) + assert locals == {"foo": 13, "bar": 99} + + def test_get_corresponding_lineno_traceback(self, fs_env): + tmpl = fs_env.get_template("test.html") + assert tmpl.get_corresponding_lineno(1) == 1 diff --git a/contrib/python/Jinja2/py2/tests/test_ext.py b/contrib/python/Jinja2/py2/tests/test_ext.py new file mode 100644 index 0000000000..8e4b411e1b --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_ext.py @@ -0,0 +1,655 @@ +# -*- coding: utf-8 -*- +import re + +import pytest + +from jinja2 import contextfunction +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import nodes +from jinja2._compat import BytesIO +from jinja2._compat import itervalues +from jinja2._compat import text_type +from jinja2.exceptions import TemplateAssertionError +from jinja2.ext import Extension +from jinja2.lexer import count_newlines +from jinja2.lexer import Token + +importable_object = 23 + +_gettext_re = re.compile(r"_\((.*?)\)", re.DOTALL) + + +i18n_templates = { + "master.html": '<title>{{ page_title|default(_("missing")) }}</title>' + "{% block body %}{% endblock %}", + "child.html": '{% extends "master.html" %}{% block body %}' + "{% trans %}watch out{% endtrans %}{% endblock %}", + "plural.html": "{% trans user_count %}One user online{% pluralize %}" + "{{ user_count }} users online{% endtrans %}", + "plural2.html": "{% trans user_count=get_user_count() %}{{ user_count }}s" + "{% pluralize %}{{ user_count }}p{% endtrans %}", + "stringformat.html": '{{ _("User: %(num)s")|format(num=user_count) }}', +} + +newstyle_i18n_templates = { + "master.html": '<title>{{ page_title|default(_("missing")) }}</title>' + "{% block body %}{% endblock %}", + "child.html": '{% extends "master.html" %}{% block body %}' + "{% trans %}watch out{% endtrans %}{% endblock %}", + "plural.html": "{% trans user_count %}One user online{% pluralize %}" + "{{ user_count }} users online{% endtrans %}", + "stringformat.html": '{{ _("User: %(num)s", num=user_count) }}', + "ngettext.html": '{{ ngettext("%(num)s apple", "%(num)s apples", apples) }}', + "ngettext_long.html": "{% trans num=apples %}{{ num }} apple{% pluralize %}" + "{{ num }} apples{% endtrans %}", + "transvars1.html": "{% trans %}User: {{ num }}{% endtrans %}", + "transvars2.html": "{% trans num=count %}User: {{ num }}{% endtrans %}", + "transvars3.html": "{% trans count=num %}User: {{ count }}{% endtrans %}", + "novars.html": "{% trans %}%(hello)s{% endtrans %}", + "vars.html": "{% trans %}{{ foo }}%(foo)s{% endtrans %}", + "explicitvars.html": '{% trans foo="42" %}%(foo)s{% endtrans %}', +} + + +languages = { + "de": { + "missing": u"fehlend", + "watch out": u"pass auf", + "One user online": u"Ein Benutzer online", + "%(user_count)s users online": u"%(user_count)s Benutzer online", + "User: %(num)s": u"Benutzer: %(num)s", + "User: %(count)s": u"Benutzer: %(count)s", + "%(num)s apple": u"%(num)s Apfel", + "%(num)s apples": u"%(num)s Äpfel", + } +} + + +@contextfunction +def gettext(context, string): + language = context.get("LANGUAGE", "en") + return languages.get(language, {}).get(string, string) + + +@contextfunction +def ngettext(context, s, p, n): + language = context.get("LANGUAGE", "en") + if n != 1: + return languages.get(language, {}).get(p, p) + return languages.get(language, {}).get(s, s) + + +i18n_env = Environment( + loader=DictLoader(i18n_templates), extensions=["jinja2.ext.i18n"] +) +i18n_env.globals.update({"_": gettext, "gettext": gettext, "ngettext": ngettext}) +i18n_env_trimmed = Environment(extensions=["jinja2.ext.i18n"]) +i18n_env_trimmed.policies["ext.i18n.trimmed"] = True +i18n_env_trimmed.globals.update( + {"_": gettext, "gettext": gettext, "ngettext": ngettext} +) + +newstyle_i18n_env = Environment( + loader=DictLoader(newstyle_i18n_templates), extensions=["jinja2.ext.i18n"] +) +newstyle_i18n_env.install_gettext_callables(gettext, ngettext, newstyle=True) + + +class ExampleExtension(Extension): + tags = set(["test"]) + ext_attr = 42 + context_reference_node_cls = nodes.ContextReference + + def parse(self, parser): + return nodes.Output( + [ + self.call_method( + "_dump", + [ + nodes.EnvironmentAttribute("sandboxed"), + self.attr("ext_attr"), + nodes.ImportedName(__name__ + ".importable_object"), + self.context_reference_node_cls(), + ], + ) + ] + ).set_lineno(next(parser.stream).lineno) + + def _dump(self, sandboxed, ext_attr, imported_object, context): + return "%s|%s|%s|%s|%s" % ( + sandboxed, + ext_attr, + imported_object, + context.blocks, + context.get("test_var"), + ) + + +class DerivedExampleExtension(ExampleExtension): + context_reference_node_cls = nodes.DerivedContextReference + + +class PreprocessorExtension(Extension): + def preprocess(self, source, name, filename=None): + return source.replace("[[TEST]]", "({{ foo }})") + + +class StreamFilterExtension(Extension): + def filter_stream(self, stream): + for token in stream: + if token.type == "data": + for t in self.interpolate(token): + yield t + else: + yield token + + def interpolate(self, token): + pos = 0 + end = len(token.value) + lineno = token.lineno + while 1: + match = _gettext_re.search(token.value, pos) + if match is None: + break + value = token.value[pos : match.start()] + if value: + yield Token(lineno, "data", value) + lineno += count_newlines(token.value) + yield Token(lineno, "variable_begin", None) + yield Token(lineno, "name", "gettext") + yield Token(lineno, "lparen", None) + yield Token(lineno, "string", match.group(1)) + yield Token(lineno, "rparen", None) + yield Token(lineno, "variable_end", None) + pos = match.end() + if pos < end: + yield Token(lineno, "data", token.value[pos:]) + + +class TestExtensions(object): + def test_extend_late(self): + env = Environment() + env.add_extension("jinja2.ext.autoescape") + t = env.from_string('{% autoescape true %}{{ "<test>" }}{% endautoescape %}') + assert t.render() == "<test>" + + def test_loop_controls(self): + env = Environment(extensions=["jinja2.ext.loopcontrols"]) + + tmpl = env.from_string( + """ + {%- for item in [1, 2, 3, 4] %} + {%- if item % 2 == 0 %}{% continue %}{% endif -%} + {{ item }} + {%- endfor %}""" + ) + assert tmpl.render() == "13" + + tmpl = env.from_string( + """ + {%- for item in [1, 2, 3, 4] %} + {%- if item > 2 %}{% break %}{% endif -%} + {{ item }} + {%- endfor %}""" + ) + assert tmpl.render() == "12" + + def test_do(self): + env = Environment(extensions=["jinja2.ext.do"]) + tmpl = env.from_string( + """ + {%- set items = [] %} + {%- for char in "foo" %} + {%- do items.append(loop.index0 ~ char) %} + {%- endfor %}{{ items|join(', ') }}""" + ) + assert tmpl.render() == "0f, 1o, 2o" + + def test_extension_nodes(self): + env = Environment(extensions=[ExampleExtension]) + tmpl = env.from_string("{% test %}") + assert tmpl.render() == "False|42|23|{}|None" + + def test_contextreference_node_passes_context(self): + env = Environment(extensions=[ExampleExtension]) + tmpl = env.from_string('{% set test_var="test_content" %}{% test %}') + assert tmpl.render() == "False|42|23|{}|test_content" + + def test_contextreference_node_can_pass_locals(self): + env = Environment(extensions=[DerivedExampleExtension]) + tmpl = env.from_string( + '{% for test_var in ["test_content"] %}{% test %}{% endfor %}' + ) + assert tmpl.render() == "False|42|23|{}|test_content" + + def test_identifier(self): + assert ExampleExtension.identifier == __name__ + ".ExampleExtension" + + def test_rebinding(self): + original = Environment(extensions=[ExampleExtension]) + overlay = original.overlay() + for env in original, overlay: + for ext in itervalues(env.extensions): + assert ext.environment is env + + def test_preprocessor_extension(self): + env = Environment(extensions=[PreprocessorExtension]) + tmpl = env.from_string("{[[TEST]]}") + assert tmpl.render(foo=42) == "{(42)}" + + def test_streamfilter_extension(self): + env = Environment(extensions=[StreamFilterExtension]) + env.globals["gettext"] = lambda x: x.upper() + tmpl = env.from_string("Foo _(bar) Baz") + out = tmpl.render() + assert out == "Foo BAR Baz" + + def test_extension_ordering(self): + class T1(Extension): + priority = 1 + + class T2(Extension): + priority = 2 + + env = Environment(extensions=[T1, T2]) + ext = list(env.iter_extensions()) + assert ext[0].__class__ is T1 + assert ext[1].__class__ is T2 + + def test_debug(self): + env = Environment(extensions=["jinja2.ext.debug"]) + t = env.from_string("Hello\n{% debug %}\nGoodbye") + out = t.render() + + for value in ("context", "cycler", "filters", "abs", "tests", "!="): + assert "'{}'".format(value) in out + + +class TestInternationalization(object): + def test_trans(self): + tmpl = i18n_env.get_template("child.html") + assert tmpl.render(LANGUAGE="de") == "<title>fehlend</title>pass auf" + + def test_trans_plural(self): + tmpl = i18n_env.get_template("plural.html") + assert tmpl.render(LANGUAGE="de", user_count=1) == "Ein Benutzer online" + assert tmpl.render(LANGUAGE="de", user_count=2) == "2 Benutzer online" + + def test_trans_plural_with_functions(self): + tmpl = i18n_env.get_template("plural2.html") + + def get_user_count(): + get_user_count.called += 1 + return 1 + + get_user_count.called = 0 + assert tmpl.render(LANGUAGE="de", get_user_count=get_user_count) == "1s" + assert get_user_count.called == 1 + + def test_complex_plural(self): + tmpl = i18n_env.from_string( + "{% trans foo=42, count=2 %}{{ count }} item{% " + "pluralize count %}{{ count }} items{% endtrans %}" + ) + assert tmpl.render() == "2 items" + pytest.raises( + TemplateAssertionError, + i18n_env.from_string, + "{% trans foo %}...{% pluralize bar %}...{% endtrans %}", + ) + + def test_trans_stringformatting(self): + tmpl = i18n_env.get_template("stringformat.html") + assert tmpl.render(LANGUAGE="de", user_count=5) == "Benutzer: 5" + + def test_trimmed(self): + tmpl = i18n_env.from_string( + "{%- trans trimmed %} hello\n world {% endtrans -%}" + ) + assert tmpl.render() == "hello world" + + def test_trimmed_policy(self): + s = "{%- trans %} hello\n world {% endtrans -%}" + tmpl = i18n_env.from_string(s) + trimmed_tmpl = i18n_env_trimmed.from_string(s) + assert tmpl.render() == " hello\n world " + assert trimmed_tmpl.render() == "hello world" + + def test_trimmed_policy_override(self): + tmpl = i18n_env_trimmed.from_string( + "{%- trans notrimmed %} hello\n world {% endtrans -%}" + ) + assert tmpl.render() == " hello\n world " + + def test_trimmed_vars(self): + tmpl = i18n_env.from_string( + '{%- trans trimmed x="world" %} hello\n {{ x }} {% endtrans -%}' + ) + assert tmpl.render() == "hello world" + + def test_trimmed_varname_trimmed(self): + # unlikely variable name, but when used as a variable + # it should not enable trimming + tmpl = i18n_env.from_string( + "{%- trans trimmed = 'world' %} hello\n {{ trimmed }} {% endtrans -%}" + ) + assert tmpl.render() == " hello\n world " + + def test_extract(self): + from jinja2.ext import babel_extract + + source = BytesIO( + """ + {{ gettext('Hello World') }} + {% trans %}Hello World{% endtrans %} + {% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %} + """.encode( + "ascii" + ) + ) # make python 3 happy + assert list(babel_extract(source, ("gettext", "ngettext", "_"), [], {})) == [ + (2, "gettext", u"Hello World", []), + (3, "gettext", u"Hello World", []), + (4, "ngettext", (u"%(users)s user", u"%(users)s users", None), []), + ] + + def test_extract_trimmed(self): + from jinja2.ext import babel_extract + + source = BytesIO( + """ + {{ gettext(' Hello \n World') }} + {% trans trimmed %} Hello \n World{% endtrans %} + {% trans trimmed %}{{ users }} \n user + {%- pluralize %}{{ users }} \n users{% endtrans %} + """.encode( + "ascii" + ) + ) # make python 3 happy + assert list(babel_extract(source, ("gettext", "ngettext", "_"), [], {})) == [ + (2, "gettext", u" Hello \n World", []), + (4, "gettext", u"Hello World", []), + (6, "ngettext", (u"%(users)s user", u"%(users)s users", None), []), + ] + + def test_extract_trimmed_option(self): + from jinja2.ext import babel_extract + + source = BytesIO( + """ + {{ gettext(' Hello \n World') }} + {% trans %} Hello \n World{% endtrans %} + {% trans %}{{ users }} \n user + {%- pluralize %}{{ users }} \n users{% endtrans %} + """.encode( + "ascii" + ) + ) # make python 3 happy + opts = {"trimmed": "true"} + assert list(babel_extract(source, ("gettext", "ngettext", "_"), [], opts)) == [ + (2, "gettext", u" Hello \n World", []), + (4, "gettext", u"Hello World", []), + (6, "ngettext", (u"%(users)s user", u"%(users)s users", None), []), + ] + + def test_comment_extract(self): + from jinja2.ext import babel_extract + + source = BytesIO( + """ + {# trans first #} + {{ gettext('Hello World') }} + {% trans %}Hello World{% endtrans %}{# trans second #} + {#: third #} + {% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %} + """.encode( + "utf-8" + ) + ) # make python 3 happy + assert list( + babel_extract(source, ("gettext", "ngettext", "_"), ["trans", ":"], {}) + ) == [ + (3, "gettext", u"Hello World", ["first"]), + (4, "gettext", u"Hello World", ["second"]), + (6, "ngettext", (u"%(users)s user", u"%(users)s users", None), ["third"]), + ] + + +class TestScope(object): + def test_basic_scope_behavior(self): + # This is what the old with statement compiled down to + class ScopeExt(Extension): + tags = set(["scope"]) + + def parse(self, parser): + node = nodes.Scope(lineno=next(parser.stream).lineno) + assignments = [] + while parser.stream.current.type != "block_end": + lineno = parser.stream.current.lineno + if assignments: + parser.stream.expect("comma") + target = parser.parse_assign_target() + parser.stream.expect("assign") + expr = parser.parse_expression() + assignments.append(nodes.Assign(target, expr, lineno=lineno)) + node.body = assignments + list( + parser.parse_statements(("name:endscope",), drop_needle=True) + ) + return node + + env = Environment(extensions=[ScopeExt]) + tmpl = env.from_string( + """\ + {%- scope a=1, b=2, c=b, d=e, e=5 -%} + {{ a }}|{{ b }}|{{ c }}|{{ d }}|{{ e }} + {%- endscope -%} + """ + ) + assert tmpl.render(b=3, e=4) == "1|2|2|4|5" + + +class TestNewstyleInternationalization(object): + def test_trans(self): + tmpl = newstyle_i18n_env.get_template("child.html") + assert tmpl.render(LANGUAGE="de") == "<title>fehlend</title>pass auf" + + def test_trans_plural(self): + tmpl = newstyle_i18n_env.get_template("plural.html") + assert tmpl.render(LANGUAGE="de", user_count=1) == "Ein Benutzer online" + assert tmpl.render(LANGUAGE="de", user_count=2) == "2 Benutzer online" + + def test_complex_plural(self): + tmpl = newstyle_i18n_env.from_string( + "{% trans foo=42, count=2 %}{{ count }} item{% " + "pluralize count %}{{ count }} items{% endtrans %}" + ) + assert tmpl.render() == "2 items" + pytest.raises( + TemplateAssertionError, + i18n_env.from_string, + "{% trans foo %}...{% pluralize bar %}...{% endtrans %}", + ) + + def test_trans_stringformatting(self): + tmpl = newstyle_i18n_env.get_template("stringformat.html") + assert tmpl.render(LANGUAGE="de", user_count=5) == "Benutzer: 5" + + def test_newstyle_plural(self): + tmpl = newstyle_i18n_env.get_template("ngettext.html") + assert tmpl.render(LANGUAGE="de", apples=1) == "1 Apfel" + assert tmpl.render(LANGUAGE="de", apples=5) == u"5 Äpfel" + + def test_autoescape_support(self): + env = Environment(extensions=["jinja2.ext.autoescape", "jinja2.ext.i18n"]) + env.install_gettext_callables( + lambda x: u"<strong>Wert: %(name)s</strong>", + lambda s, p, n: s, + newstyle=True, + ) + t = env.from_string( + '{% autoescape ae %}{{ gettext("foo", name=' + '"<test>") }}{% endautoescape %}' + ) + assert t.render(ae=True) == "<strong>Wert: <test></strong>" + assert t.render(ae=False) == "<strong>Wert: <test></strong>" + + def test_autoescape_macros(self): + env = Environment(autoescape=False, extensions=["jinja2.ext.autoescape"]) + template = ( + "{% macro m() %}<html>{% endmacro %}" + "{% autoescape true %}{{ m() }}{% endautoescape %}" + ) + assert env.from_string(template).render() == "<html>" + + def test_num_used_twice(self): + tmpl = newstyle_i18n_env.get_template("ngettext_long.html") + assert tmpl.render(apples=5, LANGUAGE="de") == u"5 Äpfel" + + def test_num_called_num(self): + source = newstyle_i18n_env.compile( + """ + {% trans num=3 %}{{ num }} apple{% pluralize + %}{{ num }} apples{% endtrans %} + """, + raw=True, + ) + # quite hacky, but the only way to properly test that. The idea is + # that the generated code does not pass num twice (although that + # would work) for better performance. This only works on the + # newstyle gettext of course + assert ( + re.search(r"u?'\%\(num\)s apple', u?'\%\(num\)s " r"apples', 3", source) + is not None + ) + + def test_trans_vars(self): + t1 = newstyle_i18n_env.get_template("transvars1.html") + t2 = newstyle_i18n_env.get_template("transvars2.html") + t3 = newstyle_i18n_env.get_template("transvars3.html") + assert t1.render(num=1, LANGUAGE="de") == "Benutzer: 1" + assert t2.render(count=23, LANGUAGE="de") == "Benutzer: 23" + assert t3.render(num=42, LANGUAGE="de") == "Benutzer: 42" + + def test_novars_vars_escaping(self): + t = newstyle_i18n_env.get_template("novars.html") + assert t.render() == "%(hello)s" + t = newstyle_i18n_env.get_template("vars.html") + assert t.render(foo="42") == "42%(foo)s" + t = newstyle_i18n_env.get_template("explicitvars.html") + assert t.render() == "%(foo)s" + + +class TestAutoEscape(object): + def test_scoped_setting(self): + env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True) + tmpl = env.from_string( + """ + {{ "<HelloWorld>" }} + {% autoescape false %} + {{ "<HelloWorld>" }} + {% endautoescape %} + {{ "<HelloWorld>" }} + """ + ) + assert tmpl.render().split() == [ + u"<HelloWorld>", + u"<HelloWorld>", + u"<HelloWorld>", + ] + + env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=False) + tmpl = env.from_string( + """ + {{ "<HelloWorld>" }} + {% autoescape true %} + {{ "<HelloWorld>" }} + {% endautoescape %} + {{ "<HelloWorld>" }} + """ + ) + assert tmpl.render().split() == [ + u"<HelloWorld>", + u"<HelloWorld>", + u"<HelloWorld>", + ] + + def test_nonvolatile(self): + env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True) + tmpl = env.from_string('{{ {"foo": "<test>"}|xmlattr|escape }}') + assert tmpl.render() == ' foo="<test>"' + tmpl = env.from_string( + '{% autoescape false %}{{ {"foo": "<test>"}' + "|xmlattr|escape }}{% endautoescape %}" + ) + assert tmpl.render() == " foo="&lt;test&gt;"" + + def test_volatile(self): + env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True) + tmpl = env.from_string( + '{% autoescape foo %}{{ {"foo": "<test>"}' + "|xmlattr|escape }}{% endautoescape %}" + ) + assert tmpl.render(foo=False) == " foo="&lt;test&gt;"" + assert tmpl.render(foo=True) == ' foo="<test>"' + + def test_scoping(self): + env = Environment(extensions=["jinja2.ext.autoescape"]) + tmpl = env.from_string( + '{% autoescape true %}{% set x = "<x>" %}{{ x }}' + '{% endautoescape %}{{ x }}{{ "<y>" }}' + ) + assert tmpl.render(x=1) == "<x>1<y>" + + def test_volatile_scoping(self): + env = Environment(extensions=["jinja2.ext.autoescape"]) + tmplsource = """ + {% autoescape val %} + {% macro foo(x) %} + [{{ x }}] + {% endmacro %} + {{ foo().__class__.__name__ }} + {% endautoescape %} + {{ '<testing>' }} + """ + tmpl = env.from_string(tmplsource) + assert tmpl.render(val=True).split()[0] == "Markup" + assert tmpl.render(val=False).split()[0] == text_type.__name__ + + # looking at the source we should see <testing> there in raw + # (and then escaped as well) + env = Environment(extensions=["jinja2.ext.autoescape"]) + pysource = env.compile(tmplsource, raw=True) + assert "<testing>\\n" in pysource + + env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True) + pysource = env.compile(tmplsource, raw=True) + assert "<testing>\\n" in pysource + + def test_overlay_scopes(self): + class MagicScopeExtension(Extension): + tags = set(["overlay"]) + + def parse(self, parser): + node = nodes.OverlayScope(lineno=next(parser.stream).lineno) + node.body = list( + parser.parse_statements(("name:endoverlay",), drop_needle=True) + ) + node.context = self.call_method("get_scope") + return node + + def get_scope(self): + return {"x": [1, 2, 3]} + + env = Environment(extensions=[MagicScopeExtension]) + + tmpl = env.from_string( + """ + {{- x }}|{% set z = 99 %} + {%- overlay %} + {{- y }}|{{ z }}|{% for item in x %}[{{ item }}]{% endfor %} + {%- endoverlay %}| + {{- x -}} + """ + ) + assert tmpl.render(x=42, y=23) == "42|23|99|[1][2][3]|42" diff --git a/contrib/python/Jinja2/py2/tests/test_features.py b/contrib/python/Jinja2/py2/tests/test_features.py new file mode 100644 index 0000000000..34b6f200de --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_features.py @@ -0,0 +1,42 @@ +import sys + +import pytest + +from jinja2 import contextfilter +from jinja2 import Environment +from jinja2 import Template +from jinja2._compat import text_type + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason="Requires 3.5 or later") +def test_generator_stop(): + class X(object): + def __getattr__(self, name): + raise StopIteration() + + t = Template("a{{ bad.bar() }}b") + with pytest.raises(RuntimeError): + t.render(bad=X()) + + +@pytest.mark.skipif(sys.version_info[0] > 2, reason="Feature only supported on 2.x") +def test_ascii_str(): + @contextfilter + def assert_func(context, value): + assert type(value) is context["expected_type"] + + env = Environment() + env.filters["assert"] = assert_func + + env.policies["compiler.ascii_str"] = False + t = env.from_string('{{ "foo"|assert }}') + t.render(expected_type=text_type) + + env.policies["compiler.ascii_str"] = True + t = env.from_string('{{ "foo"|assert }}') + t.render(expected_type=str) + + for val in True, False: + env.policies["compiler.ascii_str"] = val + t = env.from_string(u'{{ "\N{SNOWMAN}"|assert }}') + t.render(expected_type=text_type) diff --git a/contrib/python/Jinja2/py2/tests/test_filters.py b/contrib/python/Jinja2/py2/tests/test_filters.py new file mode 100644 index 0000000000..388c346212 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_filters.py @@ -0,0 +1,751 @@ +# -*- coding: utf-8 -*- +import random +from collections import namedtuple + +import pytest + +from jinja2 import Environment +from jinja2 import Markup +from jinja2 import StrictUndefined +from jinja2 import UndefinedError +from jinja2._compat import implements_to_string +from jinja2._compat import text_type + + +@implements_to_string +class Magic(object): + def __init__(self, value): + self.value = value + + def __str__(self): + return text_type(self.value) + + +@implements_to_string +class Magic2(object): + def __init__(self, value1, value2): + self.value1 = value1 + self.value2 = value2 + + def __str__(self): + return u"(%s,%s)" % (text_type(self.value1), text_type(self.value2)) + + +class TestFilter(object): + def test_filter_calling(self, env): + rv = env.call_filter("sum", [1, 2, 3]) + assert rv == 6 + + def test_capitalize(self, env): + tmpl = env.from_string('{{ "foo bar"|capitalize }}') + assert tmpl.render() == "Foo bar" + + def test_center(self, env): + tmpl = env.from_string('{{ "foo"|center(9) }}') + assert tmpl.render() == " foo " + + def test_default(self, env): + tmpl = env.from_string( + "{{ missing|default('no') }}|{{ false|default('no') }}|" + "{{ false|default('no', true) }}|{{ given|default('no') }}" + ) + assert tmpl.render(given="yes") == "no|False|no|yes" + + @pytest.mark.parametrize( + "args,expect", + ( + ("", "[('aa', 0), ('AB', 3), ('b', 1), ('c', 2)]"), + ("true", "[('AB', 3), ('aa', 0), ('b', 1), ('c', 2)]"), + ('by="value"', "[('aa', 0), ('b', 1), ('c', 2), ('AB', 3)]"), + ("reverse=true", "[('c', 2), ('b', 1), ('AB', 3), ('aa', 0)]"), + ), + ) + def test_dictsort(self, env, args, expect): + t = env.from_string("{{{{ foo|dictsort({args}) }}}}".format(args=args)) + out = t.render(foo={"aa": 0, "b": 1, "c": 2, "AB": 3}) + assert out == expect + + def test_batch(self, env): + tmpl = env.from_string("{{ foo|batch(3)|list }}|{{ foo|batch(3, 'X')|list }}") + out = tmpl.render(foo=list(range(10))) + assert out == ( + "[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]|" + "[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 'X', 'X']]" + ) + + def test_slice(self, env): + tmpl = env.from_string("{{ foo|slice(3)|list }}|{{ foo|slice(3, 'X')|list }}") + out = tmpl.render(foo=list(range(10))) + assert out == ( + "[[0, 1, 2, 3], [4, 5, 6], [7, 8, 9]]|" + "[[0, 1, 2, 3], [4, 5, 6, 'X'], [7, 8, 9, 'X']]" + ) + + def test_escape(self, env): + tmpl = env.from_string("""{{ '<">&'|escape }}""") + out = tmpl.render() + assert out == "<">&" + + @pytest.mark.parametrize( + ("chars", "expect"), [(None, "..stays.."), (".", " ..stays"), (" .", "stays")] + ) + def test_trim(self, env, chars, expect): + tmpl = env.from_string("{{ foo|trim(chars) }}") + out = tmpl.render(foo=" ..stays..", chars=chars) + assert out == expect + + def test_striptags(self, env): + tmpl = env.from_string("""{{ foo|striptags }}""") + out = tmpl.render( + foo=' <p>just a small \n <a href="#">' + "example</a> link</p>\n<p>to a webpage</p> " + "<!-- <p>and some commented stuff</p> -->" + ) + assert out == "just a small example link to a webpage" + + def test_filesizeformat(self, env): + tmpl = env.from_string( + "{{ 100|filesizeformat }}|" + "{{ 1000|filesizeformat }}|" + "{{ 1000000|filesizeformat }}|" + "{{ 1000000000|filesizeformat }}|" + "{{ 1000000000000|filesizeformat }}|" + "{{ 100|filesizeformat(true) }}|" + "{{ 1000|filesizeformat(true) }}|" + "{{ 1000000|filesizeformat(true) }}|" + "{{ 1000000000|filesizeformat(true) }}|" + "{{ 1000000000000|filesizeformat(true) }}" + ) + out = tmpl.render() + assert out == ( + "100 Bytes|1.0 kB|1.0 MB|1.0 GB|1.0 TB|100 Bytes|" + "1000 Bytes|976.6 KiB|953.7 MiB|931.3 GiB" + ) + + def test_filesizeformat_issue59(self, env): + tmpl = env.from_string( + "{{ 300|filesizeformat }}|" + "{{ 3000|filesizeformat }}|" + "{{ 3000000|filesizeformat }}|" + "{{ 3000000000|filesizeformat }}|" + "{{ 3000000000000|filesizeformat }}|" + "{{ 300|filesizeformat(true) }}|" + "{{ 3000|filesizeformat(true) }}|" + "{{ 3000000|filesizeformat(true) }}" + ) + out = tmpl.render() + assert out == ( + "300 Bytes|3.0 kB|3.0 MB|3.0 GB|3.0 TB|300 Bytes|2.9 KiB|2.9 MiB" + ) + + def test_first(self, env): + tmpl = env.from_string("{{ foo|first }}") + out = tmpl.render(foo=list(range(10))) + assert out == "0" + + @pytest.mark.parametrize( + ("value", "expect"), (("42", "42.0"), ("abc", "0.0"), ("32.32", "32.32"),) + ) + def test_float(self, env, value, expect): + t = env.from_string("{{ '%s'|float }}" % value) + assert t.render() == expect + + def test_float_default(self, env): + t = env.from_string("{{ value|float(default=1.0) }}") + assert t.render(value="abc") == "1.0" + + def test_format(self, env): + tmpl = env.from_string("""{{ "%s|%s"|format("a", "b") }}""") + out = tmpl.render() + assert out == "a|b" + + @staticmethod + def _test_indent_multiline_template(env, markup=False): + text = "\n".join(["", "foo bar", '"baz"', ""]) + if markup: + text = Markup(text) + t = env.from_string("{{ foo|indent(2, false, false) }}") + assert t.render(foo=text) == '\n foo bar\n "baz"\n' + t = env.from_string("{{ foo|indent(2, false, true) }}") + assert t.render(foo=text) == '\n foo bar\n "baz"\n ' + t = env.from_string("{{ foo|indent(2, true, false) }}") + assert t.render(foo=text) == ' \n foo bar\n "baz"\n' + t = env.from_string("{{ foo|indent(2, true, true) }}") + assert t.render(foo=text) == ' \n foo bar\n "baz"\n ' + + def test_indent(self, env): + self._test_indent_multiline_template(env) + t = env.from_string('{{ "jinja"|indent }}') + assert t.render() == "jinja" + t = env.from_string('{{ "jinja"|indent(first=true) }}') + assert t.render() == " jinja" + t = env.from_string('{{ "jinja"|indent(blank=true) }}') + assert t.render() == "jinja" + + def test_indent_markup_input(self, env): + """ + Tests cases where the filter input is a Markup type + """ + self._test_indent_multiline_template(env, markup=True) + + @pytest.mark.parametrize( + ("value", "expect"), + ( + ("42", "42"), + ("abc", "0"), + ("32.32", "32"), + ("12345678901234567890", "12345678901234567890"), + ), + ) + def test_int(self, env, value, expect): + t = env.from_string("{{ '%s'|int }}" % value) + assert t.render() == expect + + @pytest.mark.parametrize( + ("value", "base", "expect"), + (("0x4d32", 16, "19762"), ("011", 8, "9"), ("0x33Z", 16, "0"),), + ) + def test_int_base(self, env, value, base, expect): + t = env.from_string("{{ '%s'|int(base=%d) }}" % (value, base)) + assert t.render() == expect + + def test_int_default(self, env): + t = env.from_string("{{ value|int(default=1) }}") + assert t.render(value="abc") == "1" + + def test_int_special_method(self, env): + class IntIsh(object): + def __int__(self): + return 42 + + t = env.from_string("{{ value|int }}") + assert t.render(value=IntIsh()) == "42" + + def test_join(self, env): + tmpl = env.from_string('{{ [1, 2, 3]|join("|") }}') + out = tmpl.render() + assert out == "1|2|3" + + env2 = Environment(autoescape=True) + tmpl = env2.from_string('{{ ["<foo>", "<span>foo</span>"|safe]|join }}') + assert tmpl.render() == "<foo><span>foo</span>" + + def test_join_attribute(self, env): + User = namedtuple("User", "username") + tmpl = env.from_string("""{{ users|join(', ', 'username') }}""") + assert tmpl.render(users=map(User, ["foo", "bar"])) == "foo, bar" + + def test_last(self, env): + tmpl = env.from_string("""{{ foo|last }}""") + out = tmpl.render(foo=list(range(10))) + assert out == "9" + + def test_length(self, env): + tmpl = env.from_string("""{{ "hello world"|length }}""") + out = tmpl.render() + assert out == "11" + + def test_lower(self, env): + tmpl = env.from_string("""{{ "FOO"|lower }}""") + out = tmpl.render() + assert out == "foo" + + def test_pprint(self, env): + from pprint import pformat + + tmpl = env.from_string("""{{ data|pprint }}""") + data = list(range(1000)) + assert tmpl.render(data=data) == pformat(data) + + def test_random(self, env, request): + # restore the random state when the test ends + state = random.getstate() + request.addfinalizer(lambda: random.setstate(state)) + # generate the random values from a known seed + random.seed("jinja") + expected = [random.choice("1234567890") for _ in range(10)] + + # check that the random sequence is generated again by a template + # ensures that filter result is not constant folded + random.seed("jinja") + t = env.from_string('{{ "1234567890"|random }}') + + for value in expected: + assert t.render() == value + + def test_reverse(self, env): + tmpl = env.from_string( + "{{ 'foobar'|reverse|join }}|{{ [1, 2, 3]|reverse|list }}" + ) + assert tmpl.render() == "raboof|[3, 2, 1]" + + def test_string(self, env): + x = [1, 2, 3, 4, 5] + tmpl = env.from_string("""{{ obj|string }}""") + assert tmpl.render(obj=x) == text_type(x) + + def test_title(self, env): + tmpl = env.from_string("""{{ "foo bar"|title }}""") + assert tmpl.render() == "Foo Bar" + tmpl = env.from_string("""{{ "foo's bar"|title }}""") + assert tmpl.render() == "Foo's Bar" + tmpl = env.from_string("""{{ "foo bar"|title }}""") + assert tmpl.render() == "Foo Bar" + tmpl = env.from_string("""{{ "f bar f"|title }}""") + assert tmpl.render() == "F Bar F" + tmpl = env.from_string("""{{ "foo-bar"|title }}""") + assert tmpl.render() == "Foo-Bar" + tmpl = env.from_string("""{{ "foo\tbar"|title }}""") + assert tmpl.render() == "Foo\tBar" + tmpl = env.from_string("""{{ "FOO\tBAR"|title }}""") + assert tmpl.render() == "Foo\tBar" + tmpl = env.from_string("""{{ "foo (bar)"|title }}""") + assert tmpl.render() == "Foo (Bar)" + tmpl = env.from_string("""{{ "foo {bar}"|title }}""") + assert tmpl.render() == "Foo {Bar}" + tmpl = env.from_string("""{{ "foo [bar]"|title }}""") + assert tmpl.render() == "Foo [Bar]" + tmpl = env.from_string("""{{ "foo <bar>"|title }}""") + assert tmpl.render() == "Foo <Bar>" + + class Foo: + def __str__(self): + return "foo-bar" + + tmpl = env.from_string("""{{ data|title }}""") + out = tmpl.render(data=Foo()) + assert out == "Foo-Bar" + + def test_truncate(self, env): + tmpl = env.from_string( + '{{ data|truncate(15, true, ">>>") }}|' + '{{ data|truncate(15, false, ">>>") }}|' + "{{ smalldata|truncate(15) }}" + ) + out = tmpl.render(data="foobar baz bar" * 1000, smalldata="foobar baz bar") + msg = "Current output: %s" % out + assert out == "foobar baz b>>>|foobar baz>>>|foobar baz bar", msg + + def test_truncate_very_short(self, env): + tmpl = env.from_string( + '{{ "foo bar baz"|truncate(9) }}|{{ "foo bar baz"|truncate(9, true) }}' + ) + out = tmpl.render() + assert out == "foo bar baz|foo bar baz", out + + def test_truncate_end_length(self, env): + tmpl = env.from_string('{{ "Joel is a slug"|truncate(7, true) }}') + out = tmpl.render() + assert out == "Joel...", "Current output: %s" % out + + def test_upper(self, env): + tmpl = env.from_string('{{ "foo"|upper }}') + assert tmpl.render() == "FOO" + + def test_urlize(self, env): + tmpl = env.from_string('{{ "foo http://www.example.com/ bar"|urlize }}') + assert tmpl.render() == ( + 'foo <a href="http://www.example.com/" rel="noopener">' + "http://www.example.com/</a> bar" + ) + + def test_urlize_rel_policy(self): + env = Environment() + env.policies["urlize.rel"] = None + tmpl = env.from_string('{{ "foo http://www.example.com/ bar"|urlize }}') + assert tmpl.render() == ( + 'foo <a href="http://www.example.com/">http://www.example.com/</a> bar' + ) + + def test_urlize_target_parameter(self, env): + tmpl = env.from_string( + '{{ "foo http://www.example.com/ bar"|urlize(target="_blank") }}' + ) + assert ( + tmpl.render() + == 'foo <a href="http://www.example.com/" rel="noopener" target="_blank">' + "http://www.example.com/</a> bar" + ) + + def test_wordcount(self, env): + tmpl = env.from_string('{{ "foo bar baz"|wordcount }}') + assert tmpl.render() == "3" + + strict_env = Environment(undefined=StrictUndefined) + t = strict_env.from_string("{{ s|wordcount }}") + with pytest.raises(UndefinedError): + t.render() + + def test_block(self, env): + tmpl = env.from_string("{% filter lower|escape %}<HEHE>{% endfilter %}") + assert tmpl.render() == "<hehe>" + + def test_chaining(self, env): + tmpl = env.from_string("""{{ ['<foo>', '<bar>']|first|upper|escape }}""") + assert tmpl.render() == "<FOO>" + + def test_sum(self, env): + tmpl = env.from_string("""{{ [1, 2, 3, 4, 5, 6]|sum }}""") + assert tmpl.render() == "21" + + def test_sum_attributes(self, env): + tmpl = env.from_string("""{{ values|sum('value') }}""") + assert tmpl.render(values=[{"value": 23}, {"value": 1}, {"value": 18}]) == "42" + + def test_sum_attributes_nested(self, env): + tmpl = env.from_string("""{{ values|sum('real.value') }}""") + assert ( + tmpl.render( + values=[ + {"real": {"value": 23}}, + {"real": {"value": 1}}, + {"real": {"value": 18}}, + ] + ) + == "42" + ) + + def test_sum_attributes_tuple(self, env): + tmpl = env.from_string("""{{ values.items()|sum('1') }}""") + assert tmpl.render(values={"foo": 23, "bar": 1, "baz": 18}) == "42" + + def test_abs(self, env): + tmpl = env.from_string("""{{ -1|abs }}|{{ 1|abs }}""") + assert tmpl.render() == "1|1", tmpl.render() + + def test_round_positive(self, env): + tmpl = env.from_string( + "{{ 2.7|round }}|{{ 2.1|round }}|" + "{{ 2.1234|round(3, 'floor') }}|" + "{{ 2.1|round(0, 'ceil') }}" + ) + assert tmpl.render() == "3.0|2.0|2.123|3.0", tmpl.render() + + def test_round_negative(self, env): + tmpl = env.from_string( + "{{ 21.3|round(-1)}}|" + "{{ 21.3|round(-1, 'ceil')}}|" + "{{ 21.3|round(-1, 'floor')}}" + ) + assert tmpl.render() == "20.0|30.0|20.0", tmpl.render() + + def test_xmlattr(self, env): + tmpl = env.from_string( + "{{ {'foo': 42, 'bar': 23, 'fish': none, " + "'spam': missing, 'blub:blub': '<?>'}|xmlattr }}" + ) + out = tmpl.render().split() + assert len(out) == 3 + assert 'foo="42"' in out + assert 'bar="23"' in out + assert 'blub:blub="<?>"' in out + + def test_sort1(self, env): + tmpl = env.from_string("{{ [2, 3, 1]|sort }}|{{ [2, 3, 1]|sort(true) }}") + assert tmpl.render() == "[1, 2, 3]|[3, 2, 1]" + + def test_sort2(self, env): + tmpl = env.from_string('{{ "".join(["c", "A", "b", "D"]|sort) }}') + assert tmpl.render() == "AbcD" + + def test_sort3(self, env): + tmpl = env.from_string("""{{ ['foo', 'Bar', 'blah']|sort }}""") + assert tmpl.render() == "['Bar', 'blah', 'foo']" + + def test_sort4(self, env): + tmpl = env.from_string("""{{ items|sort(attribute='value')|join }}""") + assert tmpl.render(items=map(Magic, [3, 2, 4, 1])) == "1234" + + def test_sort5(self, env): + tmpl = env.from_string("""{{ items|sort(attribute='value.0')|join }}""") + assert tmpl.render(items=map(Magic, [[3], [2], [4], [1]])) == "[1][2][3][4]" + + def test_sort6(self, env): + tmpl = env.from_string("""{{ items|sort(attribute='value1,value2')|join }}""") + assert ( + tmpl.render( + items=map( + lambda x: Magic2(x[0], x[1]), [(3, 1), (2, 2), (2, 1), (2, 5)] + ) + ) + == "(2,1)(2,2)(2,5)(3,1)" + ) + + def test_sort7(self, env): + tmpl = env.from_string("""{{ items|sort(attribute='value2,value1')|join }}""") + assert ( + tmpl.render( + items=map( + lambda x: Magic2(x[0], x[1]), [(3, 1), (2, 2), (2, 1), (2, 5)] + ) + ) + == "(2,1)(3,1)(2,2)(2,5)" + ) + + def test_sort8(self, env): + tmpl = env.from_string( + """{{ items|sort(attribute='value1.0,value2.0')|join }}""" + ) + assert ( + tmpl.render( + items=map( + lambda x: Magic2(x[0], x[1]), + [([3], [1]), ([2], [2]), ([2], [1]), ([2], [5])], + ) + ) + == "([2],[1])([2],[2])([2],[5])([3],[1])" + ) + + def test_unique(self, env): + t = env.from_string('{{ "".join(["b", "A", "a", "b"]|unique) }}') + assert t.render() == "bA" + + def test_unique_case_sensitive(self, env): + t = env.from_string('{{ "".join(["b", "A", "a", "b"]|unique(true)) }}') + assert t.render() == "bAa" + + def test_unique_attribute(self, env): + t = env.from_string("{{ items|unique(attribute='value')|join }}") + assert t.render(items=map(Magic, [3, 2, 4, 1, 2])) == "3241" + + @pytest.mark.parametrize( + "source,expect", + ( + ('{{ ["a", "B"]|min }}', "a"), + ('{{ ["a", "B"]|min(case_sensitive=true) }}', "B"), + ("{{ []|min }}", ""), + ('{{ ["a", "B"]|max }}', "B"), + ('{{ ["a", "B"]|max(case_sensitive=true) }}', "a"), + ("{{ []|max }}", ""), + ), + ) + def test_min_max(self, env, source, expect): + t = env.from_string(source) + assert t.render() == expect + + @pytest.mark.parametrize("name,expect", (("min", "1"), ("max", "9"),)) + def test_min_max_attribute(self, env, name, expect): + t = env.from_string("{{ items|" + name + '(attribute="value") }}') + assert t.render(items=map(Magic, [5, 1, 9])) == expect + + def test_groupby(self, env): + tmpl = env.from_string( + """ + {%- for grouper, list in [{'foo': 1, 'bar': 2}, + {'foo': 2, 'bar': 3}, + {'foo': 1, 'bar': 1}, + {'foo': 3, 'bar': 4}]|groupby('foo') -%} + {{ grouper }}{% for x in list %}: {{ x.foo }}, {{ x.bar }}{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render().split("|") == ["1: 1, 2: 1, 1", "2: 2, 3", "3: 3, 4", ""] + + def test_groupby_tuple_index(self, env): + tmpl = env.from_string( + """ + {%- for grouper, list in [('a', 1), ('a', 2), ('b', 1)]|groupby(0) -%} + {{ grouper }}{% for x in list %}:{{ x.1 }}{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render() == "a:1:2|b:1|" + + def test_groupby_multidot(self, env): + Date = namedtuple("Date", "day,month,year") + Article = namedtuple("Article", "title,date") + articles = [ + Article("aha", Date(1, 1, 1970)), + Article("interesting", Date(2, 1, 1970)), + Article("really?", Date(3, 1, 1970)), + Article("totally not", Date(1, 1, 1971)), + ] + tmpl = env.from_string( + """ + {%- for year, list in articles|groupby('date.year') -%} + {{ year }}{% for x in list %}[{{ x.title }}]{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render(articles=articles).split("|") == [ + "1970[aha][interesting][really?]", + "1971[totally not]", + "", + ] + + def test_filtertag(self, env): + tmpl = env.from_string( + "{% filter upper|replace('FOO', 'foo') %}foobar{% endfilter %}" + ) + assert tmpl.render() == "fooBAR" + + def test_replace(self, env): + env = Environment() + tmpl = env.from_string('{{ string|replace("o", 42) }}') + assert tmpl.render(string="<foo>") == "<f4242>" + env = Environment(autoescape=True) + tmpl = env.from_string('{{ string|replace("o", 42) }}') + assert tmpl.render(string="<foo>") == "<f4242>" + tmpl = env.from_string('{{ string|replace("<", 42) }}') + assert tmpl.render(string="<foo>") == "42foo>" + tmpl = env.from_string('{{ string|replace("o", ">x<") }}') + assert tmpl.render(string=Markup("foo")) == "f>x<>x<" + + def test_forceescape(self, env): + tmpl = env.from_string("{{ x|forceescape }}") + assert tmpl.render(x=Markup("<div />")) == u"<div />" + + def test_safe(self, env): + env = Environment(autoescape=True) + tmpl = env.from_string('{{ "<div>foo</div>"|safe }}') + assert tmpl.render() == "<div>foo</div>" + tmpl = env.from_string('{{ "<div>foo</div>" }}') + assert tmpl.render() == "<div>foo</div>" + + @pytest.mark.parametrize( + ("value", "expect"), + [ + ("Hello, world!", "Hello%2C%20world%21"), + (u"Hello, world\u203d", "Hello%2C%20world%E2%80%BD"), + ({"f": 1}, "f=1"), + ([("f", 1), ("z", 2)], "f=1&z=2"), + ({u"\u203d": 1}, "%E2%80%BD=1"), + ({0: 1}, "0=1"), + ([("a b/c", "a b/c")], "a+b%2Fc=a+b%2Fc"), + ("a b/c", "a%20b/c"), + ], + ) + def test_urlencode(self, value, expect): + e = Environment(autoescape=True) + t = e.from_string("{{ value|urlencode }}") + assert t.render(value=value) == expect + + def test_simple_map(self, env): + env = Environment() + tmpl = env.from_string('{{ ["1", "2", "3"]|map("int")|sum }}') + assert tmpl.render() == "6" + + def test_map_sum(self, env): + tmpl = env.from_string('{{ [[1,2], [3], [4,5,6]]|map("sum")|list }}') + assert tmpl.render() == "[3, 3, 15]" + + def test_attribute_map(self, env): + User = namedtuple("User", "name") + env = Environment() + users = [ + User("john"), + User("jane"), + User("mike"), + ] + tmpl = env.from_string('{{ users|map(attribute="name")|join("|") }}') + assert tmpl.render(users=users) == "john|jane|mike" + + def test_empty_map(self, env): + env = Environment() + tmpl = env.from_string('{{ none|map("upper")|list }}') + assert tmpl.render() == "[]" + + def test_map_default(self, env): + Fullname = namedtuple("Fullname", "firstname,lastname") + Firstname = namedtuple("Firstname", "firstname") + env = Environment() + tmpl = env.from_string( + '{{ users|map(attribute="lastname", default="smith")|join(", ") }}' + ) + users = [ + Fullname("john", "lennon"), + Fullname("jane", "edwards"), + Fullname("jon", None), + Firstname("mike"), + ] + assert tmpl.render(users=users) == "lennon, edwards, None, smith" + + def test_simple_select(self, env): + env = Environment() + tmpl = env.from_string('{{ [1, 2, 3, 4, 5]|select("odd")|join("|") }}') + assert tmpl.render() == "1|3|5" + + def test_bool_select(self, env): + env = Environment() + tmpl = env.from_string('{{ [none, false, 0, 1, 2, 3, 4, 5]|select|join("|") }}') + assert tmpl.render() == "1|2|3|4|5" + + def test_simple_reject(self, env): + env = Environment() + tmpl = env.from_string('{{ [1, 2, 3, 4, 5]|reject("odd")|join("|") }}') + assert tmpl.render() == "2|4" + + def test_bool_reject(self, env): + env = Environment() + tmpl = env.from_string('{{ [none, false, 0, 1, 2, 3, 4, 5]|reject|join("|") }}') + assert tmpl.render() == "None|False|0" + + def test_simple_select_attr(self, env): + User = namedtuple("User", "name,is_active") + env = Environment() + users = [ + User("john", True), + User("jane", True), + User("mike", False), + ] + tmpl = env.from_string( + '{{ users|selectattr("is_active")|map(attribute="name")|join("|") }}' + ) + assert tmpl.render(users=users) == "john|jane" + + def test_simple_reject_attr(self, env): + User = namedtuple("User", "name,is_active") + env = Environment() + users = [ + User("john", True), + User("jane", True), + User("mike", False), + ] + tmpl = env.from_string( + '{{ users|rejectattr("is_active")|map(attribute="name")|join("|") }}' + ) + assert tmpl.render(users=users) == "mike" + + def test_func_select_attr(self, env): + User = namedtuple("User", "id,name") + env = Environment() + users = [ + User(1, "john"), + User(2, "jane"), + User(3, "mike"), + ] + tmpl = env.from_string( + '{{ users|selectattr("id", "odd")|map(attribute="name")|join("|") }}' + ) + assert tmpl.render(users=users) == "john|mike" + + def test_func_reject_attr(self, env): + User = namedtuple("User", "id,name") + env = Environment() + users = [ + User(1, "john"), + User(2, "jane"), + User(3, "mike"), + ] + tmpl = env.from_string( + '{{ users|rejectattr("id", "odd")|map(attribute="name")|join("|") }}' + ) + assert tmpl.render(users=users) == "jane" + + def test_json_dump(self): + env = Environment(autoescape=True) + t = env.from_string("{{ x|tojson }}") + assert t.render(x={"foo": "bar"}) == '{"foo": "bar"}' + assert t.render(x="\"ba&r'") == r'"\"ba\u0026r\u0027"' + assert t.render(x="<bar>") == r'"\u003cbar\u003e"' + + def my_dumps(value, **options): + assert options == {"foo": "bar"} + return "42" + + env.policies["json.dumps_function"] = my_dumps + env.policies["json.dumps_kwargs"] = {"foo": "bar"} + assert t.render(x=23) == "42" + + def test_wordwrap(self, env): + env.newline_sequence = "\n" + t = env.from_string("{{ s|wordwrap(20) }}") + result = t.render(s="Hello!\nThis is Jinja saying something.") + assert result == "Hello!\nThis is Jinja saying\nsomething." diff --git a/contrib/python/Jinja2/py2/tests/test_idtracking.py b/contrib/python/Jinja2/py2/tests/test_idtracking.py new file mode 100644 index 0000000000..4c79ee6c47 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_idtracking.py @@ -0,0 +1,289 @@ +from jinja2 import nodes +from jinja2.idtracking import symbols_for_node + + +def test_basics(): + for_loop = nodes.For( + nodes.Name("foo", "store"), + nodes.Name("seq", "load"), + [nodes.Output([nodes.Name("foo", "load")])], + [], + None, + False, + ) + tmpl = nodes.Template( + [nodes.Assign(nodes.Name("foo", "store"), nodes.Name("bar", "load")), for_loop] + ) + + sym = symbols_for_node(tmpl) + assert sym.refs == { + "foo": "l_0_foo", + "bar": "l_0_bar", + "seq": "l_0_seq", + } + assert sym.loads == { + "l_0_foo": ("undefined", None), + "l_0_bar": ("resolve", "bar"), + "l_0_seq": ("resolve", "seq"), + } + + sym = symbols_for_node(for_loop, sym) + assert sym.refs == { + "foo": "l_1_foo", + } + assert sym.loads == { + "l_1_foo": ("param", None), + } + + +def test_complex(): + title_block = nodes.Block( + "title", [nodes.Output([nodes.TemplateData(u"Page Title")])], False + ) + + render_title_macro = nodes.Macro( + "render_title", + [nodes.Name("title", "param")], + [], + [ + nodes.Output( + [ + nodes.TemplateData(u'\n <div class="title">\n <h1>'), + nodes.Name("title", "load"), + nodes.TemplateData(u"</h1>\n <p>"), + nodes.Name("subtitle", "load"), + nodes.TemplateData(u"</p>\n "), + ] + ), + nodes.Assign( + nodes.Name("subtitle", "store"), nodes.Const("something else") + ), + nodes.Output( + [ + nodes.TemplateData(u"\n <p>"), + nodes.Name("subtitle", "load"), + nodes.TemplateData(u"</p>\n </div>\n"), + nodes.If( + nodes.Name("something", "load"), + [ + nodes.Assign( + nodes.Name("title_upper", "store"), + nodes.Filter( + nodes.Name("title", "load"), + "upper", + [], + [], + None, + None, + ), + ), + nodes.Output( + [ + nodes.Name("title_upper", "load"), + nodes.Call( + nodes.Name("render_title", "load"), + [nodes.Const("Aha")], + [], + None, + None, + ), + ] + ), + ], + [], + [], + ), + ] + ), + ], + ) + + for_loop = nodes.For( + nodes.Name("item", "store"), + nodes.Name("seq", "load"), + [ + nodes.Output( + [ + nodes.TemplateData(u"\n <li>"), + nodes.Name("item", "load"), + nodes.TemplateData(u"</li>\n <span>"), + ] + ), + nodes.Include(nodes.Const("helper.html"), True, False), + nodes.Output([nodes.TemplateData(u"</span>\n ")]), + ], + [], + None, + False, + ) + + body_block = nodes.Block( + "body", + [ + nodes.Output( + [ + nodes.TemplateData(u"\n "), + nodes.Call( + nodes.Name("render_title", "load"), + [nodes.Name("item", "load")], + [], + None, + None, + ), + nodes.TemplateData(u"\n <ul>\n "), + ] + ), + for_loop, + nodes.Output([nodes.TemplateData(u"\n </ul>\n")]), + ], + False, + ) + + tmpl = nodes.Template( + [ + nodes.Extends(nodes.Const("layout.html")), + title_block, + render_title_macro, + body_block, + ] + ) + + tmpl_sym = symbols_for_node(tmpl) + assert tmpl_sym.refs == { + "render_title": "l_0_render_title", + } + assert tmpl_sym.loads == { + "l_0_render_title": ("undefined", None), + } + assert tmpl_sym.stores == set(["render_title"]) + assert tmpl_sym.dump_stores() == { + "render_title": "l_0_render_title", + } + + macro_sym = symbols_for_node(render_title_macro, tmpl_sym) + assert macro_sym.refs == { + "subtitle": "l_1_subtitle", + "something": "l_1_something", + "title": "l_1_title", + "title_upper": "l_1_title_upper", + } + assert macro_sym.loads == { + "l_1_subtitle": ("resolve", "subtitle"), + "l_1_something": ("resolve", "something"), + "l_1_title": ("param", None), + "l_1_title_upper": ("resolve", "title_upper"), + } + assert macro_sym.stores == set(["title", "title_upper", "subtitle"]) + assert macro_sym.find_ref("render_title") == "l_0_render_title" + assert macro_sym.dump_stores() == { + "title": "l_1_title", + "title_upper": "l_1_title_upper", + "subtitle": "l_1_subtitle", + "render_title": "l_0_render_title", + } + + body_sym = symbols_for_node(body_block) + assert body_sym.refs == { + "item": "l_0_item", + "seq": "l_0_seq", + "render_title": "l_0_render_title", + } + assert body_sym.loads == { + "l_0_item": ("resolve", "item"), + "l_0_seq": ("resolve", "seq"), + "l_0_render_title": ("resolve", "render_title"), + } + assert body_sym.stores == set([]) + + for_sym = symbols_for_node(for_loop, body_sym) + assert for_sym.refs == { + "item": "l_1_item", + } + assert for_sym.loads == { + "l_1_item": ("param", None), + } + assert for_sym.stores == set(["item"]) + assert for_sym.dump_stores() == { + "item": "l_1_item", + } + + +def test_if_branching_stores(): + tmpl = nodes.Template( + [ + nodes.If( + nodes.Name("expression", "load"), + [nodes.Assign(nodes.Name("variable", "store"), nodes.Const(42))], + [], + [], + ) + ] + ) + + sym = symbols_for_node(tmpl) + assert sym.refs == {"variable": "l_0_variable", "expression": "l_0_expression"} + assert sym.stores == set(["variable"]) + assert sym.loads == { + "l_0_variable": ("resolve", "variable"), + "l_0_expression": ("resolve", "expression"), + } + assert sym.dump_stores() == { + "variable": "l_0_variable", + } + + +def test_if_branching_stores_undefined(): + tmpl = nodes.Template( + [ + nodes.Assign(nodes.Name("variable", "store"), nodes.Const(23)), + nodes.If( + nodes.Name("expression", "load"), + [nodes.Assign(nodes.Name("variable", "store"), nodes.Const(42))], + [], + [], + ), + ] + ) + + sym = symbols_for_node(tmpl) + assert sym.refs == {"variable": "l_0_variable", "expression": "l_0_expression"} + assert sym.stores == set(["variable"]) + assert sym.loads == { + "l_0_variable": ("undefined", None), + "l_0_expression": ("resolve", "expression"), + } + assert sym.dump_stores() == { + "variable": "l_0_variable", + } + + +def test_if_branching_multi_scope(): + for_loop = nodes.For( + nodes.Name("item", "store"), + nodes.Name("seq", "load"), + [ + nodes.If( + nodes.Name("expression", "load"), + [nodes.Assign(nodes.Name("x", "store"), nodes.Const(42))], + [], + [], + ), + nodes.Include(nodes.Const("helper.html"), True, False), + ], + [], + None, + False, + ) + + tmpl = nodes.Template( + [nodes.Assign(nodes.Name("x", "store"), nodes.Const(23)), for_loop] + ) + + tmpl_sym = symbols_for_node(tmpl) + for_sym = symbols_for_node(for_loop, tmpl_sym) + assert for_sym.stores == set(["item", "x"]) + assert for_sym.loads == { + "l_1_x": ("alias", "l_0_x"), + "l_1_item": ("param", None), + "l_1_expression": ("resolve", "expression"), + } diff --git a/contrib/python/Jinja2/py2/tests/test_imports.py b/contrib/python/Jinja2/py2/tests/test_imports.py new file mode 100644 index 0000000000..fad2edafb6 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_imports.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +import pytest + +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2.exceptions import TemplateNotFound +from jinja2.exceptions import TemplatesNotFound +from jinja2.exceptions import TemplateSyntaxError + + +@pytest.fixture +def test_env(): + env = Environment( + loader=DictLoader( + dict( + module="{% macro test() %}[{{ foo }}|{{ bar }}]{% endmacro %}", + header="[{{ foo }}|{{ 23 }}]", + o_printer="({{ o }})", + ) + ) + ) + env.globals["bar"] = 23 + return env + + +class TestImports(object): + def test_context_imports(self, test_env): + t = test_env.from_string('{% import "module" as m %}{{ m.test() }}') + assert t.render(foo=42) == "[|23]" + t = test_env.from_string( + '{% import "module" as m without context %}{{ m.test() }}' + ) + assert t.render(foo=42) == "[|23]" + t = test_env.from_string( + '{% import "module" as m with context %}{{ m.test() }}' + ) + assert t.render(foo=42) == "[42|23]" + t = test_env.from_string('{% from "module" import test %}{{ test() }}') + assert t.render(foo=42) == "[|23]" + t = test_env.from_string( + '{% from "module" import test without context %}{{ test() }}' + ) + assert t.render(foo=42) == "[|23]" + t = test_env.from_string( + '{% from "module" import test with context %}{{ test() }}' + ) + assert t.render(foo=42) == "[42|23]" + + def test_import_needs_name(self, test_env): + test_env.from_string('{% from "foo" import bar %}') + test_env.from_string('{% from "foo" import bar, baz %}') + + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import %}') + + def test_no_trailing_comma(self, test_env): + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import bar, %}') + + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import bar,, %}') + + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import, %}') + + def test_trailing_comma_with_context(self, test_env): + test_env.from_string('{% from "foo" import bar, baz with context %}') + test_env.from_string('{% from "foo" import bar, baz, with context %}') + test_env.from_string('{% from "foo" import bar, with context %}') + test_env.from_string('{% from "foo" import bar, with, context %}') + test_env.from_string('{% from "foo" import bar, with with context %}') + + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import bar,, with context %}') + + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import bar with context, %}') + + def test_exports(self, test_env): + m = test_env.from_string( + """ + {% macro toplevel() %}...{% endmacro %} + {% macro __private() %}...{% endmacro %} + {% set variable = 42 %} + {% for item in [1] %} + {% macro notthere() %}{% endmacro %} + {% endfor %} + """ + ).module + assert m.toplevel() == "..." + assert not hasattr(m, "__missing") + assert m.variable == 42 + assert not hasattr(m, "notthere") + + +class TestIncludes(object): + def test_context_include(self, test_env): + t = test_env.from_string('{% include "header" %}') + assert t.render(foo=42) == "[42|23]" + t = test_env.from_string('{% include "header" with context %}') + assert t.render(foo=42) == "[42|23]" + t = test_env.from_string('{% include "header" without context %}') + assert t.render(foo=42) == "[|23]" + + def test_choice_includes(self, test_env): + t = test_env.from_string('{% include ["missing", "header"] %}') + assert t.render(foo=42) == "[42|23]" + + t = test_env.from_string('{% include ["missing", "missing2"] ignore missing %}') + assert t.render(foo=42) == "" + + t = test_env.from_string('{% include ["missing", "missing2"] %}') + pytest.raises(TemplateNotFound, t.render) + with pytest.raises(TemplatesNotFound) as e: + t.render() + + assert e.value.templates == ["missing", "missing2"] + assert e.value.name == "missing2" + + def test_includes(t, **ctx): + ctx["foo"] = 42 + assert t.render(ctx) == "[42|23]" + + t = test_env.from_string('{% include ["missing", "header"] %}') + test_includes(t) + t = test_env.from_string("{% include x %}") + test_includes(t, x=["missing", "header"]) + t = test_env.from_string('{% include [x, "header"] %}') + test_includes(t, x="missing") + t = test_env.from_string("{% include x %}") + test_includes(t, x="header") + t = test_env.from_string("{% include [x] %}") + test_includes(t, x="header") + + def test_include_ignoring_missing(self, test_env): + t = test_env.from_string('{% include "missing" %}') + pytest.raises(TemplateNotFound, t.render) + for extra in "", "with context", "without context": + t = test_env.from_string( + '{% include "missing" ignore missing ' + extra + " %}" + ) + assert t.render() == "" + + def test_context_include_with_overrides(self, test_env): + env = Environment( + loader=DictLoader( + dict( + main="{% for item in [1, 2, 3] %}{% include 'item' %}{% endfor %}", + item="{{ item }}", + ) + ) + ) + assert env.get_template("main").render() == "123" + + def test_unoptimized_scopes(self, test_env): + t = test_env.from_string( + """ + {% macro outer(o) %} + {% macro inner() %} + {% include "o_printer" %} + {% endmacro %} + {{ inner() }} + {% endmacro %} + {{ outer("FOO") }} + """ + ) + assert t.render().strip() == "(FOO)" + + def test_import_from_with_context(self): + env = Environment( + loader=DictLoader({"a": "{% macro x() %}{{ foobar }}{% endmacro %}"}) + ) + t = env.from_string( + "{% set foobar = 42 %}{% from 'a' import x with context %}{{ x() }}" + ) + assert t.render() == "42" diff --git a/contrib/python/Jinja2/py2/tests/test_inheritance.py b/contrib/python/Jinja2/py2/tests/test_inheritance.py new file mode 100644 index 0000000000..e513d2e3eb --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_inheritance.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- +import pytest + +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import TemplateRuntimeError + +LAYOUTTEMPLATE = """\ +|{% block block1 %}block 1 from layout{% endblock %} +|{% block block2 %}block 2 from layout{% endblock %} +|{% block block3 %} +{% block block4 %}nested block 4 from layout{% endblock %} +{% endblock %}|""" + +LEVEL1TEMPLATE = """\ +{% extends "layout" %} +{% block block1 %}block 1 from level1{% endblock %}""" + +LEVEL2TEMPLATE = """\ +{% extends "level1" %} +{% block block2 %}{% block block5 %}nested block 5 from level2{% +endblock %}{% endblock %}""" + +LEVEL3TEMPLATE = """\ +{% extends "level2" %} +{% block block5 %}block 5 from level3{% endblock %} +{% block block4 %}block 4 from level3{% endblock %} +""" + +LEVEL4TEMPLATE = """\ +{% extends "level3" %} +{% block block3 %}block 3 from level4{% endblock %} +""" + +WORKINGTEMPLATE = """\ +{% extends "layout" %} +{% block block1 %} + {% if false %} + {% block block2 %} + this should workd + {% endblock %} + {% endif %} +{% endblock %} +""" + +DOUBLEEXTENDS = """\ +{% extends "layout" %} +{% extends "layout" %} +{% block block1 %} + {% if false %} + {% block block2 %} + this should workd + {% endblock %} + {% endif %} +{% endblock %} +""" + + +@pytest.fixture +def env(): + return Environment( + loader=DictLoader( + { + "layout": LAYOUTTEMPLATE, + "level1": LEVEL1TEMPLATE, + "level2": LEVEL2TEMPLATE, + "level3": LEVEL3TEMPLATE, + "level4": LEVEL4TEMPLATE, + "working": WORKINGTEMPLATE, + "doublee": DOUBLEEXTENDS, + } + ), + trim_blocks=True, + ) + + +class TestInheritance(object): + def test_layout(self, env): + tmpl = env.get_template("layout") + assert tmpl.render() == ( + "|block 1 from layout|block 2 from layout|nested block 4 from layout|" + ) + + def test_level1(self, env): + tmpl = env.get_template("level1") + assert tmpl.render() == ( + "|block 1 from level1|block 2 from layout|nested block 4 from layout|" + ) + + def test_level2(self, env): + tmpl = env.get_template("level2") + assert tmpl.render() == ( + "|block 1 from level1|nested block 5 from " + "level2|nested block 4 from layout|" + ) + + def test_level3(self, env): + tmpl = env.get_template("level3") + assert tmpl.render() == ( + "|block 1 from level1|block 5 from level3|block 4 from level3|" + ) + + def test_level4(self, env): + tmpl = env.get_template("level4") + assert tmpl.render() == ( + "|block 1 from level1|block 5 from level3|block 3 from level4|" + ) + + def test_super(self, env): + env = Environment( + loader=DictLoader( + { + "a": "{% block intro %}INTRO{% endblock %}|" + "BEFORE|{% block data %}INNER{% endblock %}|AFTER", + "b": '{% extends "a" %}{% block data %}({{ ' + "super() }}){% endblock %}", + "c": '{% extends "b" %}{% block intro %}--{{ ' + "super() }}--{% endblock %}\n{% block data " + "%}[{{ super() }}]{% endblock %}", + } + ) + ) + tmpl = env.get_template("c") + assert tmpl.render() == "--INTRO--|BEFORE|[(INNER)]|AFTER" + + def test_working(self, env): + env.get_template("working") + + def test_reuse_blocks(self, env): + tmpl = env.from_string( + "{{ self.foo() }}|{% block foo %}42{% endblock %}|{{ self.foo() }}" + ) + assert tmpl.render() == "42|42|42" + + def test_preserve_blocks(self, env): + env = Environment( + loader=DictLoader( + { + "a": "{% if false %}{% block x %}A{% endblock %}" + "{% endif %}{{ self.x() }}", + "b": '{% extends "a" %}{% block x %}B{{ super() }}{% endblock %}', + } + ) + ) + tmpl = env.get_template("b") + assert tmpl.render() == "BA" + + def test_dynamic_inheritance(self, env): + env = Environment( + loader=DictLoader( + { + "master1": "MASTER1{% block x %}{% endblock %}", + "master2": "MASTER2{% block x %}{% endblock %}", + "child": "{% extends master %}{% block x %}CHILD{% endblock %}", + } + ) + ) + tmpl = env.get_template("child") + for m in range(1, 3): + assert tmpl.render(master="master%d" % m) == "MASTER%dCHILD" % m + + def test_multi_inheritance(self, env): + env = Environment( + loader=DictLoader( + { + "master1": "MASTER1{% block x %}{% endblock %}", + "master2": "MASTER2{% block x %}{% endblock %}", + "child": """{% if master %}{% extends master %}{% else %}{% extends + 'master1' %}{% endif %}{% block x %}CHILD{% endblock %}""", + } + ) + ) + tmpl = env.get_template("child") + assert tmpl.render(master="master2") == "MASTER2CHILD" + assert tmpl.render(master="master1") == "MASTER1CHILD" + assert tmpl.render() == "MASTER1CHILD" + + def test_scoped_block(self, env): + env = Environment( + loader=DictLoader( + { + "master.html": "{% for item in seq %}[{% block item scoped %}" + "{% endblock %}]{% endfor %}" + } + ) + ) + t = env.from_string( + "{% extends 'master.html' %}{% block item %}{{ item }}{% endblock %}" + ) + assert t.render(seq=list(range(5))) == "[0][1][2][3][4]" + + def test_super_in_scoped_block(self, env): + env = Environment( + loader=DictLoader( + { + "master.html": "{% for item in seq %}[{% block item scoped %}" + "{{ item }}{% endblock %}]{% endfor %}" + } + ) + ) + t = env.from_string( + '{% extends "master.html" %}{% block item %}' + "{{ super() }}|{{ item * 2 }}{% endblock %}" + ) + assert t.render(seq=list(range(5))) == "[0|0][1|2][2|4][3|6][4|8]" + + def test_scoped_block_after_inheritance(self, env): + env = Environment( + loader=DictLoader( + { + "layout.html": """ + {% block useless %}{% endblock %} + """, + "index.html": """ + {%- extends 'layout.html' %} + {% from 'helpers.html' import foo with context %} + {% block useless %} + {% for x in [1, 2, 3] %} + {% block testing scoped %} + {{ foo(x) }} + {% endblock %} + {% endfor %} + {% endblock %} + """, + "helpers.html": """ + {% macro foo(x) %}{{ the_foo + x }}{% endmacro %} + """, + } + ) + ) + rv = env.get_template("index.html").render(the_foo=42).split() + assert rv == ["43", "44", "45"] + + +class TestBugFix(object): + def test_fixed_macro_scoping_bug(self, env): + assert ( + Environment( + loader=DictLoader( + { + "test.html": """\ + {% extends 'details.html' %} + + {% macro my_macro() %} + my_macro + {% endmacro %} + + {% block inner_box %} + {{ my_macro() }} + {% endblock %} + """, + "details.html": """\ + {% extends 'standard.html' %} + + {% macro my_macro() %} + my_macro + {% endmacro %} + + {% block content %} + {% block outer_box %} + outer_box + {% block inner_box %} + inner_box + {% endblock %} + {% endblock %} + {% endblock %} + """, + "standard.html": """ + {% block content %} {% endblock %} + """, + } + ) + ) + .get_template("test.html") + .render() + .split() + == [u"outer_box", u"my_macro"] + ) + + def test_double_extends(self, env): + """Ensures that a template with more than 1 {% extends ... %} usage + raises a ``TemplateError``. + """ + with pytest.raises(TemplateRuntimeError, match="extended multiple times"): + env.get_template("doublee").render() diff --git a/contrib/python/Jinja2/py2/tests/test_lexnparse.py b/contrib/python/Jinja2/py2/tests/test_lexnparse.py new file mode 100644 index 0000000000..83ae75e05c --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_lexnparse.py @@ -0,0 +1,930 @@ +# -*- coding: utf-8 -*- +import pytest + +from jinja2 import Environment +from jinja2 import nodes +from jinja2 import Template +from jinja2 import TemplateSyntaxError +from jinja2 import UndefinedError +from jinja2._compat import iteritems +from jinja2._compat import PY2 +from jinja2._compat import text_type +from jinja2.lexer import Token +from jinja2.lexer import TOKEN_BLOCK_BEGIN +from jinja2.lexer import TOKEN_BLOCK_END +from jinja2.lexer import TOKEN_EOF +from jinja2.lexer import TokenStream + + +# how does a string look like in jinja syntax? +if PY2: + + def jinja_string_repr(string): + return repr(string)[1:] + + +else: + jinja_string_repr = repr + + +class TestTokenStream(object): + test_tokens = [ + Token(1, TOKEN_BLOCK_BEGIN, ""), + Token(2, TOKEN_BLOCK_END, ""), + ] + + def test_simple(self, env): + ts = TokenStream(self.test_tokens, "foo", "bar") + assert ts.current.type is TOKEN_BLOCK_BEGIN + assert bool(ts) + assert not bool(ts.eos) + next(ts) + assert ts.current.type is TOKEN_BLOCK_END + assert bool(ts) + assert not bool(ts.eos) + next(ts) + assert ts.current.type is TOKEN_EOF + assert not bool(ts) + assert bool(ts.eos) + + def test_iter(self, env): + token_types = [t.type for t in TokenStream(self.test_tokens, "foo", "bar")] + assert token_types == [ + "block_begin", + "block_end", + ] + + +class TestLexer(object): + def test_raw1(self, env): + tmpl = env.from_string( + "{% raw %}foo{% endraw %}|" + "{%raw%}{{ bar }}|{% baz %}{% endraw %}" + ) + assert tmpl.render() == "foo|{{ bar }}|{% baz %}" + + def test_raw2(self, env): + tmpl = env.from_string("1 {%- raw -%} 2 {%- endraw -%} 3") + assert tmpl.render() == "123" + + def test_raw3(self, env): + # The second newline after baz exists because it is AFTER the + # {% raw %} and is ignored. + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string("bar\n{% raw %}\n {{baz}}2 spaces\n{% endraw %}\nfoo") + assert tmpl.render(baz="test") == "bar\n\n {{baz}}2 spaces\nfoo" + + def test_raw4(self, env): + # The trailing dash of the {% raw -%} cleans both the spaces and + # newlines up to the first character of data. + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + "bar\n{%- raw -%}\n\n \n 2 spaces\n space{%- endraw -%}\nfoo" + ) + assert tmpl.render() == "bar2 spaces\n spacefoo" + + def test_balancing(self, env): + env = Environment("{%", "%}", "${", "}") + tmpl = env.from_string( + """{% for item in seq + %}${{'foo': item}|upper}{% endfor %}""" + ) + assert tmpl.render(seq=list(range(3))) == "{'FOO': 0}{'FOO': 1}{'FOO': 2}" + + def test_comments(self, env): + env = Environment("<!--", "-->", "{", "}") + tmpl = env.from_string( + """\ +<ul> +<!--- for item in seq --> + <li>{item}</li> +<!--- endfor --> +</ul>""" + ) + assert tmpl.render(seq=list(range(3))) == ( + "<ul>\n <li>0</li>\n <li>1</li>\n <li>2</li>\n</ul>" + ) + + def test_string_escapes(self, env): + for char in u"\0", u"\u2668", u"\xe4", u"\t", u"\r", u"\n": + tmpl = env.from_string("{{ %s }}" % jinja_string_repr(char)) + assert tmpl.render() == char + assert env.from_string('{{ "\N{HOT SPRINGS}" }}').render() == u"\u2668" + + def test_bytefallback(self, env): + from pprint import pformat + + tmpl = env.from_string(u"""{{ 'foo'|pprint }}|{{ 'bär'|pprint }}""") + assert tmpl.render() == pformat("foo") + "|" + pformat(u"bär") + + def test_operators(self, env): + from jinja2.lexer import operators + + for test, expect in iteritems(operators): + if test in "([{}])": + continue + stream = env.lexer.tokenize("{{ %s }}" % test) + next(stream) + assert stream.current.type == expect + + def test_normalizing(self, env): + for seq in "\r", "\r\n", "\n": + env = Environment(newline_sequence=seq) + tmpl = env.from_string("1\n2\r\n3\n4\n") + result = tmpl.render() + assert result.replace(seq, "X") == "1X2X3X4" + + def test_trailing_newline(self, env): + for keep in [True, False]: + env = Environment(keep_trailing_newline=keep) + for template, expected in [ + ("", {}), + ("no\nnewline", {}), + ("with\nnewline\n", {False: "with\nnewline"}), + ("with\nseveral\n\n\n", {False: "with\nseveral\n\n"}), + ]: + tmpl = env.from_string(template) + expect = expected.get(keep, template) + result = tmpl.render() + assert result == expect, (keep, template, result, expect) + + @pytest.mark.parametrize( + "name,valid2,valid3", + ( + (u"foo", True, True), + (u"föö", False, True), + (u"き", False, True), + (u"_", True, True), + (u"1a", False, False), # invalid ascii start + (u"a-", False, False), # invalid ascii continue + (u"🐍", False, False), # invalid unicode start + (u"a🐍", False, False), # invalid unicode continue + # start characters not matched by \w + (u"\u1885", False, True), + (u"\u1886", False, True), + (u"\u2118", False, True), + (u"\u212e", False, True), + # continue character not matched by \w + (u"\xb7", False, False), + (u"a\xb7", False, True), + ), + ) + def test_name(self, env, name, valid2, valid3): + t = u"{{ " + name + u" }}" + + if (valid2 and PY2) or (valid3 and not PY2): + # valid for version being tested, shouldn't raise + env.from_string(t) + else: + pytest.raises(TemplateSyntaxError, env.from_string, t) + + def test_lineno_with_strip(self, env): + tokens = env.lex( + """\ +<html> + <body> + {%- block content -%} + <hr> + {{ item }} + {% endblock %} + </body> +</html>""" + ) + for tok in tokens: + lineno, token_type, value = tok + if token_type == "name" and value == "item": + assert lineno == 5 + break + + +class TestParser(object): + def test_php_syntax(self, env): + env = Environment("<?", "?>", "<?=", "?>", "<!--", "-->") + tmpl = env.from_string( + """\ +<!-- I'm a comment, I'm not interesting -->\ +<? for item in seq -?> + <?= item ?> +<?- endfor ?>""" + ) + assert tmpl.render(seq=list(range(5))) == "01234" + + def test_erb_syntax(self, env): + env = Environment("<%", "%>", "<%=", "%>", "<%#", "%>") + tmpl = env.from_string( + """\ +<%# I'm a comment, I'm not interesting %>\ +<% for item in seq -%> + <%= item %> +<%- endfor %>""" + ) + assert tmpl.render(seq=list(range(5))) == "01234" + + def test_comment_syntax(self, env): + env = Environment("<!--", "-->", "${", "}", "<!--#", "-->") + tmpl = env.from_string( + """\ +<!--# I'm a comment, I'm not interesting -->\ +<!-- for item in seq ---> + ${item} +<!--- endfor -->""" + ) + assert tmpl.render(seq=list(range(5))) == "01234" + + def test_balancing(self, env): + tmpl = env.from_string("""{{{'foo':'bar'}.foo}}""") + assert tmpl.render() == "bar" + + def test_start_comment(self, env): + tmpl = env.from_string( + """{# foo comment +and bar comment #} +{% macro blub() %}foo{% endmacro %} +{{ blub() }}""" + ) + assert tmpl.render().strip() == "foo" + + def test_line_syntax(self, env): + env = Environment("<%", "%>", "${", "}", "<%#", "%>", "%") + tmpl = env.from_string( + """\ +<%# regular comment %> +% for item in seq: + ${item} +% endfor""" + ) + assert [ + int(x.strip()) for x in tmpl.render(seq=list(range(5))).split() + ] == list(range(5)) + + env = Environment("<%", "%>", "${", "}", "<%#", "%>", "%", "##") + tmpl = env.from_string( + """\ +<%# regular comment %> +% for item in seq: + ${item} ## the rest of the stuff +% endfor""" + ) + assert [ + int(x.strip()) for x in tmpl.render(seq=list(range(5))).split() + ] == list(range(5)) + + def test_line_syntax_priority(self, env): + # XXX: why is the whitespace there in front of the newline? + env = Environment("{%", "%}", "${", "}", "/*", "*/", "##", "#") + tmpl = env.from_string( + """\ +/* ignore me. + I'm a multiline comment */ +## for item in seq: +* ${item} # this is just extra stuff +## endfor""" + ) + assert tmpl.render(seq=[1, 2]).strip() == "* 1\n* 2" + env = Environment("{%", "%}", "${", "}", "/*", "*/", "#", "##") + tmpl = env.from_string( + """\ +/* ignore me. + I'm a multiline comment */ +# for item in seq: +* ${item} ## this is just extra stuff + ## extra stuff i just want to ignore +# endfor""" + ) + assert tmpl.render(seq=[1, 2]).strip() == "* 1\n\n* 2" + + def test_error_messages(self, env): + def assert_error(code, expected): + with pytest.raises(TemplateSyntaxError, match=expected): + Template(code) + + assert_error( + "{% for item in seq %}...{% endif %}", + "Encountered unknown tag 'endif'. Jinja was looking " + "for the following tags: 'endfor' or 'else'. The " + "innermost block that needs to be closed is 'for'.", + ) + assert_error( + "{% if foo %}{% for item in seq %}...{% endfor %}{% endfor %}", + "Encountered unknown tag 'endfor'. Jinja was looking for " + "the following tags: 'elif' or 'else' or 'endif'. The " + "innermost block that needs to be closed is 'if'.", + ) + assert_error( + "{% if foo %}", + "Unexpected end of template. Jinja was looking for the " + "following tags: 'elif' or 'else' or 'endif'. The " + "innermost block that needs to be closed is 'if'.", + ) + assert_error( + "{% for item in seq %}", + "Unexpected end of template. Jinja was looking for the " + "following tags: 'endfor' or 'else'. The innermost block " + "that needs to be closed is 'for'.", + ) + assert_error( + "{% block foo-bar-baz %}", + "Block names in Jinja have to be valid Python identifiers " + "and may not contain hyphens, use an underscore instead.", + ) + assert_error("{% unknown_tag %}", "Encountered unknown tag 'unknown_tag'.") + + +class TestSyntax(object): + def test_call(self, env): + env = Environment() + env.globals["foo"] = lambda a, b, c, e, g: a + b + c + e + g + tmpl = env.from_string("{{ foo('a', c='d', e='f', *['b'], **{'g': 'h'}) }}") + assert tmpl.render() == "abdfh" + + def test_slicing(self, env): + tmpl = env.from_string("{{ [1, 2, 3][:] }}|{{ [1, 2, 3][::-1] }}") + assert tmpl.render() == "[1, 2, 3]|[3, 2, 1]" + + def test_attr(self, env): + tmpl = env.from_string("{{ foo.bar }}|{{ foo['bar'] }}") + assert tmpl.render(foo={"bar": 42}) == "42|42" + + def test_subscript(self, env): + tmpl = env.from_string("{{ foo[0] }}|{{ foo[-1] }}") + assert tmpl.render(foo=[0, 1, 2]) == "0|2" + + def test_tuple(self, env): + tmpl = env.from_string("{{ () }}|{{ (1,) }}|{{ (1, 2) }}") + assert tmpl.render() == "()|(1,)|(1, 2)" + + def test_math(self, env): + tmpl = env.from_string("{{ (1 + 1 * 2) - 3 / 2 }}|{{ 2**3 }}") + assert tmpl.render() == "1.5|8" + + def test_div(self, env): + tmpl = env.from_string("{{ 3 // 2 }}|{{ 3 / 2 }}|{{ 3 % 2 }}") + assert tmpl.render() == "1|1.5|1" + + def test_unary(self, env): + tmpl = env.from_string("{{ +3 }}|{{ -3 }}") + assert tmpl.render() == "3|-3" + + def test_concat(self, env): + tmpl = env.from_string("{{ [1, 2] ~ 'foo' }}") + assert tmpl.render() == "[1, 2]foo" + + @pytest.mark.parametrize( + ("a", "op", "b"), + [ + (1, ">", 0), + (1, ">=", 1), + (2, "<", 3), + (3, "<=", 4), + (4, "==", 4), + (4, "!=", 5), + ], + ) + def test_compare(self, env, a, op, b): + t = env.from_string("{{ %d %s %d }}" % (a, op, b)) + assert t.render() == "True" + + def test_compare_parens(self, env): + t = env.from_string("{{ i * (j < 5) }}") + assert t.render(i=2, j=3) == "2" + + @pytest.mark.parametrize( + ("src", "expect"), + [ + ("{{ 4 < 2 < 3 }}", "False"), + ("{{ a < b < c }}", "False"), + ("{{ 4 > 2 > 3 }}", "False"), + ("{{ a > b > c }}", "False"), + ("{{ 4 > 2 < 3 }}", "True"), + ("{{ a > b < c }}", "True"), + ], + ) + def test_compare_compound(self, env, src, expect): + t = env.from_string(src) + assert t.render(a=4, b=2, c=3) == expect + + def test_inop(self, env): + tmpl = env.from_string("{{ 1 in [1, 2, 3] }}|{{ 1 not in [1, 2, 3] }}") + assert tmpl.render() == "True|False" + + @pytest.mark.parametrize("value", ("[]", "{}", "()")) + def test_collection_literal(self, env, value): + t = env.from_string("{{ %s }}" % value) + assert t.render() == value + + @pytest.mark.parametrize( + ("value", "expect"), + ( + ("1", "1"), + ("123", "123"), + ("12_34_56", "123456"), + ("1.2", "1.2"), + ("34.56", "34.56"), + ("3_4.5_6", "34.56"), + ("1e0", "1.0"), + ("10e1", "100.0"), + ("2.5e100", "2.5e+100"), + ("2.5e+100", "2.5e+100"), + ("25.6e-10", "2.56e-09"), + ("1_2.3_4e5_6", "1.234e+57"), + ), + ) + def test_numeric_literal(self, env, value, expect): + t = env.from_string("{{ %s }}" % value) + assert t.render() == expect + + def test_bool(self, env): + tmpl = env.from_string( + "{{ true and false }}|{{ false or true }}|{{ not false }}" + ) + assert tmpl.render() == "False|True|True" + + def test_grouping(self, env): + tmpl = env.from_string( + "{{ (true and false) or (false and true) and not false }}" + ) + assert tmpl.render() == "False" + + def test_django_attr(self, env): + tmpl = env.from_string("{{ [1, 2, 3].0 }}|{{ [[1]].0.0 }}") + assert tmpl.render() == "1|1" + + def test_conditional_expression(self, env): + tmpl = env.from_string("""{{ 0 if true else 1 }}""") + assert tmpl.render() == "0" + + def test_short_conditional_expression(self, env): + tmpl = env.from_string("<{{ 1 if false }}>") + assert tmpl.render() == "<>" + + tmpl = env.from_string("<{{ (1 if false).bar }}>") + pytest.raises(UndefinedError, tmpl.render) + + def test_filter_priority(self, env): + tmpl = env.from_string('{{ "foo"|upper + "bar"|upper }}') + assert tmpl.render() == "FOOBAR" + + def test_function_calls(self, env): + tests = [ + (True, "*foo, bar"), + (True, "*foo, *bar"), + (True, "**foo, *bar"), + (True, "**foo, bar"), + (True, "**foo, **bar"), + (True, "**foo, bar=42"), + (False, "foo, bar"), + (False, "foo, bar=42"), + (False, "foo, bar=23, *args"), + (False, "foo, *args, bar=23"), + (False, "a, b=c, *d, **e"), + (False, "*foo, bar=42"), + (False, "*foo, **bar"), + (False, "*foo, bar=42, **baz"), + (False, "foo, *args, bar=23, **baz"), + ] + for should_fail, sig in tests: + if should_fail: + pytest.raises( + TemplateSyntaxError, env.from_string, "{{ foo(%s) }}" % sig + ) + else: + env.from_string("foo(%s)" % sig) + + def test_tuple_expr(self, env): + for tmpl in [ + "{{ () }}", + "{{ (1, 2) }}", + "{{ (1, 2,) }}", + "{{ 1, }}", + "{{ 1, 2 }}", + "{% for foo, bar in seq %}...{% endfor %}", + "{% for x in foo, bar %}...{% endfor %}", + "{% for x in foo, %}...{% endfor %}", + ]: + assert env.from_string(tmpl) + + def test_trailing_comma(self, env): + tmpl = env.from_string("{{ (1, 2,) }}|{{ [1, 2,] }}|{{ {1: 2,} }}") + assert tmpl.render().lower() == "(1, 2)|[1, 2]|{1: 2}" + + def test_block_end_name(self, env): + env.from_string("{% block foo %}...{% endblock foo %}") + pytest.raises( + TemplateSyntaxError, env.from_string, "{% block x %}{% endblock y %}" + ) + + def test_constant_casing(self, env): + for const in True, False, None: + tmpl = env.from_string( + "{{ %s }}|{{ %s }}|{{ %s }}" + % (str(const), str(const).lower(), str(const).upper()) + ) + assert tmpl.render() == "%s|%s|" % (const, const) + + def test_test_chaining(self, env): + pytest.raises( + TemplateSyntaxError, env.from_string, "{{ foo is string is sequence }}" + ) + assert env.from_string("{{ 42 is string or 42 is number }}").render() == "True" + + def test_string_concatenation(self, env): + tmpl = env.from_string('{{ "foo" "bar" "baz" }}') + assert tmpl.render() == "foobarbaz" + + def test_notin(self, env): + bar = range(100) + tmpl = env.from_string("""{{ not 42 in bar }}""") + assert tmpl.render(bar=bar) == "False" + + def test_operator_precedence(self, env): + tmpl = env.from_string("""{{ 2 * 3 + 4 % 2 + 1 - 2 }}""") + assert tmpl.render() == text_type(2 * 3 + 4 % 2 + 1 - 2) + + def test_implicit_subscribed_tuple(self, env): + class Foo(object): + def __getitem__(self, x): + return x + + t = env.from_string("{{ foo[1, 2] }}") + assert t.render(foo=Foo()) == u"(1, 2)" + + def test_raw2(self, env): + tmpl = env.from_string("{% raw %}{{ FOO }} and {% BAR %}{% endraw %}") + assert tmpl.render() == "{{ FOO }} and {% BAR %}" + + def test_const(self, env): + tmpl = env.from_string( + "{{ true }}|{{ false }}|{{ none }}|" + "{{ none is defined }}|{{ missing is defined }}" + ) + assert tmpl.render() == "True|False|None|True|False" + + def test_neg_filter_priority(self, env): + node = env.parse("{{ -1|foo }}") + assert isinstance(node.body[0].nodes[0], nodes.Filter) + assert isinstance(node.body[0].nodes[0].node, nodes.Neg) + + def test_const_assign(self, env): + constass1 = """{% set true = 42 %}""" + constass2 = """{% for none in seq %}{% endfor %}""" + for tmpl in constass1, constass2: + pytest.raises(TemplateSyntaxError, env.from_string, tmpl) + + def test_localset(self, env): + tmpl = env.from_string( + """{% set foo = 0 %}\ +{% for item in [1, 2] %}{% set foo = 1 %}{% endfor %}\ +{{ foo }}""" + ) + assert tmpl.render() == "0" + + def test_parse_unary(self, env): + tmpl = env.from_string('{{ -foo["bar"] }}') + assert tmpl.render(foo={"bar": 42}) == "-42" + tmpl = env.from_string('{{ -foo["bar"]|abs }}') + assert tmpl.render(foo={"bar": 42}) == "42" + + +class TestLstripBlocks(object): + def test_lstrip(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(""" {% if True %}\n {% endif %}""") + assert tmpl.render() == "\n" + + def test_lstrip_trim(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string(""" {% if True %}\n {% endif %}""") + assert tmpl.render() == "" + + def test_no_lstrip(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(""" {%+ if True %}\n {%+ endif %}""") + assert tmpl.render() == " \n " + + def test_lstrip_blocks_false_with_no_lstrip(self, env): + # Test that + is a NOP (but does not cause an error) if lstrip_blocks=False + env = Environment(lstrip_blocks=False, trim_blocks=False) + tmpl = env.from_string(""" {% if True %}\n {% endif %}""") + assert tmpl.render() == " \n " + tmpl = env.from_string(""" {%+ if True %}\n {%+ endif %}""") + assert tmpl.render() == " \n " + + def test_lstrip_endline(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(""" hello{% if True %}\n goodbye{% endif %}""") + assert tmpl.render() == " hello\n goodbye" + + def test_lstrip_inline(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(""" {% if True %}hello {% endif %}""") + assert tmpl.render() == "hello " + + def test_lstrip_nested(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + """ {% if True %}a {% if True %}b {% endif %}c {% endif %}""" + ) + assert tmpl.render() == "a b c " + + def test_lstrip_left_chars(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + """ abc {% if True %} + hello{% endif %}""" + ) + assert tmpl.render() == " abc \n hello" + + def test_lstrip_embeded_strings(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(""" {% set x = " {% str %} " %}{{ x }}""") + assert tmpl.render() == " {% str %} " + + def test_lstrip_preserve_leading_newlines(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string("""\n\n\n{% set hello = 1 %}""") + assert tmpl.render() == "\n\n\n" + + def test_lstrip_comment(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + """ {# if True #} +hello + {#endif#}""" + ) + assert tmpl.render() == "\nhello\n" + + def test_lstrip_angle_bracket_simple(self, env): + env = Environment( + "<%", + "%>", + "${", + "}", + "<%#", + "%>", + "%", + "##", + lstrip_blocks=True, + trim_blocks=True, + ) + tmpl = env.from_string(""" <% if True %>hello <% endif %>""") + assert tmpl.render() == "hello " + + def test_lstrip_angle_bracket_comment(self, env): + env = Environment( + "<%", + "%>", + "${", + "}", + "<%#", + "%>", + "%", + "##", + lstrip_blocks=True, + trim_blocks=True, + ) + tmpl = env.from_string(""" <%# if True %>hello <%# endif %>""") + assert tmpl.render() == "hello " + + def test_lstrip_angle_bracket(self, env): + env = Environment( + "<%", + "%>", + "${", + "}", + "<%#", + "%>", + "%", + "##", + lstrip_blocks=True, + trim_blocks=True, + ) + tmpl = env.from_string( + """\ + <%# regular comment %> + <% for item in seq %> +${item} ## the rest of the stuff + <% endfor %>""" + ) + assert tmpl.render(seq=range(5)) == "".join("%s\n" % x for x in range(5)) + + def test_lstrip_angle_bracket_compact(self, env): + env = Environment( + "<%", + "%>", + "${", + "}", + "<%#", + "%>", + "%", + "##", + lstrip_blocks=True, + trim_blocks=True, + ) + tmpl = env.from_string( + """\ + <%#regular comment%> + <%for item in seq%> +${item} ## the rest of the stuff + <%endfor%>""" + ) + assert tmpl.render(seq=range(5)) == "".join("%s\n" % x for x in range(5)) + + def test_lstrip_blocks_outside_with_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + " {% if kvs %}(\n" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n" + " ){% endif %}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == "(\na=1 b=2 \n )" + + def test_lstrip_trim_blocks_outside_with_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string( + " {% if kvs %}(\n" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n" + " ){% endif %}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == "(\na=1 b=2 )" + + def test_lstrip_blocks_inside_with_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + " ({% if kvs %}\n" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n" + " {% endif %})" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == " (\na=1 b=2 \n)" + + def test_lstrip_trim_blocks_inside_with_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string( + " ({% if kvs %}\n" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n" + " {% endif %})" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == " (a=1 b=2 )" + + def test_lstrip_blocks_without_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + " {% if kvs %}" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}" + " {% endif %}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == " a=1 b=2 " + + def test_lstrip_trim_blocks_without_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string( + " {% if kvs %}" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}" + " {% endif %}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == " a=1 b=2 " + + def test_lstrip_blocks_consume_after_without_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + " {% if kvs -%}" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor -%}" + " {% endif -%}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == "a=1 b=2 " + + def test_lstrip_trim_blocks_consume_before_without_new_line(self): + env = Environment(lstrip_blocks=False, trim_blocks=False) + tmpl = env.from_string( + " {%- if kvs %}" + " {%- for k, v in kvs %}{{ k }}={{ v }} {% endfor -%}" + " {%- endif %}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == "a=1 b=2 " + + def test_lstrip_trim_blocks_comment(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string(" {# 1 space #}\n {# 2 spaces #} {# 4 spaces #}") + out = tmpl.render() + assert out == " " * 4 + + def test_lstrip_trim_blocks_raw(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string("{{x}}\n{%- raw %} {% endraw -%}\n{{ y }}") + out = tmpl.render(x=1, y=2) + assert out == "1 2" + + def test_php_syntax_with_manual(self, env): + env = Environment( + "<?", "?>", "<?=", "?>", "<!--", "-->", lstrip_blocks=True, trim_blocks=True + ) + tmpl = env.from_string( + """\ + <!-- I'm a comment, I'm not interesting --> + <? for item in seq -?> + <?= item ?> + <?- endfor ?>""" + ) + assert tmpl.render(seq=range(5)) == "01234" + + def test_php_syntax(self, env): + env = Environment( + "<?", "?>", "<?=", "?>", "<!--", "-->", lstrip_blocks=True, trim_blocks=True + ) + tmpl = env.from_string( + """\ + <!-- I'm a comment, I'm not interesting --> + <? for item in seq ?> + <?= item ?> + <? endfor ?>""" + ) + assert tmpl.render(seq=range(5)) == "".join( + " %s\n" % x for x in range(5) + ) + + def test_php_syntax_compact(self, env): + env = Environment( + "<?", "?>", "<?=", "?>", "<!--", "-->", lstrip_blocks=True, trim_blocks=True + ) + tmpl = env.from_string( + """\ + <!-- I'm a comment, I'm not interesting --> + <?for item in seq?> + <?=item?> + <?endfor?>""" + ) + assert tmpl.render(seq=range(5)) == "".join( + " %s\n" % x for x in range(5) + ) + + def test_erb_syntax(self, env): + env = Environment( + "<%", "%>", "<%=", "%>", "<%#", "%>", lstrip_blocks=True, trim_blocks=True + ) + # env.from_string('') + # for n,r in env.lexer.rules.iteritems(): + # print n + # print env.lexer.rules['root'][0][0].pattern + # print "'%s'" % tmpl.render(seq=range(5)) + tmpl = env.from_string( + """\ +<%# I'm a comment, I'm not interesting %> + <% for item in seq %> + <%= item %> + <% endfor %> +""" + ) + assert tmpl.render(seq=range(5)) == "".join(" %s\n" % x for x in range(5)) + + def test_erb_syntax_with_manual(self, env): + env = Environment( + "<%", "%>", "<%=", "%>", "<%#", "%>", lstrip_blocks=True, trim_blocks=True + ) + tmpl = env.from_string( + """\ +<%# I'm a comment, I'm not interesting %> + <% for item in seq -%> + <%= item %> + <%- endfor %>""" + ) + assert tmpl.render(seq=range(5)) == "01234" + + def test_erb_syntax_no_lstrip(self, env): + env = Environment( + "<%", "%>", "<%=", "%>", "<%#", "%>", lstrip_blocks=True, trim_blocks=True + ) + tmpl = env.from_string( + """\ +<%# I'm a comment, I'm not interesting %> + <%+ for item in seq -%> + <%= item %> + <%- endfor %>""" + ) + assert tmpl.render(seq=range(5)) == " 01234" + + def test_comment_syntax(self, env): + env = Environment( + "<!--", + "-->", + "${", + "}", + "<!--#", + "-->", + lstrip_blocks=True, + trim_blocks=True, + ) + tmpl = env.from_string( + """\ +<!--# I'm a comment, I'm not interesting -->\ +<!-- for item in seq ---> + ${item} +<!--- endfor -->""" + ) + assert tmpl.render(seq=range(5)) == "01234" diff --git a/contrib/python/Jinja2/py2/tests/test_loader.py b/contrib/python/Jinja2/py2/tests/test_loader.py new file mode 100644 index 0000000000..c613186606 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_loader.py @@ -0,0 +1,325 @@ +# -*- coding: utf-8 -*- +import os +import shutil +import sys +import tempfile +import time +import weakref + +import pytest + +from jinja2 import Environment +from jinja2 import loaders +from jinja2._compat import PY2 +from jinja2._compat import PYPY +from jinja2.exceptions import TemplateNotFound +from jinja2.loaders import split_template_path + +import yatest.common as yc + + +class TestLoaders(object): + def test_dict_loader(self, dict_loader): + env = Environment(loader=dict_loader) + tmpl = env.get_template("justdict.html") + assert tmpl.render().strip() == "FOO" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + def test_package_loader(self, package_loader): + env = Environment(loader=package_loader) + tmpl = env.get_template("test.html") + assert tmpl.render().strip() == "BAR" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + def test_filesystem_loader_overlapping_names(self, filesystem_loader): + res = os.path.dirname(filesystem_loader.searchpath[0]) + t2_dir = os.path.join(res, "templates2") + # Make "foo" show up before "foo/test.html". + filesystem_loader.searchpath.insert(0, t2_dir) + e = Environment(loader=filesystem_loader) + e.get_template("foo") + # This would raise NotADirectoryError if "t2/foo" wasn't skipped. + e.get_template("foo/test.html") + + def test_choice_loader(self, choice_loader): + env = Environment(loader=choice_loader) + tmpl = env.get_template("justdict.html") + assert tmpl.render().strip() == "FOO" + tmpl = env.get_template("test.html") + assert tmpl.render().strip() == "BAR" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + def test_function_loader(self, function_loader): + env = Environment(loader=function_loader) + tmpl = env.get_template("justfunction.html") + assert tmpl.render().strip() == "FOO" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + def test_prefix_loader(self, prefix_loader): + env = Environment(loader=prefix_loader) + tmpl = env.get_template("a/test.html") + assert tmpl.render().strip() == "BAR" + tmpl = env.get_template("b/justdict.html") + assert tmpl.render().strip() == "FOO" + pytest.raises(TemplateNotFound, env.get_template, "missing") + + def test_caching(self): + changed = False + + class TestLoader(loaders.BaseLoader): + def get_source(self, environment, template): + return u"foo", None, lambda: not changed + + env = Environment(loader=TestLoader(), cache_size=-1) + tmpl = env.get_template("template") + assert tmpl is env.get_template("template") + changed = True + assert tmpl is not env.get_template("template") + changed = False + + def test_no_cache(self): + mapping = {"foo": "one"} + env = Environment(loader=loaders.DictLoader(mapping), cache_size=0) + assert env.get_template("foo") is not env.get_template("foo") + + def test_limited_size_cache(self): + mapping = {"one": "foo", "two": "bar", "three": "baz"} + loader = loaders.DictLoader(mapping) + env = Environment(loader=loader, cache_size=2) + t1 = env.get_template("one") + t2 = env.get_template("two") + assert t2 is env.get_template("two") + assert t1 is env.get_template("one") + env.get_template("three") + loader_ref = weakref.ref(loader) + assert (loader_ref, "one") in env.cache + assert (loader_ref, "two") not in env.cache + assert (loader_ref, "three") in env.cache + + def test_cache_loader_change(self): + loader1 = loaders.DictLoader({"foo": "one"}) + loader2 = loaders.DictLoader({"foo": "two"}) + env = Environment(loader=loader1, cache_size=2) + assert env.get_template("foo").render() == "one" + env.loader = loader2 + assert env.get_template("foo").render() == "two" + + def test_dict_loader_cache_invalidates(self): + mapping = {"foo": "one"} + env = Environment(loader=loaders.DictLoader(mapping)) + assert env.get_template("foo").render() == "one" + mapping["foo"] = "two" + assert env.get_template("foo").render() == "two" + + def test_split_template_path(self): + assert split_template_path("foo/bar") == ["foo", "bar"] + assert split_template_path("./foo/bar") == ["foo", "bar"] + pytest.raises(TemplateNotFound, split_template_path, "../foo") + + +class TestFileSystemLoader(object): + searchpath = os.path.join( + yc.test_source_path(), "res", "templates" + ) + + @staticmethod + def _test_common(env): + tmpl = env.get_template("test.html") + assert tmpl.render().strip() == "BAR" + tmpl = env.get_template("foo/test.html") + assert tmpl.render().strip() == "FOO" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + def test_searchpath_as_str(self): + filesystem_loader = loaders.FileSystemLoader(self.searchpath) + + env = Environment(loader=filesystem_loader) + self._test_common(env) + + @pytest.mark.skipif(PY2, reason="pathlib is not available in Python 2") + def test_searchpath_as_pathlib(self): + import pathlib + + searchpath = pathlib.Path(self.searchpath) + + filesystem_loader = loaders.FileSystemLoader(searchpath) + + env = Environment(loader=filesystem_loader) + self._test_common(env) + + @pytest.mark.skipif(PY2, reason="pathlib is not available in Python 2") + def test_searchpath_as_list_including_pathlib(self): + import pathlib + + searchpath = pathlib.Path(self.searchpath) + + filesystem_loader = loaders.FileSystemLoader(["/tmp/templates", searchpath]) + + env = Environment(loader=filesystem_loader) + self._test_common(env) + + @pytest.mark.skip("Arcadia read only") + def test_caches_template_based_on_mtime(self): + filesystem_loader = loaders.FileSystemLoader(self.searchpath) + + env = Environment(loader=filesystem_loader) + tmpl1 = env.get_template("test.html") + tmpl2 = env.get_template("test.html") + assert tmpl1 is tmpl2 + + os.utime(os.path.join(self.searchpath, "test.html"), (time.time(), time.time())) + tmpl3 = env.get_template("test.html") + assert tmpl1 is not tmpl3 + + @pytest.mark.parametrize( + ("encoding", "expect"), + [ + ("utf-8", u"文字化け"), + ("iso-8859-1", u"æ\x96\x87\xe5\xad\x97\xe5\x8c\x96\xe3\x81\x91"), + ], + ) + def test_uses_specified_encoding(self, encoding, expect): + loader = loaders.FileSystemLoader(self.searchpath, encoding=encoding) + e = Environment(loader=loader) + t = e.get_template("mojibake.txt") + assert t.render() == expect + + +class TestModuleLoader(object): + archive = None + + def compile_down(self, prefix_loader, zip="deflated", py_compile=False): + log = [] + self.reg_env = Environment(loader=prefix_loader) + if zip is not None: + fd, self.archive = tempfile.mkstemp(suffix=".zip") + os.close(fd) + else: + self.archive = tempfile.mkdtemp() + self.reg_env.compile_templates( + self.archive, zip=zip, log_function=log.append, py_compile=py_compile + ) + self.mod_env = Environment(loader=loaders.ModuleLoader(self.archive)) + return "".join(log) + + def teardown(self): + if hasattr(self, "mod_env"): + if os.path.isfile(self.archive): + os.remove(self.archive) + else: + shutil.rmtree(self.archive) + self.archive = None + + def test_log(self, prefix_loader): + log = self.compile_down(prefix_loader) + assert ( + 'Compiled "a/foo/test.html" as ' + "tmpl_a790caf9d669e39ea4d280d597ec891c4ef0404a" in log + ) + assert "Finished compiling templates" in log + assert ( + 'Could not compile "a/syntaxerror.html": ' + "Encountered unknown tag 'endif'" in log + ) + + def _test_common(self): + tmpl1 = self.reg_env.get_template("a/test.html") + tmpl2 = self.mod_env.get_template("a/test.html") + assert tmpl1.render() == tmpl2.render() + + tmpl1 = self.reg_env.get_template("b/justdict.html") + tmpl2 = self.mod_env.get_template("b/justdict.html") + assert tmpl1.render() == tmpl2.render() + + def test_deflated_zip_compile(self, prefix_loader): + self.compile_down(prefix_loader, zip="deflated") + self._test_common() + + def test_stored_zip_compile(self, prefix_loader): + self.compile_down(prefix_loader, zip="stored") + self._test_common() + + def test_filesystem_compile(self, prefix_loader): + self.compile_down(prefix_loader, zip=None) + self._test_common() + + def test_weak_references(self, prefix_loader): + self.compile_down(prefix_loader) + self.mod_env.get_template("a/test.html") + key = loaders.ModuleLoader.get_template_key("a/test.html") + name = self.mod_env.loader.module.__name__ + + assert hasattr(self.mod_env.loader.module, key) + assert name in sys.modules + + # unset all, ensure the module is gone from sys.modules + self.mod_env = None + + try: + import gc + + gc.collect() + except BaseException: + pass + + assert name not in sys.modules + + # This test only makes sense on non-pypy python 2 + @pytest.mark.skipif( + not (PY2 and not PYPY), reason="This test only makes sense on non-pypy python 2" + ) + def test_byte_compilation(self, prefix_loader): + log = self.compile_down(prefix_loader, py_compile=True) + assert 'Byte-compiled "a/test.html"' in log + self.mod_env.get_template("a/test.html") + mod = self.mod_env.loader.module.tmpl_3c4ddf650c1a73df961a6d3d2ce2752f1b8fd490 + assert mod.__file__.endswith(".pyc") + + def test_choice_loader(self, prefix_loader): + self.compile_down(prefix_loader) + self.mod_env.loader = loaders.ChoiceLoader( + [self.mod_env.loader, loaders.DictLoader({"DICT_SOURCE": "DICT_TEMPLATE"})] + ) + tmpl1 = self.mod_env.get_template("a/test.html") + assert tmpl1.render() == "BAR" + tmpl2 = self.mod_env.get_template("DICT_SOURCE") + assert tmpl2.render() == "DICT_TEMPLATE" + + def test_prefix_loader(self, prefix_loader): + self.compile_down(prefix_loader) + self.mod_env.loader = loaders.PrefixLoader( + { + "MOD": self.mod_env.loader, + "DICT": loaders.DictLoader({"test.html": "DICT_TEMPLATE"}), + } + ) + tmpl1 = self.mod_env.get_template("MOD/a/test.html") + assert tmpl1.render() == "BAR" + tmpl2 = self.mod_env.get_template("DICT/test.html") + assert tmpl2.render() == "DICT_TEMPLATE" + + @pytest.mark.skipif(PY2, reason="pathlib is not available in Python 2") + def test_path_as_pathlib(self, prefix_loader): + self.compile_down(prefix_loader) + + mod_path = self.mod_env.loader.module.__path__[0] + + import pathlib + + mod_loader = loaders.ModuleLoader(pathlib.Path(mod_path)) + self.mod_env = Environment(loader=mod_loader) + + self._test_common() + + @pytest.mark.skipif(PY2, reason="pathlib is not available in Python 2") + def test_supports_pathlib_in_list_of_paths(self, prefix_loader): + self.compile_down(prefix_loader) + + mod_path = self.mod_env.loader.module.__path__[0] + + import pathlib + + mod_loader = loaders.ModuleLoader([pathlib.Path(mod_path), "/tmp/templates"]) + self.mod_env = Environment(loader=mod_loader) + + self._test_common() diff --git a/contrib/python/Jinja2/py2/tests/test_nativetypes.py b/contrib/python/Jinja2/py2/tests/test_nativetypes.py new file mode 100644 index 0000000000..76871ab5de --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_nativetypes.py @@ -0,0 +1,148 @@ +import pytest + +from jinja2._compat import text_type +from jinja2.exceptions import UndefinedError +from jinja2.nativetypes import NativeEnvironment +from jinja2.nativetypes import NativeTemplate +from jinja2.runtime import Undefined + + +@pytest.fixture +def env(): + return NativeEnvironment() + + +def test_is_defined_native_return(env): + t = env.from_string("{{ missing is defined }}") + assert not t.render() + + +def test_undefined_native_return(env): + t = env.from_string("{{ missing }}") + assert isinstance(t.render(), Undefined) + + +def test_adding_undefined_native_return(env): + t = env.from_string("{{ 3 + missing }}") + + with pytest.raises(UndefinedError): + t.render() + + +def test_cast_int(env): + t = env.from_string("{{ value|int }}") + result = t.render(value="3") + assert isinstance(result, int) + assert result == 3 + + +def test_list_add(env): + t = env.from_string("{{ a + b }}") + result = t.render(a=["a", "b"], b=["c", "d"]) + assert isinstance(result, list) + assert result == ["a", "b", "c", "d"] + + +def test_multi_expression_add(env): + t = env.from_string("{{ a }} + {{ b }}") + result = t.render(a=["a", "b"], b=["c", "d"]) + assert not isinstance(result, list) + assert result == "['a', 'b'] + ['c', 'd']" + + +def test_loops(env): + t = env.from_string("{% for x in value %}{{ x }}{% endfor %}") + result = t.render(value=["a", "b", "c", "d"]) + assert isinstance(result, text_type) + assert result == "abcd" + + +def test_loops_with_ints(env): + t = env.from_string("{% for x in value %}{{ x }}{% endfor %}") + result = t.render(value=[1, 2, 3, 4]) + assert isinstance(result, int) + assert result == 1234 + + +def test_loop_look_alike(env): + t = env.from_string("{% for x in value %}{{ x }}{% endfor %}") + result = t.render(value=[1]) + assert isinstance(result, int) + assert result == 1 + + +@pytest.mark.parametrize( + ("source", "expect"), + ( + ("{{ value }}", True), + ("{{ value }}", False), + ("{{ 1 == 1 }}", True), + ("{{ 2 + 2 == 5 }}", False), + ("{{ None is none }}", True), + ("{{ '' == None }}", False), + ), +) +def test_booleans(env, source, expect): + t = env.from_string(source) + result = t.render(value=expect) + assert isinstance(result, bool) + assert result is expect + + +def test_variable_dunder(env): + t = env.from_string("{{ x.__class__ }}") + result = t.render(x=True) + assert isinstance(result, type) + + +def test_constant_dunder(env): + t = env.from_string("{{ true.__class__ }}") + result = t.render() + assert isinstance(result, type) + + +def test_constant_dunder_to_string(env): + t = env.from_string("{{ true.__class__|string }}") + result = t.render() + assert not isinstance(result, type) + assert result in {"<type 'bool'>", "<class 'bool'>"} + + +def test_string_literal_var(env): + t = env.from_string("[{{ 'all' }}]") + result = t.render() + assert isinstance(result, text_type) + assert result == "[all]" + + +def test_string_top_level(env): + t = env.from_string("'Jinja'") + result = t.render() + assert result == "Jinja" + + +def test_tuple_of_variable_strings(env): + t = env.from_string("'{{ a }}', 'data', '{{ b }}', b'{{ c }}'") + result = t.render(a=1, b=2, c="bytes") + assert isinstance(result, tuple) + assert result == ("1", "data", "2", b"bytes") + + +def test_concat_strings_with_quotes(env): + t = env.from_string("--host='{{ host }}' --user \"{{ user }}\"") + result = t.render(host="localhost", user="Jinja") + assert result == "--host='localhost' --user \"Jinja\"" + + +def test_no_intermediate_eval(env): + t = env.from_string("0.000{{ a }}") + result = t.render(a=7) + assert isinstance(result, float) + # If intermediate eval happened, 0.000 would render 0.0, then 7 + # would be appended, resulting in 0.07. + assert result < 0.007 # TODO use math.isclose in Python 3 + + +def test_spontaneous_env(): + t = NativeTemplate("{{ true }}") + assert isinstance(t.environment, NativeEnvironment) diff --git a/contrib/python/Jinja2/py2/tests/test_regression.py b/contrib/python/Jinja2/py2/tests/test_regression.py new file mode 100644 index 0000000000..c5a0d68068 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_regression.py @@ -0,0 +1,624 @@ +# -*- coding: utf-8 -*- +import sys + +import pytest + +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import PrefixLoader +from jinja2 import Template +from jinja2 import TemplateAssertionError +from jinja2 import TemplateNotFound +from jinja2 import TemplateSyntaxError +from jinja2._compat import text_type + + +class TestCorner(object): + def test_assigned_scoping(self, env): + t = env.from_string( + """ + {%- for item in (1, 2, 3, 4) -%} + [{{ item }}] + {%- endfor %} + {{- item -}} + """ + ) + assert t.render(item=42) == "[1][2][3][4]42" + + t = env.from_string( + """ + {%- for item in (1, 2, 3, 4) -%} + [{{ item }}] + {%- endfor %} + {%- set item = 42 %} + {{- item -}} + """ + ) + assert t.render() == "[1][2][3][4]42" + + t = env.from_string( + """ + {%- set item = 42 %} + {%- for item in (1, 2, 3, 4) -%} + [{{ item }}] + {%- endfor %} + {{- item -}} + """ + ) + assert t.render() == "[1][2][3][4]42" + + def test_closure_scoping(self, env): + t = env.from_string( + """ + {%- set wrapper = "<FOO>" %} + {%- for item in (1, 2, 3, 4) %} + {%- macro wrapper() %}[{{ item }}]{% endmacro %} + {{- wrapper() }} + {%- endfor %} + {{- wrapper -}} + """ + ) + assert t.render() == "[1][2][3][4]<FOO>" + + t = env.from_string( + """ + {%- for item in (1, 2, 3, 4) %} + {%- macro wrapper() %}[{{ item }}]{% endmacro %} + {{- wrapper() }} + {%- endfor %} + {%- set wrapper = "<FOO>" %} + {{- wrapper -}} + """ + ) + assert t.render() == "[1][2][3][4]<FOO>" + + t = env.from_string( + """ + {%- for item in (1, 2, 3, 4) %} + {%- macro wrapper() %}[{{ item }}]{% endmacro %} + {{- wrapper() }} + {%- endfor %} + {{- wrapper -}} + """ + ) + assert t.render(wrapper=23) == "[1][2][3][4]23" + + +class TestBug(object): + def test_keyword_folding(self, env): + env = Environment() + env.filters["testing"] = lambda value, some: value + some + assert ( + env.from_string("{{ 'test'|testing(some='stuff') }}").render() + == "teststuff" + ) + + def test_extends_output_bugs(self, env): + env = Environment( + loader=DictLoader({"parent.html": "(({% block title %}{% endblock %}))"}) + ) + + t = env.from_string( + '{% if expr %}{% extends "parent.html" %}{% endif %}' + "[[{% block title %}title{% endblock %}]]" + "{% for item in [1, 2, 3] %}({{ item }}){% endfor %}" + ) + assert t.render(expr=False) == "[[title]](1)(2)(3)" + assert t.render(expr=True) == "((title))" + + def test_urlize_filter_escaping(self, env): + tmpl = env.from_string('{{ "http://www.example.org/<foo"|urlize }}') + assert ( + tmpl.render() == '<a href="http://www.example.org/<foo" rel="noopener">' + "http://www.example.org/<foo</a>" + ) + + def test_loop_call_loop(self, env): + tmpl = env.from_string( + """ + + {% macro test() %} + {{ caller() }} + {% endmacro %} + + {% for num1 in range(5) %} + {% call test() %} + {% for num2 in range(10) %} + {{ loop.index }} + {% endfor %} + {% endcall %} + {% endfor %} + + """ + ) + + assert tmpl.render().split() == [text_type(x) for x in range(1, 11)] * 5 + + def test_weird_inline_comment(self, env): + env = Environment(line_statement_prefix="%") + pytest.raises( + TemplateSyntaxError, + env.from_string, + "% for item in seq {# missing #}\n...% endfor", + ) + + def test_old_macro_loop_scoping_bug(self, env): + tmpl = env.from_string( + "{% for i in (1, 2) %}{{ i }}{% endfor %}" + "{% macro i() %}3{% endmacro %}{{ i() }}" + ) + assert tmpl.render() == "123" + + def test_partial_conditional_assignments(self, env): + tmpl = env.from_string("{% if b %}{% set a = 42 %}{% endif %}{{ a }}") + assert tmpl.render(a=23) == "23" + assert tmpl.render(b=True) == "42" + + def test_stacked_locals_scoping_bug(self, env): + env = Environment(line_statement_prefix="#") + t = env.from_string( + """\ +# for j in [1, 2]: +# set x = 1 +# for i in [1, 2]: +# print x +# if i % 2 == 0: +# set x = x + 1 +# endif +# endfor +# endfor +# if a +# print 'A' +# elif b +# print 'B' +# elif c == d +# print 'C' +# else +# print 'D' +# endif + """ + ) + assert t.render(a=0, b=False, c=42, d=42.0) == "1111C" + + def test_stacked_locals_scoping_bug_twoframe(self, env): + t = Template( + """ + {% set x = 1 %} + {% for item in foo %} + {% if item == 1 %} + {% set x = 2 %} + {% endif %} + {% endfor %} + {{ x }} + """ + ) + rv = t.render(foo=[1]).strip() + assert rv == u"1" + + def test_call_with_args(self, env): + t = Template( + """{% macro dump_users(users) -%} + <ul> + {%- for user in users -%} + <li><p>{{ user.username|e }}</p>{{ caller(user) }}</li> + {%- endfor -%} + </ul> + {%- endmacro -%} + + {% call(user) dump_users(list_of_user) -%} + <dl> + <dl>Realname</dl> + <dd>{{ user.realname|e }}</dd> + <dl>Description</dl> + <dd>{{ user.description }}</dd> + </dl> + {% endcall %}""" + ) + + assert [ + x.strip() + for x in t.render( + list_of_user=[ + { + "username": "apo", + "realname": "something else", + "description": "test", + } + ] + ).splitlines() + ] == [ + u"<ul><li><p>apo</p><dl>", + u"<dl>Realname</dl>", + u"<dd>something else</dd>", + u"<dl>Description</dl>", + u"<dd>test</dd>", + u"</dl>", + u"</li></ul>", + ] + + def test_empty_if_condition_fails(self, env): + pytest.raises(TemplateSyntaxError, Template, "{% if %}....{% endif %}") + pytest.raises( + TemplateSyntaxError, Template, "{% if foo %}...{% elif %}...{% endif %}" + ) + pytest.raises(TemplateSyntaxError, Template, "{% for x in %}..{% endfor %}") + + def test_recursive_loop_compile(self, env): + Template( + """ + {% for p in foo recursive%} + {{p.bar}} + {% for f in p.fields recursive%} + {{f.baz}} + {{p.bar}} + {% if f.rec %} + {{ loop(f.sub) }} + {% endif %} + {% endfor %} + {% endfor %} + """ + ) + Template( + """ + {% for p in foo%} + {{p.bar}} + {% for f in p.fields recursive%} + {{f.baz}} + {{p.bar}} + {% if f.rec %} + {{ loop(f.sub) }} + {% endif %} + {% endfor %} + {% endfor %} + """ + ) + + def test_else_loop_bug(self, env): + t = Template( + """ + {% for x in y %} + {{ loop.index0 }} + {% else %} + {% for i in range(3) %}{{ i }}{% endfor %} + {% endfor %} + """ + ) + assert t.render(y=[]).strip() == "012" + + def test_correct_prefix_loader_name(self, env): + env = Environment(loader=PrefixLoader({"foo": DictLoader({})})) + with pytest.raises(TemplateNotFound) as e: + env.get_template("foo/bar.html") + + assert e.value.name == "foo/bar.html" + + def test_contextfunction_callable_classes(self, env): + from jinja2.utils import contextfunction + + class CallableClass(object): + @contextfunction + def __call__(self, ctx): + return ctx.resolve("hello") + + tpl = Template("""{{ callableclass() }}""") + output = tpl.render(callableclass=CallableClass(), hello="TEST") + expected = "TEST" + + assert output == expected + + @pytest.mark.skipif(sys.version_info[0] > 2, reason="This only works on 2.x") + def test_old_style_attribute(self, env): + class Foo: + x = 42 + + assert env.getitem(Foo(), "x") == 42 + + def test_block_set_with_extends(self): + env = Environment( + loader=DictLoader({"main": "{% block body %}[{{ x }}]{% endblock %}"}) + ) + t = env.from_string('{% extends "main" %}{% set x %}42{% endset %}') + assert t.render() == "[42]" + + def test_nested_for_else(self, env): + tmpl = env.from_string( + "{% for x in y %}{{ loop.index0 }}{% else %}" + "{% for i in range(3) %}{{ i }}{% endfor %}" + "{% endfor %}" + ) + assert tmpl.render() == "012" + + def test_macro_var_bug(self, env): + tmpl = env.from_string( + """ + {% set i = 1 %} + {% macro test() %} + {% for i in range(0, 10) %}{{ i }}{% endfor %} + {% endmacro %}{{ test() }} + """ + ) + assert tmpl.render().strip() == "0123456789" + + def test_macro_var_bug_advanced(self, env): + tmpl = env.from_string( + """ + {% macro outer() %} + {% set i = 1 %} + {% macro test() %} + {% for i in range(0, 10) %}{{ i }}{% endfor %} + {% endmacro %}{{ test() }} + {% endmacro %}{{ outer() }} + """ + ) + assert tmpl.render().strip() == "0123456789" + + def test_callable_defaults(self): + env = Environment() + env.globals["get_int"] = lambda: 42 + t = env.from_string( + """ + {% macro test(a, b, c=get_int()) -%} + {{ a + b + c }} + {%- endmacro %} + {{ test(1, 2) }}|{{ test(1, 2, 3) }} + """ + ) + assert t.render().strip() == "45|6" + + def test_macro_escaping(self): + env = Environment( + autoescape=lambda x: False, extensions=["jinja2.ext.autoescape"] + ) + template = "{% macro m() %}<html>{% endmacro %}" + template += "{% autoescape true %}{{ m() }}{% endautoescape %}" + assert env.from_string(template).render() + + def test_macro_scoping(self, env): + tmpl = env.from_string( + """ + {% set n=[1,2,3,4,5] %} + {% for n in [[1,2,3], [3,4,5], [5,6,7]] %} + + {% macro x(l) %} + {{ l.pop() }} + {% if l %}{{ x(l) }}{% endif %} + {% endmacro %} + + {{ x(n) }} + + {% endfor %} + """ + ) + assert list(map(int, tmpl.render().split())) == [3, 2, 1, 5, 4, 3, 7, 6, 5] + + def test_scopes_and_blocks(self): + env = Environment( + loader=DictLoader( + { + "a.html": """ + {%- set foo = 'bar' -%} + {% include 'x.html' -%} + """, + "b.html": """ + {%- set foo = 'bar' -%} + {% block test %}{% include 'x.html' %}{% endblock -%} + """, + "c.html": """ + {%- set foo = 'bar' -%} + {% block test %}{% set foo = foo + %}{% include 'x.html' %}{% endblock -%} + """, + "x.html": """{{ foo }}|{{ test }}""", + } + ) + ) + + a = env.get_template("a.html") + b = env.get_template("b.html") + c = env.get_template("c.html") + + assert a.render(test="x").strip() == "bar|x" + assert b.render(test="x").strip() == "bar|x" + assert c.render(test="x").strip() == "bar|x" + + def test_scopes_and_include(self): + env = Environment( + loader=DictLoader( + { + "include.html": "{{ var }}", + "base.html": '{% include "include.html" %}', + "child.html": '{% extends "base.html" %}{% set var = 42 %}', + } + ) + ) + t = env.get_template("child.html") + assert t.render() == "42" + + def test_caller_scoping(self, env): + t = env.from_string( + """ + {% macro detail(icon, value) -%} + {% if value -%} + <p><span class="fa fa-fw fa-{{ icon }}"></span> + {%- if caller is undefined -%} + {{ value }} + {%- else -%} + {{ caller(value, *varargs) }} + {%- endif -%}</p> + {%- endif %} + {%- endmacro %} + + + {% macro link_detail(icon, value, href) -%} + {% call(value, href) detail(icon, value, href) -%} + <a href="{{ href }}">{{ value }}</a> + {%- endcall %} + {%- endmacro %} + """ + ) + + assert t.module.link_detail("circle", "Index", "/") == ( + '<p><span class="fa fa-fw fa-circle"></span><a href="/">Index</a></p>' + ) + + def test_variable_reuse(self, env): + t = env.from_string("{% for x in x.y %}{{ x }}{% endfor %}") + assert t.render(x={"y": [0, 1, 2]}) == "012" + + t = env.from_string("{% for x in x.y %}{{ loop.index0 }}|{{ x }}{% endfor %}") + assert t.render(x={"y": [0, 1, 2]}) == "0|01|12|2" + + t = env.from_string("{% for x in x.y recursive %}{{ x }}{% endfor %}") + assert t.render(x={"y": [0, 1, 2]}) == "012" + + def test_double_caller(self, env): + t = env.from_string( + "{% macro x(caller=none) %}[{% if caller %}" + "{{ caller() }}{% endif %}]{% endmacro %}" + "{{ x() }}{% call x() %}aha!{% endcall %}" + ) + assert t.render() == "[][aha!]" + + def test_double_caller_no_default(self, env): + with pytest.raises(TemplateAssertionError) as exc_info: + env.from_string( + "{% macro x(caller) %}[{% if caller %}" + "{{ caller() }}{% endif %}]{% endmacro %}" + ) + assert exc_info.match( + r'"caller" argument must be omitted or ' r"be given a default" + ) + + t = env.from_string( + "{% macro x(caller=none) %}[{% if caller %}" + "{{ caller() }}{% endif %}]{% endmacro %}" + ) + with pytest.raises(TypeError) as exc_info: + t.module.x(None, caller=lambda: 42) + assert exc_info.match( + r"\'x\' was invoked with two values for the " r"special caller argument" + ) + + def test_macro_blocks(self, env): + t = env.from_string( + "{% macro x() %}{% block foo %}x{% endblock %}{% endmacro %}{{ x() }}" + ) + assert t.render() == "x" + + def test_scoped_block(self, env): + t = env.from_string( + "{% set x = 1 %}{% with x = 2 %}{% block y scoped %}" + "{{ x }}{% endblock %}{% endwith %}" + ) + assert t.render() == "2" + + def test_recursive_loop_filter(self, env): + t = env.from_string( + """ + <?xml version="1.0" encoding="UTF-8"?> + <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> + {%- for page in [site.root] if page.url != this recursive %} + <url><loc>{{ page.url }}</loc></url> + {{- loop(page.children) }} + {%- endfor %} + </urlset> + """ + ) + sm = t.render( + this="/foo", + site={"root": {"url": "/", "children": [{"url": "/foo"}, {"url": "/bar"}]}}, + ) + lines = [x.strip() for x in sm.splitlines() if x.strip()] + assert lines == [ + '<?xml version="1.0" encoding="UTF-8"?>', + '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">', + "<url><loc>/</loc></url>", + "<url><loc>/bar</loc></url>", + "</urlset>", + ] + + def test_empty_if(self, env): + t = env.from_string("{% if foo %}{% else %}42{% endif %}") + assert t.render(foo=False) == "42" + + def test_subproperty_if(self, env): + t = env.from_string( + "{% if object1.subproperty1 is eq object2.subproperty2 %}42{% endif %}" + ) + assert ( + t.render( + object1={"subproperty1": "value"}, object2={"subproperty2": "value"} + ) + == "42" + ) + + def test_set_and_include(self): + env = Environment( + loader=DictLoader( + { + "inc": "bar", + "main": '{% set foo = "foo" %}{{ foo }}{% include "inc" %}', + } + ) + ) + assert env.get_template("main").render() == "foobar" + + def test_loop_include(self): + env = Environment( + loader=DictLoader( + { + "inc": "{{ i }}", + "main": '{% for i in [1, 2, 3] %}{% include "inc" %}{% endfor %}', + } + ) + ) + assert env.get_template("main").render() == "123" + + def test_grouper_repr(self): + from jinja2.filters import _GroupTuple + + t = _GroupTuple("foo", [1, 2]) + assert t.grouper == "foo" + assert t.list == [1, 2] + assert repr(t) == "('foo', [1, 2])" + assert str(t) == "('foo', [1, 2])" + + def test_custom_context(self, env): + from jinja2.runtime import Context + + class MyContext(Context): + pass + + class MyEnvironment(Environment): + context_class = MyContext + + loader = DictLoader({"base": "{{ foobar }}", "test": '{% extends "base" %}'}) + env = MyEnvironment(loader=loader) + assert env.get_template("test").render(foobar="test") == "test" + + def test_legacy_custom_context(self, env): + from jinja2.runtime import Context, missing + + class MyContext(Context): + def resolve(self, name): + if name == "foo": + return 42 + return super(MyContext, self).resolve(name) + + x = MyContext(env, parent={"bar": 23}, name="foo", blocks={}) + assert x._legacy_resolve_mode + assert x.resolve_or_missing("foo") == 42 + assert x.resolve_or_missing("bar") == 23 + assert x.resolve_or_missing("baz") is missing + + def test_recursive_loop_bug(self, env): + tmpl = env.from_string( + "{%- for value in values recursive %}1{% else %}0{% endfor -%}" + ) + assert tmpl.render(values=[]) == "0" + + def test_markup_and_chainable_undefined(self): + from jinja2 import Markup + from jinja2.runtime import ChainableUndefined + + assert str(Markup(ChainableUndefined())) == "" diff --git a/contrib/python/Jinja2/py2/tests/test_runtime.py b/contrib/python/Jinja2/py2/tests/test_runtime.py new file mode 100644 index 0000000000..5e4686c083 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_runtime.py @@ -0,0 +1,75 @@ +import itertools + +from jinja2 import Template +from jinja2.runtime import LoopContext + +TEST_IDX_TEMPLATE_STR_1 = ( + "[{% for i in lst|reverse %}(len={{ loop.length }}," + " revindex={{ loop.revindex }}, index={{ loop.index }}, val={{ i }}){% endfor %}]" +) +TEST_IDX0_TEMPLATE_STR_1 = ( + "[{% for i in lst|reverse %}(len={{ loop.length }}," + " revindex0={{ loop.revindex0 }}, index0={{ loop.index0 }}, val={{ i }})" + "{% endfor %}]" +) + + +def test_loop_idx(): + t = Template(TEST_IDX_TEMPLATE_STR_1) + lst = [10] + excepted_render = "[(len=1, revindex=1, index=1, val=10)]" + assert excepted_render == t.render(lst=lst) + + +def test_loop_idx0(): + t = Template(TEST_IDX0_TEMPLATE_STR_1) + lst = [10] + excepted_render = "[(len=1, revindex0=0, index0=0, val=10)]" + assert excepted_render == t.render(lst=lst) + + +def test_loopcontext0(): + in_lst = [] + lc = LoopContext(reversed(in_lst), None) + assert lc.length == len(in_lst) + + +def test_loopcontext1(): + in_lst = [10] + lc = LoopContext(reversed(in_lst), None) + assert lc.length == len(in_lst) + + +def test_loopcontext2(): + in_lst = [10, 11] + lc = LoopContext(reversed(in_lst), None) + assert lc.length == len(in_lst) + + +def test_iterator_not_advanced_early(): + t = Template("{% for _, g in gs %}{{ loop.index }} {{ g|list }}\n{% endfor %}") + out = t.render( + gs=itertools.groupby([(1, "a"), (1, "b"), (2, "c"), (3, "d")], lambda x: x[0]) + ) + # groupby groups depend on the current position of the iterator. If + # it was advanced early, the lists would appear empty. + assert out == "1 [(1, 'a'), (1, 'b')]\n2 [(2, 'c')]\n3 [(3, 'd')]\n" + + +def test_mock_not_contextfunction(): + """If a callable class has a ``__getattr__`` that returns True-like + values for arbitrary attrs, it should not be incorrectly identified + as a ``contextfunction``. + """ + + class Calc(object): + def __getattr__(self, item): + return object() + + def __call__(self, *args, **kwargs): + return len(args) + len(kwargs) + + t = Template("{{ calc() }}") + out = t.render(calc=Calc()) + # Would be "1" if context argument was passed. + assert out == "0" diff --git a/contrib/python/Jinja2/py2/tests/test_security.py b/contrib/python/Jinja2/py2/tests/test_security.py new file mode 100644 index 0000000000..7e8974c891 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_security.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- +import pytest + +from jinja2 import Environment +from jinja2 import escape +from jinja2 import Markup +from jinja2._compat import text_type +from jinja2.exceptions import SecurityError +from jinja2.exceptions import TemplateRuntimeError +from jinja2.exceptions import TemplateSyntaxError +from jinja2.nodes import EvalContext +from jinja2.sandbox import ImmutableSandboxedEnvironment +from jinja2.sandbox import SandboxedEnvironment +from jinja2.sandbox import unsafe + + +class PrivateStuff(object): + def bar(self): + return 23 + + @unsafe + def foo(self): + return 42 + + def __repr__(self): + return "PrivateStuff" + + +class PublicStuff(object): + def bar(self): + return 23 + + def _foo(self): + return 42 + + def __repr__(self): + return "PublicStuff" + + +class TestSandbox(object): + def test_unsafe(self, env): + env = SandboxedEnvironment() + pytest.raises( + SecurityError, env.from_string("{{ foo.foo() }}").render, foo=PrivateStuff() + ) + assert env.from_string("{{ foo.bar() }}").render(foo=PrivateStuff()) == "23" + + pytest.raises( + SecurityError, env.from_string("{{ foo._foo() }}").render, foo=PublicStuff() + ) + assert env.from_string("{{ foo.bar() }}").render(foo=PublicStuff()) == "23" + assert env.from_string("{{ foo.__class__ }}").render(foo=42) == "" + assert env.from_string("{{ foo.func_code }}").render(foo=lambda: None) == "" + # security error comes from __class__ already. + pytest.raises( + SecurityError, + env.from_string("{{ foo.__class__.__subclasses__() }}").render, + foo=42, + ) + + def test_immutable_environment(self, env): + env = ImmutableSandboxedEnvironment() + pytest.raises(SecurityError, env.from_string("{{ [].append(23) }}").render) + pytest.raises(SecurityError, env.from_string("{{ {1:2}.clear() }}").render) + + def test_restricted(self, env): + env = SandboxedEnvironment() + pytest.raises( + TemplateSyntaxError, + env.from_string, + "{% for item.attribute in seq %}...{% endfor %}", + ) + pytest.raises( + TemplateSyntaxError, + env.from_string, + "{% for foo, bar.baz in seq %}...{% endfor %}", + ) + + def test_markup_operations(self, env): + # adding two strings should escape the unsafe one + unsafe = '<script type="application/x-some-script">alert("foo");</script>' + safe = Markup("<em>username</em>") + assert unsafe + safe == text_type(escape(unsafe)) + text_type(safe) + + # string interpolations are safe to use too + assert Markup("<em>%s</em>") % "<bad user>" == "<em><bad user></em>" + assert ( + Markup("<em>%(username)s</em>") % {"username": "<bad user>"} + == "<em><bad user></em>" + ) + + # an escaped object is markup too + assert type(Markup("foo") + "bar") is Markup + + # and it implements __html__ by returning itself + x = Markup("foo") + assert x.__html__() is x + + # it also knows how to treat __html__ objects + class Foo(object): + def __html__(self): + return "<em>awesome</em>" + + def __unicode__(self): + return "awesome" + + assert Markup(Foo()) == "<em>awesome</em>" + assert ( + Markup("<strong>%s</strong>") % Foo() == "<strong><em>awesome</em></strong>" + ) + + # escaping and unescaping + assert escape("\"<>&'") == ""<>&'" + assert Markup("<em>Foo & Bar</em>").striptags() == "Foo & Bar" + assert Markup("<test>").unescape() == "<test>" + + def test_template_data(self, env): + env = Environment(autoescape=True) + t = env.from_string( + "{% macro say_hello(name) %}" + "<p>Hello {{ name }}!</p>{% endmacro %}" + '{{ say_hello("<blink>foo</blink>") }}' + ) + escaped_out = "<p>Hello <blink>foo</blink>!</p>" + assert t.render() == escaped_out + assert text_type(t.module) == escaped_out + assert escape(t.module) == escaped_out + assert t.module.say_hello("<blink>foo</blink>") == escaped_out + assert ( + escape(t.module.say_hello(EvalContext(env), "<blink>foo</blink>")) + == escaped_out + ) + assert escape(t.module.say_hello("<blink>foo</blink>")) == escaped_out + + def test_attr_filter(self, env): + env = SandboxedEnvironment() + tmpl = env.from_string('{{ cls|attr("__subclasses__")() }}') + pytest.raises(SecurityError, tmpl.render, cls=int) + + def test_binary_operator_intercepting(self, env): + def disable_op(left, right): + raise TemplateRuntimeError("that operator so does not work") + + for expr, ctx, rv in ("1 + 2", {}, "3"), ("a + 2", {"a": 2}, "4"): + env = SandboxedEnvironment() + env.binop_table["+"] = disable_op + t = env.from_string("{{ %s }}" % expr) + assert t.render(ctx) == rv + env.intercepted_binops = frozenset(["+"]) + t = env.from_string("{{ %s }}" % expr) + with pytest.raises(TemplateRuntimeError): + t.render(ctx) + + def test_unary_operator_intercepting(self, env): + def disable_op(arg): + raise TemplateRuntimeError("that operator so does not work") + + for expr, ctx, rv in ("-1", {}, "-1"), ("-a", {"a": 2}, "-2"): + env = SandboxedEnvironment() + env.unop_table["-"] = disable_op + t = env.from_string("{{ %s }}" % expr) + assert t.render(ctx) == rv + env.intercepted_unops = frozenset(["-"]) + t = env.from_string("{{ %s }}" % expr) + with pytest.raises(TemplateRuntimeError): + t.render(ctx) + + +class TestStringFormat(object): + def test_basic_format_safety(self): + env = SandboxedEnvironment() + t = env.from_string('{{ "a{0.__class__}b".format(42) }}') + assert t.render() == "ab" + + def test_basic_format_all_okay(self): + env = SandboxedEnvironment() + t = env.from_string('{{ "a{0.foo}b".format({"foo": 42}) }}') + assert t.render() == "a42b" + + def test_safe_format_safety(self): + env = SandboxedEnvironment() + t = env.from_string('{{ ("a{0.__class__}b{1}"|safe).format(42, "<foo>") }}') + assert t.render() == "ab<foo>" + + def test_safe_format_all_okay(self): + env = SandboxedEnvironment() + t = env.from_string('{{ ("a{0.foo}b{1}"|safe).format({"foo": 42}, "<foo>") }}') + assert t.render() == "a42b<foo>" + + +@pytest.mark.skipif( + not hasattr(str, "format_map"), reason="requires str.format_map method" +) +class TestStringFormatMap(object): + def test_basic_format_safety(self): + env = SandboxedEnvironment() + t = env.from_string('{{ "a{x.__class__}b".format_map({"x":42}) }}') + assert t.render() == "ab" + + def test_basic_format_all_okay(self): + env = SandboxedEnvironment() + t = env.from_string('{{ "a{x.foo}b".format_map({"x":{"foo": 42}}) }}') + assert t.render() == "a42b" + + def test_safe_format_all_okay(self): + env = SandboxedEnvironment() + t = env.from_string( + '{{ ("a{x.foo}b{y}"|safe).format_map({"x":{"foo": 42}, "y":"<foo>"}) }}' + ) + assert t.render() == "a42b<foo>" diff --git a/contrib/python/Jinja2/py2/tests/test_tests.py b/contrib/python/Jinja2/py2/tests/test_tests.py new file mode 100644 index 0000000000..b903e3b1e2 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_tests.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +import pytest + +from jinja2 import Environment +from jinja2 import Markup + + +class MyDict(dict): + pass + + +class TestTestsCase(object): + def test_defined(self, env): + tmpl = env.from_string("{{ missing is defined }}|{{ true is defined }}") + assert tmpl.render() == "False|True" + + def test_even(self, env): + tmpl = env.from_string("""{{ 1 is even }}|{{ 2 is even }}""") + assert tmpl.render() == "False|True" + + def test_odd(self, env): + tmpl = env.from_string("""{{ 1 is odd }}|{{ 2 is odd }}""") + assert tmpl.render() == "True|False" + + def test_lower(self, env): + tmpl = env.from_string("""{{ "foo" is lower }}|{{ "FOO" is lower }}""") + assert tmpl.render() == "True|False" + + # Test type checks + @pytest.mark.parametrize( + "op,expect", + ( + ("none is none", True), + ("false is none", False), + ("true is none", False), + ("42 is none", False), + ("none is true", False), + ("false is true", False), + ("true is true", True), + ("0 is true", False), + ("1 is true", False), + ("42 is true", False), + ("none is false", False), + ("false is false", True), + ("true is false", False), + ("0 is false", False), + ("1 is false", False), + ("42 is false", False), + ("none is boolean", False), + ("false is boolean", True), + ("true is boolean", True), + ("0 is boolean", False), + ("1 is boolean", False), + ("42 is boolean", False), + ("0.0 is boolean", False), + ("1.0 is boolean", False), + ("3.14159 is boolean", False), + ("none is integer", False), + ("false is integer", False), + ("true is integer", False), + ("42 is integer", True), + ("3.14159 is integer", False), + ("(10 ** 100) is integer", True), + ("none is float", False), + ("false is float", False), + ("true is float", False), + ("42 is float", False), + ("4.2 is float", True), + ("(10 ** 100) is float", False), + ("none is number", False), + ("false is number", True), + ("true is number", True), + ("42 is number", True), + ("3.14159 is number", True), + ("complex is number", True), + ("(10 ** 100) is number", True), + ("none is string", False), + ("false is string", False), + ("true is string", False), + ("42 is string", False), + ('"foo" is string', True), + ("none is sequence", False), + ("false is sequence", False), + ("42 is sequence", False), + ('"foo" is sequence', True), + ("[] is sequence", True), + ("[1, 2, 3] is sequence", True), + ("{} is sequence", True), + ("none is mapping", False), + ("false is mapping", False), + ("42 is mapping", False), + ('"foo" is mapping', False), + ("[] is mapping", False), + ("{} is mapping", True), + ("mydict is mapping", True), + ("none is iterable", False), + ("false is iterable", False), + ("42 is iterable", False), + ('"foo" is iterable', True), + ("[] is iterable", True), + ("{} is iterable", True), + ("range(5) is iterable", True), + ("none is callable", False), + ("false is callable", False), + ("42 is callable", False), + ('"foo" is callable', False), + ("[] is callable", False), + ("{} is callable", False), + ("range is callable", True), + ), + ) + def test_types(self, env, op, expect): + t = env.from_string("{{{{ {op} }}}}".format(op=op)) + assert t.render(mydict=MyDict(), complex=complex(1, 2)) == str(expect) + + def test_upper(self, env): + tmpl = env.from_string('{{ "FOO" is upper }}|{{ "foo" is upper }}') + assert tmpl.render() == "True|False" + + def test_equalto(self, env): + tmpl = env.from_string( + "{{ foo is eq 12 }}|" + "{{ foo is eq 0 }}|" + "{{ foo is eq (3 * 4) }}|" + '{{ bar is eq "baz" }}|' + '{{ bar is eq "zab" }}|' + '{{ bar is eq ("ba" + "z") }}|' + "{{ bar is eq bar }}|" + "{{ bar is eq foo }}" + ) + assert ( + tmpl.render(foo=12, bar="baz") + == "True|False|True|True|False|True|True|False" + ) + + @pytest.mark.parametrize( + "op,expect", + ( + ("eq 2", True), + ("eq 3", False), + ("ne 3", True), + ("ne 2", False), + ("lt 3", True), + ("lt 2", False), + ("le 2", True), + ("le 1", False), + ("gt 1", True), + ("gt 2", False), + ("ge 2", True), + ("ge 3", False), + ), + ) + def test_compare_aliases(self, env, op, expect): + t = env.from_string("{{{{ 2 is {op} }}}}".format(op=op)) + assert t.render() == str(expect) + + def test_sameas(self, env): + tmpl = env.from_string("{{ foo is sameas false }}|{{ 0 is sameas false }}") + assert tmpl.render(foo=False) == "True|False" + + def test_no_paren_for_arg1(self, env): + tmpl = env.from_string("{{ foo is sameas none }}") + assert tmpl.render(foo=None) == "True" + + def test_escaped(self, env): + env = Environment(autoescape=True) + tmpl = env.from_string("{{ x is escaped }}|{{ y is escaped }}") + assert tmpl.render(x="foo", y=Markup("foo")) == "False|True" + + def test_greaterthan(self, env): + tmpl = env.from_string("{{ 1 is greaterthan 0 }}|{{ 0 is greaterthan 1 }}") + assert tmpl.render() == "True|False" + + def test_lessthan(self, env): + tmpl = env.from_string("{{ 0 is lessthan 1 }}|{{ 1 is lessthan 0 }}") + assert tmpl.render() == "True|False" + + def test_multiple_tests(self): + items = [] + + def matching(x, y): + items.append((x, y)) + return False + + env = Environment() + env.tests["matching"] = matching + tmpl = env.from_string( + "{{ 'us-west-1' is matching '(us-east-1|ap-northeast-1)'" + " or 'stage' is matching '(dev|stage)' }}" + ) + assert tmpl.render() == "False" + assert items == [ + ("us-west-1", "(us-east-1|ap-northeast-1)"), + ("stage", "(dev|stage)"), + ] + + def test_in(self, env): + tmpl = env.from_string( + '{{ "o" is in "foo" }}|' + '{{ "foo" is in "foo" }}|' + '{{ "b" is in "foo" }}|' + "{{ 1 is in ((1, 2)) }}|" + "{{ 3 is in ((1, 2)) }}|" + "{{ 1 is in [1, 2] }}|" + "{{ 3 is in [1, 2] }}|" + '{{ "foo" is in {"foo": 1}}}|' + '{{ "baz" is in {"bar": 1}}}' + ) + assert tmpl.render() == "True|True|False|True|False|True|False|True|False" diff --git a/contrib/python/Jinja2/py2/tests/test_utils.py b/contrib/python/Jinja2/py2/tests/test_utils.py new file mode 100644 index 0000000000..d9f5bb2783 --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/test_utils.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +import pickle +import random +from collections import deque +from copy import copy as shallow_copy + +import pytest +from markupsafe import Markup + +from jinja2._compat import range_type +from jinja2._compat import string_types +from jinja2.utils import consume +from jinja2.utils import generate_lorem_ipsum +from jinja2.utils import LRUCache +from jinja2.utils import missing +from jinja2.utils import object_type_repr +from jinja2.utils import select_autoescape +from jinja2.utils import urlize + + +class TestLRUCache(object): + def test_simple(self): + d = LRUCache(3) + d["a"] = 1 + d["b"] = 2 + d["c"] = 3 + d["a"] + d["d"] = 4 + assert len(d) == 3 + assert "a" in d and "c" in d and "d" in d and "b" not in d + + def test_itervalue_deprecated(self): + cache = LRUCache(3) + cache["a"] = 1 + cache["b"] = 2 + with pytest.deprecated_call(): + cache.itervalue() + + def test_itervalues(self): + cache = LRUCache(3) + cache["b"] = 1 + cache["a"] = 2 + values = [v for v in cache.values()] + assert len(values) == 2 + assert 1 in values + assert 2 in values + + def test_itervalues_empty(self): + cache = LRUCache(2) + values = [v for v in cache.values()] + assert len(values) == 0 + + def test_pickleable(self): + cache = LRUCache(2) + cache["foo"] = 42 + cache["bar"] = 23 + cache["foo"] + + for protocol in range(3): + copy = pickle.loads(pickle.dumps(cache, protocol)) + assert copy.capacity == cache.capacity + assert copy._mapping == cache._mapping + assert copy._queue == cache._queue + + @pytest.mark.parametrize("copy_func", [LRUCache.copy, shallow_copy]) + def test_copy(self, copy_func): + cache = LRUCache(2) + cache["a"] = 1 + cache["b"] = 2 + copy = copy_func(cache) + assert copy._queue == cache._queue + copy["c"] = 3 + assert copy._queue != cache._queue + assert "a" not in copy and "b" in copy and "c" in copy + + def test_clear(self): + d = LRUCache(3) + d["a"] = 1 + d["b"] = 2 + d["c"] = 3 + d.clear() + assert d.__getstate__() == {"capacity": 3, "_mapping": {}, "_queue": deque([])} + + def test_repr(self): + d = LRUCache(3) + d["a"] = 1 + d["b"] = 2 + d["c"] = 3 + # Sort the strings - mapping is unordered + assert sorted(repr(d)) == sorted(u"<LRUCache {'a': 1, 'b': 2, 'c': 3}>") + + def test_items(self): + """Test various items, keys, values and iterators of LRUCache.""" + d = LRUCache(3) + d["a"] = 1 + d["b"] = 2 + d["c"] = 3 + assert d.items() == [("c", 3), ("b", 2), ("a", 1)] + assert d.keys() == ["c", "b", "a"] + assert d.values() == [3, 2, 1] + assert list(reversed(d)) == ["a", "b", "c"] + + # Change the cache a little + d["b"] + d["a"] = 4 + assert d.items() == [("a", 4), ("b", 2), ("c", 3)] + assert d.keys() == ["a", "b", "c"] + assert d.values() == [4, 2, 3] + assert list(reversed(d)) == ["c", "b", "a"] + + def test_setdefault(self): + d = LRUCache(3) + assert len(d) == 0 + assert d.setdefault("a") is None + assert d.setdefault("a", 1) is None + assert len(d) == 1 + assert d.setdefault("b", 2) == 2 + assert len(d) == 2 + + +class TestHelpers(object): + def test_object_type_repr(self): + class X(object): + pass + + assert object_type_repr(42) == "int object" + assert object_type_repr([]) == "list object" + assert object_type_repr(X()) == "__tests__.test_utils.X object" + assert object_type_repr(None) == "None" + assert object_type_repr(Ellipsis) == "Ellipsis" + + def test_autoescape_select(self): + func = select_autoescape( + enabled_extensions=("html", ".htm"), + disabled_extensions=("txt",), + default_for_string="STRING", + default="NONE", + ) + + assert func(None) == "STRING" + assert func("unknown.foo") == "NONE" + assert func("foo.html") + assert func("foo.htm") + assert not func("foo.txt") + assert func("FOO.HTML") + assert not func("FOO.TXT") + + +class TestEscapeUrlizeTarget(object): + def test_escape_urlize_target(self): + url = "http://example.org" + target = "<script>" + assert urlize(url, target=target) == ( + '<a href="http://example.org"' + ' target="<script>">' + "http://example.org</a>" + ) + + +class TestLoremIpsum(object): + def test_lorem_ipsum_markup(self): + """Test that output of lorem_ipsum is Markup by default.""" + assert isinstance(generate_lorem_ipsum(), Markup) + + def test_lorem_ipsum_html(self): + """Test that output of lorem_ipsum is a string_type when not html.""" + assert isinstance(generate_lorem_ipsum(html=False), string_types) + + def test_lorem_ipsum_n(self): + """Test that the n (number of lines) works as expected.""" + assert generate_lorem_ipsum(n=0, html=False) == u"" + for n in range_type(1, 50): + assert generate_lorem_ipsum(n=n, html=False).count("\n") == (n - 1) * 2 + + def test_lorem_ipsum_min(self): + """Test that at least min words are in the output of each line""" + for _ in range_type(5): + m = random.randrange(20, 99) + for _ in range_type(10): + assert generate_lorem_ipsum(n=1, min=m, html=False).count(" ") >= m - 1 + + def test_lorem_ipsum_max(self): + """Test that at least max words are in the output of each line""" + for _ in range_type(5): + m = random.randrange(21, 100) + for _ in range_type(10): + assert generate_lorem_ipsum(n=1, max=m, html=False).count(" ") < m - 1 + + +def test_missing(): + """Test the repr of missing.""" + assert repr(missing) == u"missing" + + +def test_consume(): + """Test that consume consumes an iterator.""" + x = iter([1, 2, 3, 4, 5]) + consume(x) + with pytest.raises(StopIteration): + next(x) diff --git a/contrib/python/Jinja2/py2/tests/ya.make b/contrib/python/Jinja2/py2/tests/ya.make new file mode 100644 index 0000000000..b0564740ae --- /dev/null +++ b/contrib/python/Jinja2/py2/tests/ya.make @@ -0,0 +1,50 @@ +PY2TEST() + +PEERDIR( + contrib/python/Jinja2 +) + +PY_SRCS( + TOP_LEVEL + res/__init__.py +) + +DATA( + arcadia/contrib/python/Jinja2/py2/tests/res +) + +RESOURCE_FILES( + PREFIX contrib/python/Jinja2/py2/tests/ + res/templates/broken.html + res/templates/foo/test.html + res/templates/mojibake.txt + res/templates/syntaxerror.html + res/templates/test.html + res/templates2/foo +) + +TEST_SRCS( + conftest.py + test_api.py + test_bytecode_cache.py + test_core_tags.py + test_debug.py + test_ext.py + test_features.py + test_filters.py + test_idtracking.py + test_imports.py + test_inheritance.py + test_lexnparse.py + test_loader.py + test_nativetypes.py + test_regression.py + test_runtime.py + test_security.py + test_tests.py + test_utils.py +) + +NO_LINT() + +END() diff --git a/contrib/python/Jinja2/py2/ya.make b/contrib/python/Jinja2/py2/ya.make new file mode 100644 index 0000000000..8641552007 --- /dev/null +++ b/contrib/python/Jinja2/py2/ya.make @@ -0,0 +1,56 @@ +# Generated by devtools/yamaker (pypi). + +PY2_LIBRARY() + +VERSION(2.11.3) + +LICENSE(BSD-3-Clause) + +PEERDIR( + contrib/python/MarkupSafe + contrib/python/setuptools +) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + jinja2/__init__.py + jinja2/_compat.py + jinja2/_identifier.py + jinja2/bccache.py + jinja2/compiler.py + jinja2/constants.py + jinja2/debug.py + jinja2/defaults.py + jinja2/environment.py + jinja2/exceptions.py + jinja2/ext.py + jinja2/filters.py + jinja2/idtracking.py + jinja2/lexer.py + jinja2/loaders.py + jinja2/meta.py + jinja2/nativetypes.py + jinja2/nodes.py + jinja2/optimizer.py + jinja2/parser.py + jinja2/runtime.py + jinja2/sandbox.py + jinja2/tests.py + jinja2/utils.py + jinja2/visitor.py +) + +RESOURCE_FILES( + PREFIX contrib/python/Jinja2/py2/ + .dist-info/METADATA + .dist-info/entry_points.txt + .dist-info/top_level.txt +) + +END() + +RECURSE_FOR_TESTS( + tests +) |