From f5cb50f79af310b917e6932a0c0d8e9a73261b7f Mon Sep 17 00:00:00 2001
From: Matt Martz <matt@sivel.net>
Date: Thu, 18 Jan 2024 17:17:23 -0600
Subject: [PATCH 2/2] [stable-2.14] Ensure ANSIBLE_NO_LOG is respected
 (CVE-2024-0690) (#82565) (#82568)

(cherry picked from commit 6935c8e)

Force template module to use non-native Jinja2 (#68560)

Fixes #46169

Auto unroll generators produced by jinja filters (#68014)

* Auto unroll generators produced by jinja filters

* Unroll for native in finalize

* Fix indentation

Co-authored-by: Sam Doran <sdoran@redhat.com>

* Add changelog fragment

* ci_complete

* Always unroll regardless of jinja2

* ci_complete

Co-authored-by: Sam Doran <sdoran@redhat.com>

Skip literal_eval for string filters results in native jinja. (#70988) (#71313)

Fixes #70831

(cherry picked from commit b66d66027ece03f3f0a3fdb5fd6b8213965a2f1d)

Introduce context manager for temporary templar context changes (#60513)

* Introduce context manager for temporary templar context changes. Fixes #60106

* Rename and docstring

* Make set_temporary_context more generic, don't hardcode each thing you can set, apply to template action too

* not None

* linting fix

* Ignore invalid attrs

* Catch the right things, loop the right things

* Use set_temporary_context in a few extra action plugins
---
 .../46169-non-native-template-module.yml      |   2 +
 .../60106-templar-contextmanager.yml          |   4 +
 .../68014-auto-unroll-jinja2-generators.yml   |   3 +
 ...iteral_eval-string-filter-native-jinja.yml |   2 +
 changelogs/fragments/cve-2024-0690.yml        |   2 +
 lib/ansible/config/base.yml                   |   2 +-
 lib/ansible/playbook/base.py                  |   2 +-
 lib/ansible/playbook/conditional.py           |   4 +-
 lib/ansible/playbook/play_context.py          |   4 -
 lib/ansible/plugins/action/ce_template.py     |   4 +-
 lib/ansible/plugins/action/network.py         |   4 +-
 lib/ansible/plugins/action/template.py        |  32 ++-
 lib/ansible/plugins/lookup/template.py        |  50 ++--
 lib/ansible/template/__init__.py              | 227 ++++++++++++++++--
 lib/ansible/template/native_helpers.py        |  39 +++
 lib/ansible/utils/native_jinja.py             |  13 +
 lib/ansible/utils/unsafe_proxy.py             |   7 +
 .../jinja2_native_types/test_casting.yml      |   7 +
 .../jinja2_native_types/test_dunder.yml       |   2 +-
 .../targets/no_log/no_log_config.yml          |  13 +
 test/integration/targets/no_log/runme.sh      |   5 +
 .../template_jinja2_non_native/46169.yml      |  32 +++
 .../template_jinja2_non_native/aliases        |   1 +
 .../template_jinja2_non_native/runme.sh       |   7 +
 .../templates/46169.json.j2                   |   3 +
 25 files changed, 398 insertions(+), 73 deletions(-)
 create mode 100644 changelogs/fragments/46169-non-native-template-module.yml
 create mode 100644 changelogs/fragments/60106-templar-contextmanager.yml
 create mode 100644 changelogs/fragments/68014-auto-unroll-jinja2-generators.yml
 create mode 100644 changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml
 create mode 100644 changelogs/fragments/cve-2024-0690.yml
 create mode 100644 lib/ansible/utils/native_jinja.py
 create mode 100644 test/integration/targets/no_log/no_log_config.yml
 create mode 100644 test/integration/targets/template_jinja2_non_native/46169.yml
 create mode 100644 test/integration/targets/template_jinja2_non_native/aliases
 create mode 100755 test/integration/targets/template_jinja2_non_native/runme.sh
 create mode 100644 test/integration/targets/template_jinja2_non_native/templates/46169.json.j2

diff --git a/changelogs/fragments/46169-non-native-template-module.yml b/changelogs/fragments/46169-non-native-template-module.yml
new file mode 100644
index 0000000000..7d004a6296
--- /dev/null
+++ b/changelogs/fragments/46169-non-native-template-module.yml
@@ -0,0 +1,2 @@
+minor_changes:
+  - Force the template module to use non-native Jinja2 (https://github.com/ansible/ansible/issues/46169)
diff --git a/changelogs/fragments/60106-templar-contextmanager.yml b/changelogs/fragments/60106-templar-contextmanager.yml
new file mode 100644
index 0000000000..45afc1544a
--- /dev/null
+++ b/changelogs/fragments/60106-templar-contextmanager.yml
@@ -0,0 +1,4 @@
+bugfixes:
+- template lookup - ensure changes to the templar in the lookup, do not
+  affect the templar context outside of the lookup
+  (https://github.com/ansible/ansible/issues/60106)
diff --git a/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml b/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml
new file mode 100644
index 0000000000..211d2fd665
--- /dev/null
+++ b/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml
@@ -0,0 +1,3 @@
+minor_changes:
+- Templating - Add support to auto unroll generators produced by jinja2 filters, to prevent the need of explicit use of ``|list``
+  (https://github.com/ansible/ansible/pull/68014)
diff --git a/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml b/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml
new file mode 100644
index 0000000000..40b426e50b
--- /dev/null
+++ b/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml
@@ -0,0 +1,2 @@
+bugfixes:
+  - Skip literal_eval for string filters results in native jinja. (https://github.com/ansible/ansible/issues/70831)
diff --git a/changelogs/fragments/cve-2024-0690.yml b/changelogs/fragments/cve-2024-0690.yml
new file mode 100644
index 0000000000..0e030d8886
--- /dev/null
+++ b/changelogs/fragments/cve-2024-0690.yml
@@ -0,0 +1,2 @@
+security_fixes:
+- ANSIBLE_NO_LOG - Address issue where ANSIBLE_NO_LOG was ignored (CVE-2024-0690)
diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml
index 3d3916a7fc..96d38e7f51 100644
--- a/lib/ansible/config/base.yml
+++ b/lib/ansible/config/base.yml
@@ -1757,7 +1757,7 @@ SHOW_CUSTOM_STATS:
   type: bool
 STRING_TYPE_FILTERS:
   name: Filters to preserve strings
-  default: [string, to_json, to_nice_json, to_yaml, ppretty, json]
+  default: [string, to_json, to_nice_json, to_yaml, to_nice_yaml, ppretty, json]
   description:
     - "This list of filters avoids 'type conversion' when templating variables"
     - Useful when you want to avoid conversion into lists or dictionaries for JSON strings, for example.
diff --git a/lib/ansible/playbook/base.py b/lib/ansible/playbook/base.py
index 0f4dc4e430..172963a218 100644
--- a/lib/ansible/playbook/base.py
+++ b/lib/ansible/playbook/base.py
@@ -613,7 +613,7 @@ class Base(FieldAttributeBase):
 
     # flags and misc. settings
     _environment = FieldAttribute(isa='list', extend=True, prepend=True)
-    _no_log = FieldAttribute(isa='bool')
+    _no_log = FieldAttribute(isa='bool', default=C.DEFAULT_NO_LOG)
     _run_once = FieldAttribute(isa='bool')
     _ignore_errors = FieldAttribute(isa='bool')
     _ignore_unreachable = FieldAttribute(isa='bool')
diff --git a/lib/ansible/playbook/conditional.py b/lib/ansible/playbook/conditional.py
index ac4fc0c568..be4b75986c 100644
--- a/lib/ansible/playbook/conditional.py
+++ b/lib/ansible/playbook/conditional.py
@@ -173,8 +173,8 @@ class Conditional:
                         )
             try:
                 e = templar.environment.overlay()
-                e.filters.update(templar._get_filters())
-                e.tests.update(templar._get_tests())
+                e.filters.update(templar.environment.filters)
+                e.tests.update(templar.environment.tests)
 
                 res = e._parse(conditional, None, None)
                 res = generate(res, e, None, None)
diff --git a/lib/ansible/playbook/play_context.py b/lib/ansible/playbook/play_context.py
index 10dd57aa3f..5b8b28526c 100644
--- a/lib/ansible/playbook/play_context.py
+++ b/lib/ansible/playbook/play_context.py
@@ -318,10 +318,6 @@ class PlayContext(Base):
             if not new_info.connection_user:
                 new_info.connection_user = new_info.remote_user
 
-        # set no_log to default if it was not previously set
-        if new_info.no_log is None:
-            new_info.no_log = C.DEFAULT_NO_LOG
-
         if task.check_mode is not None:
             new_info.check_mode = task.check_mode
 
diff --git a/lib/ansible/plugins/action/ce_template.py b/lib/ansible/plugins/action/ce_template.py
index 8d62b25647..4a72fbbfa8 100644
--- a/lib/ansible/plugins/action/ce_template.py
+++ b/lib/ansible/plugins/action/ce_template.py
@@ -100,5 +100,5 @@ class ActionModule(_ActionModule):
                     for role in dep_chain:
                         searchpath.append(role._role_path)
         searchpath.append(os.path.dirname(source))
-        self._templar.environment.loader.searchpath = searchpath
-        self._task.args['src'] = self._templar.template(template_data)
+        with self._templar.set_temporary_context(searchpath=searchpath):
+            self._task.args['src'] = self._templar.template(template_data)
diff --git a/lib/ansible/plugins/action/network.py b/lib/ansible/plugins/action/network.py
index f0d0ca3ba7..d91c9b2af9 100644
--- a/lib/ansible/plugins/action/network.py
+++ b/lib/ansible/plugins/action/network.py
@@ -160,8 +160,8 @@ class ActionModule(_ActionModule):
                     for role in dep_chain:
                         searchpath.append(role._role_path)
         searchpath.append(os.path.dirname(source))
-        self._templar.environment.loader.searchpath = searchpath
-        self._task.args['src'] = self._templar.template(template_data, convert_data=convert_data)
+        with self._templar.set_temporary_context(searchpath=searchpath):
+            self._task.args['src'] = self._templar.template(template_data, convert_data=convert_data)
 
     def _get_network_os(self, task_vars):
         if 'network_os' in self._task.args and self._task.args['network_os']:
diff --git a/lib/ansible/plugins/action/template.py b/lib/ansible/plugins/action/template.py
index 8fb7393ff9..cede680ca6 100644
--- a/lib/ansible/plugins/action/template.py
+++ b/lib/ansible/plugins/action/template.py
@@ -17,7 +17,7 @@ from ansible.module_utils._text import to_bytes, to_text, to_native
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.module_utils.six import string_types
 from ansible.plugins.action import ActionBase
-from ansible.template import generate_ansible_template_vars
+from ansible.template import generate_ansible_template_vars, AnsibleEnvironment
 
 
 class ActionModule(ActionBase):
@@ -127,27 +127,23 @@ class ActionModule(ActionBase):
                     newsearchpath.append(p)
                 searchpath = newsearchpath
 
-                self._templar.environment.loader.searchpath = searchpath
-                self._templar.environment.newline_sequence = newline_sequence
-                if block_start_string is not None:
-                    self._templar.environment.block_start_string = block_start_string
-                if block_end_string is not None:
-                    self._templar.environment.block_end_string = block_end_string
-                if variable_start_string is not None:
-                    self._templar.environment.variable_start_string = variable_start_string
-                if variable_end_string is not None:
-                    self._templar.environment.variable_end_string = variable_end_string
-                self._templar.environment.trim_blocks = trim_blocks
-                self._templar.environment.lstrip_blocks = lstrip_blocks
-
                 # add ansible 'template' vars
                 temp_vars = task_vars.copy()
                 temp_vars.update(generate_ansible_template_vars(source, dest))
 
-                old_vars = self._templar.available_variables
-                self._templar.available_variables = temp_vars
-                resultant = self._templar.do_template(template_data, preserve_trailing_newlines=True, escape_backslashes=False)
-                self._templar.available_variables = old_vars
+                # force templar to use AnsibleEnvironment to prevent issues with native types
+                # https://github.com/ansible/ansible/issues/46169
+                templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment,
+                                                          searchpath=searchpath,
+                                                          newline_sequence=newline_sequence,
+                                                          block_start_string=block_start_string,
+                                                          block_end_string=block_end_string,
+                                                          variable_start_string=variable_start_string,
+                                                          variable_end_string=variable_end_string,
+                                                          trim_blocks=trim_blocks,
+                                                          lstrip_blocks=lstrip_blocks,
+                                                          available_variables=temp_vars)
+                resultant = templar.do_template(template_data, preserve_trailing_newlines=True, escape_backslashes=False)
             except AnsibleAction:
                 raise
             except Exception as e:
diff --git a/lib/ansible/plugins/lookup/template.py b/lib/ansible/plugins/lookup/template.py
index 4fd3584b65..c04b5e0d6a 100644
--- a/lib/ansible/plugins/lookup/template.py
+++ b/lib/ansible/plugins/lookup/template.py
@@ -17,7 +17,9 @@ DOCUMENTATION = """
         description: list of files to template
       convert_data:
         type: bool
-        description: whether to convert YAML into data. If False, strings that are YAML will be left untouched.
+        description:
+            - Whether to convert YAML into data. If False, strings that are YAML will be left untouched.
+            - Mutually exclusive with the jinja2_native option.
       variable_start_string:
         description: The string marking the beginning of a print statement.
         default: '{{'
@@ -28,6 +30,16 @@ DOCUMENTATION = """
         default: '}}'
         version_added: '2.8'
         type: str
+      jinja2_native:
+        description:
+            - Controls whether to use Jinja2 native types.
+            - It is off by default even if global jinja2_native is True.
+            - Has no effect if global jinja2_native is False.
+            - This offers more flexibility than the template module which does not use Jinja2 native types at all.
+            - Mutually exclusive with the convert_data option.
+        default: False
+        version_added: '2.11'
+        type: bool
 """
 
 EXAMPLES = """
@@ -51,24 +63,31 @@ import os
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 from ansible.module_utils._text import to_bytes, to_text
-from ansible.template import generate_ansible_template_vars
+from ansible.template import generate_ansible_template_vars, AnsibleEnvironment, USE_JINJA2_NATIVE
 from ansible.utils.display import Display
 
+if USE_JINJA2_NATIVE:
+    from ansible.utils.native_jinja import NativeJinjaText
+
+
 display = Display()
 
 
 class LookupModule(LookupBase):
 
     def run(self, terms, variables, **kwargs):
-
         convert_data_p = kwargs.get('convert_data', True)
         lookup_template_vars = kwargs.get('template_vars', {})
+        jinja2_native = kwargs.get('jinja2_native', False)
         ret = []
 
         variable_start_string = kwargs.get('variable_start_string', None)
         variable_end_string = kwargs.get('variable_end_string', None)
 
-        old_vars = self._templar.available_variables
+        if USE_JINJA2_NATIVE and not jinja2_native:
+            templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment)
+        else:
+            templar = self._templar
 
         for term in terms:
             display.debug("File lookup term: %s" % term)
@@ -92,12 +111,6 @@ class LookupModule(LookupBase):
                     searchpath = newsearchpath
                 searchpath.insert(0, os.path.dirname(lookupfile))
 
-                self._templar.environment.loader.searchpath = searchpath
-                if variable_start_string is not None:
-                    self._templar.environment.variable_start_string = variable_start_string
-                if variable_end_string is not None:
-                    self._templar.environment.variable_end_string = variable_end_string
-
                 # The template will have access to all existing variables,
                 # plus some added by ansible (e.g., template_{path,mtime}),
                 # plus anything passed to the lookup with the template_vars=
@@ -105,17 +118,20 @@ class LookupModule(LookupBase):
                 vars = deepcopy(variables)
                 vars.update(generate_ansible_template_vars(lookupfile))
                 vars.update(lookup_template_vars)
-                self._templar.available_variables = vars
 
-                # do the templating
-                res = self._templar.template(template_data, preserve_trailing_newlines=True,
-                                             convert_data=convert_data_p, escape_backslashes=False)
+                with templar.set_temporary_context(variable_start_string=variable_start_string,
+                                                   variable_end_string=variable_end_string,
+                                                   available_variables=vars, searchpath=searchpath):
+                    res = templar.template(template_data, preserve_trailing_newlines=True,
+                                           convert_data=convert_data_p, escape_backslashes=False)
+
+                if USE_JINJA2_NATIVE and not jinja2_native:
+                    # jinja2_native is true globally but off for the lookup, we need this text
+                    # not to be processed by literal_eval anywhere in Ansible
+                    res = NativeJinjaText(res)
 
                 ret.append(res)
             else:
                 raise AnsibleError("the template file %s could not be found for the lookup" % term)
 
-        # restore old variables
-        self._templar.available_variables = old_vars
-
         return ret
diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py
index 94ab31e58d..35c9dac194 100644
--- a/lib/ansible/template/__init__.py
+++ b/lib/ansible/template/__init__.py
@@ -28,6 +28,7 @@ import re
 import time
 
 from distutils.version import LooseVersion
+from contextlib import contextmanager
 from numbers import Number
 
 try:
@@ -42,8 +43,9 @@ from jinja2.runtime import Context, StrictUndefined
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable, AnsibleAssertionError
 from ansible.module_utils.six import iteritems, string_types, text_type
+from ansible.module_utils.six.moves import range
 from ansible.module_utils._text import to_native, to_text, to_bytes
-from ansible.module_utils.common._collections_compat import Sequence, Mapping, MutableMapping
+from ansible.module_utils.common._collections_compat import Iterator, Sequence, Mapping, MappingView, MutableMapping
 from ansible.module_utils.common.collections import is_sequence
 from ansible.module_utils.compat.importlib import import_module
 from ansible.plugins.loader import filter_loader, lookup_loader, test_loader
@@ -71,12 +73,16 @@ NON_TEMPLATED_TYPES = (bool, Number)
 JINJA2_OVERRIDE = '#jinja2:'
 
 from jinja2 import __version__ as j2_version
+from jinja2 import Environment
+from jinja2.utils import concat as j2_concat
+
 
 USE_JINJA2_NATIVE = False
 if C.DEFAULT_JINJA2_NATIVE:
     try:
-        from jinja2.nativetypes import NativeEnvironment as Environment
-        from ansible.template.native_helpers import ansible_native_concat as j2_concat
+        from jinja2.nativetypes import NativeEnvironment
+        from ansible.template.native_helpers import ansible_native_concat
+        from ansible.utils.native_jinja import NativeJinjaText
         USE_JINJA2_NATIVE = True
     except ImportError:
         from jinja2 import Environment
@@ -85,15 +91,15 @@ if C.DEFAULT_JINJA2_NATIVE:
             'jinja2_native requires Jinja 2.10 and above. '
             'Version detected: %s. Falling back to default.' % j2_version
         )
-else:
-    from jinja2 import Environment
-    from jinja2.utils import concat as j2_concat
 
 
 JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin', 'raw_begin'))
 JINJA2_END_TOKENS = frozenset(('variable_end', 'block_end', 'comment_end', 'raw_end'))
 
 
+RANGE_TYPE = type(range(0))
+
+
 def generate_ansible_template_vars(path, dest_path=None):
     b_path = to_bytes(path)
     try:
@@ -230,6 +236,60 @@ def recursive_check_defined(item):
             raise AnsibleFilterError("{0} is undefined".format(item))
 
 
+def _is_rolled(value):
+    """Helper method to determine if something is an unrolled generator,
+    iterator, or similar object
+    """
+    return (
+        isinstance(value, Iterator) or
+        isinstance(value, MappingView) or
+        isinstance(value, RANGE_TYPE)
+    )
+
+
+def _unroll_iterator(func):
+    """Wrapper function, that intercepts the result of a filter
+    and auto unrolls a generator, so that users are not required to
+    explicitly use ``|list`` to unroll.
+    """
+    def wrapper(*args, **kwargs):
+        ret = func(*args, **kwargs)
+        if _is_rolled(ret):
+            return list(ret)
+        return ret
+
+    return _update_wrapper(wrapper, func)
+
+
+def _update_wrapper(wrapper, func):
+    # This code is duplicated from ``functools.update_wrapper`` from Py3.7.
+    # ``functools.update_wrapper`` was failing when the func was ``functools.partial``
+    for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'):
+        try:
+            value = getattr(func, attr)
+        except AttributeError:
+            pass
+        else:
+            setattr(wrapper, attr, value)
+    for attr in ('__dict__',):
+        getattr(wrapper, attr).update(getattr(func, attr, {}))
+    wrapper.__wrapped__ = func
+    return wrapper
+
+
+def _wrap_native_text(func):
+    """Wrapper function, that intercepts the result of a filter
+    and wraps it into NativeJinjaText which is then used
+    in ``ansible_native_concat`` to indicate that it is a text
+    which should not be passed into ``literal_eval``.
+    """
+    def wrapper(*args, **kwargs):
+        ret = func(*args, **kwargs)
+        return NativeJinjaText(ret)
+
+    return _update_wrapper(wrapper, func)
+
+
 class AnsibleUndefined(StrictUndefined):
     '''
     A custom Undefined class, which returns further Undefined objects on access,
@@ -350,10 +410,11 @@ class AnsibleContext(Context):
 
 
 class JinjaPluginIntercept(MutableMapping):
-    def __init__(self, delegatee, pluginloader, *args, **kwargs):
+    def __init__(self, delegatee, pluginloader, jinja2_native, *args, **kwargs):
         super(JinjaPluginIntercept, self).__init__(*args, **kwargs)
         self._delegatee = delegatee
         self._pluginloader = pluginloader
+        self._jinja2_native = jinja2_native
 
         if self._pluginloader.class_name == 'FilterModule':
             self._method_map_name = 'filters'
@@ -406,10 +467,13 @@ class JinjaPluginIntercept(MutableMapping):
 
             method_map = getattr(plugin_impl, self._method_map_name)
 
-            for f in iteritems(method_map()):
-                fq_name = '.'.join((parent_prefix, f[0]))
+            for func_name, func in iteritems(method_map()):
+                fq_name = '.'.join((parent_prefix, func_name))
                 # FIXME: detect/warn on intra-collection function name collisions
-                self._collection_jinja_func_cache[fq_name] = f[1]
+                if self._jinja2_native and func_name in C.STRING_TYPE_FILTERS:
+                    self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func)
+                else:
+                    self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func)
 
         function_impl = self._collection_jinja_func_cache[key]
         return function_impl
@@ -433,6 +497,9 @@ class AnsibleEnvironment(Environment):
     '''
     Our custom environment, which simply allows us to override the class-level
     values for the Template and Context classes used by jinja2 internally.
+
+    NOTE: Any changes to this class must be reflected in
+          :class:`AnsibleNativeEnvironment` as well.
     '''
     context_class = AnsibleContext
     template_class = AnsibleJ2Template
@@ -440,8 +507,27 @@ class AnsibleEnvironment(Environment):
     def __init__(self, *args, **kwargs):
         super(AnsibleEnvironment, self).__init__(*args, **kwargs)
 
-        self.filters = JinjaPluginIntercept(self.filters, filter_loader)
-        self.tests = JinjaPluginIntercept(self.tests, test_loader)
+        self.filters = JinjaPluginIntercept(self.filters, filter_loader, jinja2_native=False)
+        self.tests = JinjaPluginIntercept(self.tests, test_loader, jinja2_native=False)
+
+
+if USE_JINJA2_NATIVE:
+    class AnsibleNativeEnvironment(NativeEnvironment):
+        '''
+        Our custom environment, which simply allows us to override the class-level
+        values for the Template and Context classes used by jinja2 internally.
+
+        NOTE: Any changes to this class must be reflected in
+              :class:`AnsibleEnvironment` as well.
+        '''
+        context_class = AnsibleContext
+        template_class = AnsibleJ2Template
+
+        def __init__(self, *args, **kwargs):
+            super(AnsibleNativeEnvironment, self).__init__(*args, **kwargs)
+
+            self.filters = JinjaPluginIntercept(self.filters, filter_loader, jinja2_native=True)
+            self.tests = JinjaPluginIntercept(self.tests, test_loader, jinja2_native=True)
 
 
 class Templar:
@@ -478,7 +564,9 @@ class Templar:
         self._fail_on_filter_errors = True
         self._fail_on_undefined_errors = C.DEFAULT_UNDEFINED_VAR_BEHAVIOR
 
-        self.environment = AnsibleEnvironment(
+        environment_class = AnsibleNativeEnvironment if USE_JINJA2_NATIVE else AnsibleEnvironment
+
+        self.environment = environment_class(
             trim_blocks=True,
             undefined=AnsibleUndefined,
             extensions=self._get_extensions(),
@@ -489,17 +577,50 @@ class Templar:
         # the current rendering context under which the templar class is working
         self.cur_context = None
 
+        # FIXME these regular expressions should be re-compiled each time variable_start_string and variable_end_string are changed
         self.SINGLE_VAR = re.compile(r"^%s\s*(\w*)\s*%s$" % (self.environment.variable_start_string, self.environment.variable_end_string))
-
-        self._clean_regex = re.compile(r'(?:%s|%s|%s|%s)' % (
-            self.environment.variable_start_string,
-            self.environment.block_start_string,
-            self.environment.block_end_string,
-            self.environment.variable_end_string
-        ))
         self._no_type_regex = re.compile(r'.*?\|\s*(?:%s)(?:\([^\|]*\))?\s*\)?\s*(?:%s)' %
                                          ('|'.join(C.STRING_TYPE_FILTERS), self.environment.variable_end_string))
 
+    @property
+    def jinja2_native(self):
+        return not isinstance(self.environment, AnsibleEnvironment)
+
+    def copy_with_new_env(self, environment_class=AnsibleEnvironment, **kwargs):
+        r"""Creates a new copy of Templar with a new environment. The new environment is based on
+        given environment class and kwargs.
+
+        :kwarg environment_class: Environment class used for creating a new environment.
+        :kwarg \*\*kwargs: Optional arguments for the new environment that override existing
+            environment attributes.
+
+        :returns: Copy of Templar with updated environment.
+        """
+        # We need to use __new__ to skip __init__, mainly not to create a new
+        # environment there only to override it below
+        new_env = object.__new__(environment_class)
+        new_env.__dict__.update(self.environment.__dict__)
+
+        new_templar = object.__new__(Templar)
+        new_templar.__dict__.update(self.__dict__)
+        new_templar.environment = new_env
+
+        mapping = {
+            'available_variables': new_templar,
+            'searchpath': new_env.loader,
+        }
+
+        for key, value in kwargs.items():
+            obj = mapping.get(key, new_env)
+            try:
+                if value is not None:
+                    setattr(obj, key, value)
+            except AttributeError:
+                # Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7
+                pass
+
+        return new_templar
+
     def _get_filters(self):
         '''
         Returns filter plugins, after loading and caching them if need be
@@ -513,6 +634,17 @@ class Templar:
         for fp in self._filter_loader.all():
             self._filters.update(fp.filters())
 
+        if self.jinja2_native:
+            for string_filter in C.STRING_TYPE_FILTERS:
+                try:
+                    orig_filter = self._filters[string_filter]
+                except KeyError:
+                    try:
+                        orig_filter = self.environment.filters[string_filter]
+                    except KeyError:
+                        continue
+                self._filters[string_filter] = _wrap_native_text(orig_filter)
+
         return self._filters.copy()
 
     def _get_tests(self):
@@ -570,6 +702,36 @@ class Templar:
         )
         self.available_variables = variables
 
+    @contextmanager
+    def set_temporary_context(self, **kwargs):
+        """Context manager used to set temporary templating context, without having to worry about resetting
+        original values afterward
+
+        Use a keyword that maps to the attr you are setting. Applies to ``self.environment`` by default, to
+        set context on another object, it must be in ``mapping``.
+        """
+        mapping = {
+            'available_variables': self,
+            'searchpath': self.environment.loader,
+        }
+        original = {}
+
+        for key, value in kwargs.items():
+            obj = mapping.get(key, self.environment)
+            try:
+                original[key] = getattr(obj, key)
+                if value is not None:
+                    setattr(obj, key, value)
+            except AttributeError:
+                # Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7
+                pass
+
+        yield
+
+        for key in original:
+            obj = mapping.get(key, self.environment)
+            setattr(obj, key, original[key])
+
     def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None,
                  convert_data=True, static_vars=None, cache=True, disable_lookups=False):
         '''
@@ -632,7 +794,7 @@ class Templar:
                             disable_lookups=disable_lookups,
                         )
 
-                        if not USE_JINJA2_NATIVE:
+                        if not self.jinja2_native:
                             unsafe = hasattr(result, '__UNSAFE__')
                             if convert_data and not self._no_type_regex.match(variable):
                                 # if this looks like a dictionary or list, convert it to such using the safe_eval method
@@ -746,8 +908,18 @@ class Templar:
 
         If using ANSIBLE_JINJA2_NATIVE we bypass this and return the actual value always
         '''
-        if USE_JINJA2_NATIVE:
+        if _is_rolled(thing):
+            # Auto unroll a generator, so that users are not required to
+            # explicitly use ``|list`` to unroll
+            # This only affects the scenario where the final result of templating
+            # is a generator, and not where a filter creates a generator in the middle
+            # of a template. See ``_unroll_iterator`` for the other case. This is probably
+            # unncessary
+            return list(thing)
+
+        if self.jinja2_native:
             return thing
+
         return thing if thing is not None else ''
 
     def _fail_lookup(self, name, *args, **kwargs):
@@ -802,7 +974,10 @@ class Templar:
                     ran = wrap_var(ran)
                 else:
                     try:
-                        ran = wrap_var(",".join(ran))
+                        if self.jinja2_native and isinstance(ran[0], NativeJinjaText):
+                            ran = wrap_var(NativeJinjaText(",".join(ran)))
+                        else:
+                            ran = wrap_var(",".join(ran))
                     except TypeError:
                         # Lookup Plugins should always return lists.  Throw an error if that's not
                         # the case:
@@ -824,7 +999,7 @@ class Templar:
             raise AnsibleError("lookup plugin (%s) not found" % name)
 
     def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, disable_lookups=False):
-        if USE_JINJA2_NATIVE and not isinstance(data, string_types):
+        if self.jinja2_native and not isinstance(data, string_types):
             return data
 
         # For preserving the number of input newlines in the output (used
@@ -853,6 +1028,8 @@ class Templar:
 
             # Adds Ansible custom filters and tests
             myenv.filters.update(self._get_filters())
+            for k in myenv.filters:
+                myenv.filters[k] = _unroll_iterator(myenv.filters[k])
             myenv.tests.update(self._get_tests())
 
             if escape_backslashes:
@@ -904,7 +1081,7 @@ class Templar:
                     display.debug("failing because of a type error, template data is: %s" % to_text(data))
                     raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data), to_native(te)))
 
-            if USE_JINJA2_NATIVE and not isinstance(res, string_types):
+            if self.jinja2_native and not isinstance(res, string_types):
                 return res
 
             if preserve_trailing_newlines:
diff --git a/lib/ansible/template/native_helpers.py b/lib/ansible/template/native_helpers.py
index 11c14b7fa1..84296ad9b6 100644
--- a/lib/ansible/template/native_helpers.py
+++ b/lib/ansible/template/native_helpers.py
@@ -14,6 +14,34 @@ from jinja2._compat import text_type
 
 from jinja2.runtime import StrictUndefined
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+from ansible.utils.native_jinja import NativeJinjaText
+
+
+def _fail_on_undefined(data):
+    """Recursively find an undefined value in a nested data structure
+    and properly raise the undefined exception.
+    """
+    if isinstance(data, Mapping):
+        for value in data.values():
+            _fail_on_undefined(value)
+    elif is_sequence(data):
+        for item in data:
+            _fail_on_undefined(item)
+    else:
+        if isinstance(data, StrictUndefined):
+            # To actually raise the undefined exception we need to
+            # access the undefined object otherwise the exception would
+            # be raised on the next access which might not be properly
+            # handled.
+            # See https://github.com/ansible/ansible/issues/52158
+            # and StrictUndefined implementation in upstream Jinja2.
+            str(data)
+
+    return data
+
+
+class NativeJinjaText(text_type):
+    pass
 
 
 def ansible_native_concat(nodes):
@@ -49,9 +77,20 @@ def ansible_native_concat(nodes):
             # We do that only here because it is taken care of by text_type() in the else block below already.
             str(out)
 
+        if isinstance(out, NativeJinjaText):
+            # Sometimes (e.g. ``| string``) we need to mark variables
+            # in a special way so that they remain strings and are not
+            # passed into literal_eval.
+            # See:
+            # https://github.com/ansible/ansible/issues/70831
+            # https://github.com/pallets/jinja/issues/1200
+            # https://github.com/ansible/ansible/issues/70831#issuecomment-664190894
+            return out
+
         # short circuit literal_eval when possible
         if not isinstance(out, list):
             return out
+
     else:
         if isinstance(nodes, types.GeneratorType):
             nodes = chain(head, nodes)
diff --git a/lib/ansible/utils/native_jinja.py b/lib/ansible/utils/native_jinja.py
new file mode 100644
index 0000000000..53ef14004a
--- /dev/null
+++ b/lib/ansible/utils/native_jinja.py
@@ -0,0 +1,13 @@
+# Copyright: (c) 2020, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+from ansible.module_utils.six import text_type
+
+
+class NativeJinjaText(text_type):
+    pass
diff --git a/lib/ansible/utils/unsafe_proxy.py b/lib/ansible/utils/unsafe_proxy.py
index 0bfd6340ff..54bebd177a 100644
--- a/lib/ansible/utils/unsafe_proxy.py
+++ b/lib/ansible/utils/unsafe_proxy.py
@@ -57,6 +57,7 @@ from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.common._collections_compat import Mapping, Set
 from ansible.module_utils.common.collections import is_sequence
 from ansible.module_utils.six import string_types, binary_type, text_type
+from ansible.utils.native_jinja import NativeJinjaText
 
 
 __all__ = ['AnsibleUnsafe', 'wrap_var']
@@ -331,6 +332,10 @@ class NativeJinjaUnsafeText(NativeJinjaText, AnsibleUnsafeText):
     pass
 
 
+class NativeJinjaUnsafeText(NativeJinjaText, AnsibleUnsafeText):
+    pass
+
+
 class UnsafeProxy(object):
     def __new__(cls, obj, *args, **kwargs):
         from ansible.utils.display import Display
@@ -376,6 +381,8 @@ def wrap_var(v):
         v = _wrap_set(v)
     elif is_sequence(v):
         v = _wrap_sequence(v)
+    elif isinstance(v, NativeJinjaText):
+        v = NativeJinjaUnsafeText(v)
     elif isinstance(v, binary_type):
         v = AnsibleUnsafeBytes(v)
     elif isinstance(v, text_type):
diff --git a/test/integration/targets/jinja2_native_types/test_casting.yml b/test/integration/targets/jinja2_native_types/test_casting.yml
index 5b4fe3ac0e..da06ab2e28 100644
--- a/test/integration/targets/jinja2_native_types/test_casting.yml
+++ b/test/integration/targets/jinja2_native_types/test_casting.yml
@@ -1,17 +1,22 @@
 - name: cast things to other things
   set_fact:
       int_to_str: "{{ i_two|to_text }}"
+      int_to_str2: "{{ i_two | string }}"
       str_to_int: "{{ s_two|int }}"
       dict_to_str: "{{ dict_one|to_text }}"
       list_to_str: "{{ list_one|to_text }}"
       int_to_bool: "{{ i_one|bool }}"
       str_true_to_bool: "{{ s_true|bool }}"
       str_false_to_bool: "{{ s_false|bool }}"
+      list_to_json_str: "{{ list_one | to_json }}"
+      list_to_yaml_str: "{{ list_one | to_yaml }}"
 
 - assert:
     that:
         - 'int_to_str == "2"'
         - 'int_to_str|type_debug in ["str", "unicode"]'
+        - 'int_to_str2 == "2"'
+        - 'int_to_str2|type_debug in ["NativeJinjaText"]'
         - 'str_to_int == 2'
         - 'str_to_int|type_debug == "int"'
         - 'dict_to_str|type_debug in ["str", "unicode"]'
@@ -22,3 +27,5 @@
         - 'str_true_to_bool|type_debug == "bool"'
         - 'str_false_to_bool is sameas false'
         - 'str_false_to_bool|type_debug == "bool"'
+        - 'list_to_json_str|type_debug in ["NativeJinjaText"]'
+        - 'list_to_yaml_str|type_debug in ["NativeJinjaText"]'
diff --git a/test/integration/targets/jinja2_native_types/test_dunder.yml b/test/integration/targets/jinja2_native_types/test_dunder.yml
index 46fd4d0a90..df5ea9276b 100644
--- a/test/integration/targets/jinja2_native_types/test_dunder.yml
+++ b/test/integration/targets/jinja2_native_types/test_dunder.yml
@@ -20,4 +20,4 @@
 
 - assert:
     that:
-        - 'const_dunder|type_debug in ["str", "unicode"]'
+        - 'const_dunder|type_debug in ["str", "unicode", "NativeJinjaText"]'
diff --git a/test/integration/targets/no_log/no_log_config.yml b/test/integration/targets/no_log/no_log_config.yml
new file mode 100644
index 0000000000..8a5088059d
--- /dev/null
+++ b/test/integration/targets/no_log/no_log_config.yml
@@ -0,0 +1,13 @@
+- hosts: testhost
+  gather_facts: false
+  tasks:
+    - debug:
+      no_log: true
+
+    - debug:
+      no_log: false
+
+    - debug:
+
+    - debug:
+      loop: '{{ range(3) }}'
diff --git a/test/integration/targets/no_log/runme.sh b/test/integration/targets/no_log/runme.sh
index bb5c048fc9..8bfe019bb9 100755
--- a/test/integration/targets/no_log/runme.sh
+++ b/test/integration/targets/no_log/runme.sh
@@ -19,3 +19,8 @@ set -eux
 
 # test invalid data passed to a suboption
 [ "$(ansible-playbook no_log_suboptions_invalid.yml -i ../../inventory -vvvvv "$@" | grep -Ec '(SUPREME|IDIOM|MOCKUP|EDUCATED|FOOTREST|CRAFTY|FELINE|CRYSTAL|EXPECTANT|AGROUND|GOLIATH|FREEFALL)')" = "0" ]
+
+# test variations on ANSIBLE_NO_LOG
+[ "$(ansible-playbook no_log_config.yml -i ../../inventory -vvvvv "$@" | grep -Ec 'the output has been hidden')" = "1" ]
+[ "$(ANSIBLE_NO_LOG=0 ansible-playbook no_log_config.yml -i ../../inventory -vvvvv "$@" | grep -Ec 'the output has been hidden')" = "1" ]
+[ "$(ANSIBLE_NO_LOG=1 ansible-playbook no_log_config.yml -i ../../inventory -vvvvv "$@" | grep -Ec 'the output has been hidden')" = "6" ]
diff --git a/test/integration/targets/template_jinja2_non_native/46169.yml b/test/integration/targets/template_jinja2_non_native/46169.yml
new file mode 100644
index 0000000000..efb443eae0
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/46169.yml
@@ -0,0 +1,32 @@
+- hosts: localhost
+  gather_facts: no
+  tasks:
+    - set_fact:
+        output_dir: "{{ lookup('env', 'OUTPUT_DIR') }}"
+
+    - template:
+        src: templates/46169.json.j2
+        dest: "{{ output_dir }}/result.json"
+
+    - command: "diff templates/46169.json.j2 {{ output_dir }}/result.json"
+      register: diff_result
+
+    - assert:
+        that:
+          - diff_result.stdout == ""
+
+    - block:
+      - set_fact:
+          non_native_lookup: "{{ lookup('template', 'templates/46169.json.j2') }}"
+
+      - assert:
+          that:
+            - non_native_lookup | type_debug == 'NativeJinjaUnsafeText'
+
+      - set_fact:
+          native_lookup: "{{ lookup('template', 'templates/46169.json.j2', jinja2_native=true) }}"
+
+      - assert:
+          that:
+            - native_lookup | type_debug == 'dict'
+      when: lookup('pipe', ansible_python_interpreter ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.10', '>=')
diff --git a/test/integration/targets/template_jinja2_non_native/aliases b/test/integration/targets/template_jinja2_non_native/aliases
new file mode 100644
index 0000000000..b59832142f
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/aliases
@@ -0,0 +1 @@
+shippable/posix/group3
diff --git a/test/integration/targets/template_jinja2_non_native/runme.sh b/test/integration/targets/template_jinja2_non_native/runme.sh
new file mode 100755
index 0000000000..fe9d495a3e
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/runme.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eux
+
+export ANSIBLE_JINJA2_NATIVE=1
+ansible-playbook 46169.yml -v "$@"
+unset ANSIBLE_JINJA2_NATIVE
diff --git a/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2 b/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2
new file mode 100644
index 0000000000..a4fc3f6717
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2
@@ -0,0 +1,3 @@
+{
+    "key": "bar"
+}
-- 
2.44.0

