Refs #27795 -- Removed force_text from the template layer

Thanks Tim Graham for the review.

django/contrib/auth/forms.py

@@ -275,7 +275,7 @@ class PasswordResetForm(forms.Form):
                 'email': email,
                 'domain': domain,
                 'site_name': site_name,
-                'uid': urlsafe_base64_encode(force_bytes(user.pk)),
+                'uid': urlsafe_base64_encode(force_bytes(user.pk)).decode(),
                 'user': user,
                 'token': token_generator.make_token(user),
                 'protocol': 'https' if use_https else 'http',

django/contrib/humanize/templatetags/humanize.py

@@ -5,7 +5,6 @@ from decimal import Decimal
 from django import template
 from django.conf import settings
 from django.template import defaultfilters
-from django.utils.encoding import force_text
 from django.utils.formats import number_format
 from django.utils.safestring import mark_safe
 from django.utils.timezone import is_aware, utc
@@ -45,7 +44,7 @@ def intcomma(value, use_l10n=True):
             return intcomma(value, False)
         else:
             return number_format(value, force_grouping=True)
-    orig = force_text(value)
+    orig = str(value)
     new = re.sub(r"^(-?\d+)(\d{3})", r'\g<1>,\g<2>', orig)
     if orig == new:
         return new

django/core/signing.py

@@ -43,7 +43,7 @@ import zlib
 from django.conf import settings
 from django.utils import baseconv
 from django.utils.crypto import constant_time_compare, salted_hmac
-from django.utils.encoding import force_bytes, force_text
+from django.utils.encoding import force_bytes
 from django.utils.module_loading import import_string
 
 _SEP_UNSAFE = re.compile(r'^[A-z0-9-_=]*$')
@@ -73,7 +73,7 @@ def b64_decode(s):
 
 
 def base64_hmac(salt, value, key):
-    return b64_encode(salted_hmac(salt, value, key).digest())
+    return b64_encode(salted_hmac(salt, value, key).digest()).decode()
 
 
 def get_cookie_signer(salt='django.core.signing.get_cookie_signer'):
@@ -121,9 +121,9 @@ def dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer,
         if len(compressed) < (len(data) - 1):
             data = compressed
             is_compressed = True
-    base64d = b64_encode(data)
+    base64d = b64_encode(data).decode()
     if is_compressed:
-        base64d = b'.' + base64d
+        base64d = '.' + base64d
     return TimestampSigner(key, salt=salt).sign(base64d)
 
 
@@ -161,7 +161,7 @@ class Signer:
         self.salt = salt or '%s.%s' % (self.__class__.__module__, self.__class__.__name__)
 
     def signature(self, value):
-        return force_text(base64_hmac(self.salt + 'signer', value, self.key))
+        return base64_hmac(self.salt + 'signer', value, self.key)
 
     def sign(self, value):
         return '%s%s%s' % (value, self.sep, self.signature(value))
@@ -171,7 +171,7 @@ class Signer:
             raise BadSignature('No "%s" found in value' % self.sep)
         value, sig = signed_value.rsplit(self.sep, 1)
         if constant_time_compare(sig, self.signature(value)):
-            return force_text(value)
+            return value
         raise BadSignature('Signature "%s" does not match' % sig)
 
 
@@ -181,7 +181,7 @@ class TimestampSigner(Signer):
         return baseconv.base62.encode(int(time.time()))
 
     def sign(self, value):
-        value = '%s%s%s' % (force_text(value), self.sep, self.timestamp())
+        value = '%s%s%s' % (value, self.sep, self.timestamp())
         return super().sign(value)
 
     def unsign(self, value, max_age=None):

django/template/base.py

@@ -56,7 +56,6 @@ import re
 from django.template.context import (  # NOQA: imported for backwards compatibility
     BaseContext, Context, ContextPopException, RequestContext,
 )
-from django.utils.encoding import force_text
 from django.utils.formats import localize
 from django.utils.html import conditional_escape, escape
 from django.utils.inspect import getargspec
@@ -108,10 +107,6 @@ tag_re = (re.compile('(%s.*?%s|%s.*?%s|%s.*?%s)' %
 logger = logging.getLogger('django.template')
 
 
-class TemplateEncodingError(Exception):
-    pass
-
-
 class VariableDoesNotExist(Exception):
 
     def __init__(self, msg, params=()):
@@ -150,13 +145,6 @@ class Origin:
 
 class Template:
     def __init__(self, template_string, origin=None, name=None, engine=None):
-        try:
-            template_string = force_text(template_string)
-        except UnicodeDecodeError:
-            raise TemplateEncodingError(
-                "Templates can only be constructed from strings or UTF-8 "
-                "bytestrings."
-            )
         # If Template is instantiated directly rather than from an Engine and
         # exactly one Django template engine is configured, use that engine.
         # This is required to preserve backwards-compatibility for direct use
@@ -274,7 +262,7 @@ class Template:
         # In some rare cases exc_value.args can be empty or an invalid
         # string.
         try:
-            message = force_text(exception.args[0])
+            message = str(exception.args[0])
         except (IndexError, UnicodeDecodeError):
             message = '(Could not get exception message)'
 
@@ -957,7 +945,7 @@ class NodeList(list):
                 bit = node.render_annotated(context)
             else:
                 bit = node
-            bits.append(force_text(bit))
+            bits.append(str(bit))
         return mark_safe(''.join(bits))
 
     def get_nodes_by_type(self, nodetype):
@@ -987,11 +975,12 @@ def render_value_in_context(value, context):
     """
     value = template_localtime(value, use_tz=context.use_tz)
     value = localize(value, use_l10n=context.use_l10n)
-    value = force_text(value)
     if context.autoescape:
+        if not issubclass(type(value), str):
+            value = str(value)
         return conditional_escape(value)
     else:
-        return value
+        return str(value)
 
 
 class VariableNode(Node):

django/template/defaultfilters.py

@@ -9,7 +9,7 @@ from urllib.parse import quote
 
 from django.utils import formats
 from django.utils.dateformat import format, time_format
-from django.utils.encoding import force_text, iri_to_uri
+from django.utils.encoding import iri_to_uri
 from django.utils.html import (
     avoid_wrapping, conditional_escape, escape, escapejs, linebreaks,
     strip_tags, urlize as _urlize,
@@ -39,7 +39,7 @@ def stringfilter(func):
     def _dec(*args, **kwargs):
         if args:
             args = list(args)
-            args[0] = force_text(args[0])
+            args[0] = str(args[0])
             if (isinstance(args[0], SafeData) and
                     getattr(_dec._decorated_function, 'is_safe', False)):
                 return mark_safe(func(*args, **kwargs))
@@ -120,7 +120,7 @@ def floatformat(text, arg=-1):
         d = Decimal(input_val)
     except InvalidOperation:
         try:
-            d = Decimal(force_text(float(text)))
+            d = Decimal(str(float(text)))
         except (ValueError, InvalidOperation, TypeError):
             return ''
     try:
@@ -473,7 +473,7 @@ def safeseq(value):
     individually, as safe, after converting them to strings. Returns a list
     with the results.
     """
-    return [mark_safe(force_text(obj)) for obj in value]
+    return [mark_safe(str(obj)) for obj in value]
 
 
 @register.filter(is_safe=True)
@@ -551,7 +551,6 @@ def join(value, arg, autoescape=True):
     """
     Joins a list with a string, like Python's ``str.join(list)``.
     """
-    value = map(force_text, value)
     if autoescape:
         value = [conditional_escape(v) for v in value]
     try:
@@ -677,7 +676,7 @@ def unordered_list(value, autoescape=True):
                 sublist = '\n%s<ul>\n%s\n%s</ul>\n%s' % (
                     indent, list_formatter(children, tabs + 1), indent, indent)
             output.append('%s<li>%s%s</li>' % (
-                indent, escaper(force_text(item)), sublist))
+                indent, escaper(item), sublist))
         return '\n'.join(output)
 
     return mark_safe(list_formatter(value))
@@ -937,4 +936,4 @@ def pprint(value):
     try:
         return pformat(value)
     except Exception as e:
-        return "Error in formatting: %s: %s" % (e.__class__.__name__, force_text(e, errors="replace"))
+        return "Error in formatting: %s: %s" % (e.__class__.__name__, e)

django/template/defaulttags.py

@@ -8,7 +8,6 @@ from itertools import cycle as itertools_cycle, groupby
 
 from django.conf import settings
 from django.utils import timezone
-from django.utils.encoding import force_text
 from django.utils.html import conditional_escape, format_html
 from django.utils.lorem_ipsum import paragraphs, words
 from django.utils.safestring import mark_safe
@@ -96,9 +95,9 @@ class CycleNode(Node):
 class DebugNode(Node):
     def render(self, context):
         from pprint import pformat
-        output = [force_text(pformat(val)) for val in context]
+        output = [pformat(val) for val in context]
         output.append('\n\n')
-        output.append(force_text(pformat(sys.modules)))
+        output.append(pformat(sys.modules))
         return ''.join(output)
 
 
@@ -220,7 +219,7 @@ class ForNode(Node):
                     # don't want to leave any vars from the previous loop on the
                     # context.
                     context.pop()
-        return mark_safe(''.join(force_text(n) for n in nodelist))
+        return mark_safe(''.join(nodelist))
 
 
 class IfChangedNode(Node):
@@ -437,10 +436,7 @@ class URLNode(Node):
     def render(self, context):
         from django.urls import reverse, NoReverseMatch
         args = [arg.resolve(context) for arg in self.args]
-        kwargs = {
-            force_text(k, 'ascii'): v.resolve(context)
-            for k, v in self.kwargs.items()
-        }
+        kwargs = {k: v.resolve(context) for k, v in self.kwargs.items()}
         view_name = self.view_name.resolve(context)
         try:
             current_app = context.request.current_app

django/template/loaders/cached.py

@@ -7,7 +7,7 @@ import hashlib
 
 from django.template import TemplateDoesNotExist
 from django.template.backends.django import copy_exception
-from django.utils.encoding import force_bytes, force_text
+from django.utils.encoding import force_bytes
 
 from .base import Loader as BaseLoader
 
@@ -86,7 +86,7 @@ class Loader(BaseLoader):
             if matching:
                 skip_prefix = self.generate_hash(matching)
 
-        return '-'.join(filter(bool, [force_text(template_name), skip_prefix, dirs_prefix]))
+        return '-'.join(filter(bool, [str(template_name), skip_prefix, dirs_prefix]))
 
     def generate_hash(self, values):
         return hashlib.sha1(force_bytes('|'.join(values))).hexdigest()

django/templatetags/l10n.py

@@ -1,6 +1,5 @@
 from django.template import Library, Node, TemplateSyntaxError
 from django.utils import formats
-from django.utils.encoding import force_text
 
 register = Library()
 
@@ -11,7 +10,7 @@ def localize(value):
     Forces a value to be rendered as a localized value,
     regardless of the value of ``settings.USE_L10N``.
     """
-    return force_text(formats.localize(value, use_l10n=True))
+    return str(formats.localize(value, use_l10n=True))
 
 
 @register.filter(is_safe=False)
@@ -20,7 +19,7 @@ def unlocalize(value):
     Forces a value to be rendered as a non-localized value,
     regardless of the value of ``settings.USE_L10N``.
     """
-    return force_text(value)
+    return str(value)
 
 
 class LocalizeNode(Node):

django/utils/tree.py

@@ -5,8 +5,6 @@ ORM.
 
 import copy
 
-from django.utils.encoding import force_text
-
 
 class Node:
     """
@@ -45,7 +43,7 @@ class Node:
 
     def __str__(self):
         template = '(NOT (%s: %s))' if self.negated else '(%s: %s)'
-        return template % (self.connector, ', '.join(force_text(c) for c in self.children))
+        return template % (self.connector, ', '.join(str(c) for c in self.children))
 
     def __repr__(self):
         return "<%s: %s>" % (self.__class__.__name__, self)

tests/signing/tests.py

@@ -18,7 +18,7 @@ class TestSigner(SimpleTestCase):
         ):
             self.assertEqual(
                 signer.signature(s),
-                signing.base64_hmac(signer.salt + 'signer', s, 'predictable-secret').decode()
+                signing.base64_hmac(signer.salt + 'signer', s, 'predictable-secret')
             )
             self.assertNotEqual(signer.signature(s), signer2.signature(s))
 
@@ -27,7 +27,7 @@ class TestSigner(SimpleTestCase):
         signer = signing.Signer('predictable-secret', salt='extra-salt')
         self.assertEqual(
             signer.signature('hello'),
-            signing.base64_hmac('extra-salt' + 'signer', 'hello', 'predictable-secret').decode()
+            signing.base64_hmac('extra-salt' + 'signer', 'hello', 'predictable-secret')
         )
         self.assertNotEqual(
             signing.Signer('predictable-secret', salt='one').signature('hello'),

tests/template_tests/syntax_tests/i18n/test_blocktrans.py

@@ -17,20 +17,20 @@ class I18nBlockTransTagTests(SimpleTestCase):
     @setup({'i18n03': '{% load i18n %}{% blocktrans %}{{ anton }}{% endblocktrans %}'})
     def test_i18n03(self):
         """simple translation of a variable"""
-        output = self.engine.render_to_string('i18n03', {'anton': b'\xc3\x85'})
+        output = self.engine.render_to_string('i18n03', {'anton': 'Å'})
         self.assertEqual(output, 'Å')
 
     @setup({'i18n04': '{% load i18n %}{% blocktrans with berta=anton|lower %}{{ berta }}{% endblocktrans %}'})
     def test_i18n04(self):
         """simple translation of a variable and filter"""
-        output = self.engine.render_to_string('i18n04', {'anton': b'\xc3\x85'})
+        output = self.engine.render_to_string('i18n04', {'anton': 'Å'})
         self.assertEqual(output, 'å')
 
     @setup({'legacyi18n04': '{% load i18n %}'
                             '{% blocktrans with anton|lower as berta %}{{ berta }}{% endblocktrans %}'})
     def test_legacyi18n04(self):
         """simple translation of a variable and filter"""
-        output = self.engine.render_to_string('legacyi18n04', {'anton': b'\xc3\x85'})
+        output = self.engine.render_to_string('legacyi18n04', {'anton': 'Å'})
         self.assertEqual(output, 'å')
 
     @setup({'i18n05': '{% load i18n %}{% blocktrans %}xxx{{ anton }}xxx{% endblocktrans %}'})

tests/template_tests/test_unicode.py

@@ -1,31 +0,0 @@
-from unittest import TestCase
-
-from django.template import Context, Engine
-from django.template.base import TemplateEncodingError
-from django.utils.safestring import SafeData
-
-
-class UnicodeTests(TestCase):
-    def test_template(self):
-        # Templates can be created from strings.
-        engine = Engine()
-        t1 = engine.from_string('ŠĐĆŽćžšđ {{ var }}')
-        # Templates can also be created from bytestrings. These are assumed to
-        # be encoded using UTF-8.
-        s = b'\xc5\xa0\xc4\x90\xc4\x86\xc5\xbd\xc4\x87\xc5\xbe\xc5\xa1\xc4\x91 {{ var }}'
-        t2 = engine.from_string(s)
-        with self.assertRaises(TemplateEncodingError):
-            engine.from_string(b'\x80\xc5\xc0')
-
-        # Contexts can be constructed from strings or UTF-8 bytestrings.
-        Context({b"var": b"foo"})
-        Context({"var": b"foo"})
-        c3 = Context({b"var": "Đđ"})
-        Context({"var": b"\xc4\x90\xc4\x91"})
-
-        # Since both templates and all four contexts represent the same thing,
-        # they all render the same (and are returned as strings and
-        # "safe" objects as well, for auto-escaping purposes).
-        self.assertEqual(t1.render(c3), t2.render(c3))
-        self.assertIsInstance(t1.render(c3), str)
-        self.assertIsInstance(t1.render(c3), SafeData)