Fixed #30399 -- Changed django.utils.html.escape()/urlize() to use html.escape()/unescape().

django/utils/html.py

@@ -1,5 +1,6 @@
 """HTML utilities suitable for global use."""
 
+import html
 import json
 import re
 from html.parser import HTMLParser
@@ -24,14 +25,6 @@ word_split_re = re.compile(r'''([\s<>"']+)''')
 simple_url_re = re.compile(r'^https?://\[?\w', re.IGNORECASE)
 simple_url_2_re = re.compile(r'^www\.|^(?!http)\w[^@]+\.(com|edu|gov|int|mil|net|org)($|/.*)$', re.IGNORECASE)
 
-_html_escapes = {
-    ord('&'): '&amp;',
-    ord('<'): '&lt;',
-    ord('>'): '&gt;',
-    ord('"'): '&quot;',
-    ord("'"): '&#39;',
-}
-
 
 @keep_lazy(str, SafeString)
 def escape(text):
@@ -43,7 +36,7 @@ def escape(text):
     This may result in double-escaping. If this is a concern, use
     conditional_escape() instead.
     """
-    return mark_safe(str(text).translate(_html_escapes))
+    return mark_safe(html.escape(str(text)))
 
 
 _js_escapes = {
@@ -259,15 +252,6 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
             return x
         return '%s…' % x[:max(0, limit - 1)]
 
-    def unescape(text):
-        """
-        If input URL is HTML-escaped, unescape it so that it can be safely fed
-        to smart_urlquote. For example:
-        http://example.com?x=1&amp;y=&lt;2&gt; => http://example.com?x=1&y=<2>
-        """
-        return text.replace('&amp;', '&').replace('&lt;', '<').replace(
-            '&gt;', '>').replace('&quot;', '"').replace('&#39;', "'")
-
     def trim_punctuation(lead, middle, trail):
         """
         Trim trailing and wrapping punctuation from `middle`. Return the items
@@ -292,7 +276,7 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
             # Trim trailing punctuation (after trimming wrapping punctuation,
             # as encoded entities contain ';'). Unescape entites to avoid
             # breaking them by removing ';'.
-            middle_unescaped = unescape(middle)
+            middle_unescaped = html.unescape(middle)
             stripped = middle_unescaped.rstrip(TRAILING_PUNCTUATION_CHARS)
             if middle_unescaped != stripped:
                 trail = middle[len(stripped):] + trail
@@ -329,9 +313,9 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
             url = None
             nofollow_attr = ' rel="nofollow"' if nofollow else ''
             if simple_url_re.match(middle):
-                url = smart_urlquote(unescape(middle))
+                url = smart_urlquote(html.unescape(middle))
             elif simple_url_2_re.match(middle):
-                url = smart_urlquote('http://%s' % unescape(middle))
+                url = smart_urlquote('http://%s' % html.unescape(middle))
             elif ':' not in middle and is_email_simple(middle):
                 local, domain = middle.rsplit('@', 1)
                 try:

docs/intro/tutorial05.txt

@@ -387,7 +387,7 @@ With that ready, we can ask the client to do some work for us::
     >>> response.status_code
     200
     >>> response.content
-    b'\n    <ul>\n    \n        <li><a href="/polls/1/">What&#39;s up?</a></li>\n    \n    </ul>\n\n'
+    b'\n    <ul>\n    \n        <li><a href="/polls/1/">What&#x27;s up?</a></li>\n    \n    </ul>\n\n'
     >>> response.context['latest_question_list']
     <QuerySet [<Question: What's up?>]>
 

docs/ref/templates/builtins.txt

@@ -1603,7 +1603,7 @@ Escapes a string's HTML. Specifically, it makes these replacements:
 
 * ``<`` is converted to ``&lt;``
 * ``>`` is converted to ``&gt;``
-* ``'`` (single quote) is converted to ``&#39;``
+* ``'`` (single quote) is converted to ``&#x27;``
 * ``"`` (double quote) is converted to ``&quot;``
 * ``&`` is converted to ``&amp;``
 

docs/ref/templates/language.txt

@@ -492,7 +492,7 @@ escaped:
 
 * ``<`` is converted to ``&lt;``
 * ``>`` is converted to ``&gt;``
-* ``'`` (single quote) is converted to ``&#39;``
+* ``'`` (single quote) is converted to ``&#x27;``
 * ``"`` (double quote) is converted to ``&quot;``
 * ``&`` is converted to ``&amp;``
 

docs/ref/utils.txt

@@ -584,6 +584,11 @@ escaping HTML.
     for use in HTML. The input is first coerced to a string and the output has
     :func:`~django.utils.safestring.mark_safe` applied.
 
+    .. versionchanged:: 3.0
+
+        In older versions, ``'`` is converted to its decimal code ``'``
+        instead of the equivalent hex code ``'``.
+
 .. function:: conditional_escape(text)
 
     Similar to ``escape()``, except that it doesn't operate on pre-escaped

docs/releases/3.0.txt

@@ -348,6 +348,10 @@ Miscellaneous
   the session and :func:`django.contrib.auth.logout` no longer preserves the
   session's language after logout.
 
+* :func:`django.utils.html.escape` now uses :func:`html.escape` to escape HTML.
+  This converts ``'`` to ``'`` instead of the previous equivalent decimal
+  code ``'``.
+
 .. _deprecated-features-3.0:
 
 Features deprecated in 3.0

tests/admin_docs/test_views.py

@@ -199,7 +199,7 @@ class TestModelDetailView(TestDataMixin, AdminDocsTestCase):
         """
         Methods with keyword arguments should have their arguments displayed.
         """
-        self.assertContains(self.response, "<td>suffix=&#39;ltd&#39;</td>")
+        self.assertContains(self.response, '<td>suffix=&#x27;ltd&#x27;</td>')
 
     def test_methods_with_multiple_arguments_display_arguments(self):
         """

tests/auth_tests/test_forms.py

@@ -236,7 +236,7 @@ class UserCreationFormTest(TestDataMixin, TestCase):
         form = UserCreationForm()
         self.assertEqual(
             form.fields['password1'].help_text,
-            '<ul><li>Your password can&#39;t be too similar to your other personal information.</li></ul>'
+            '<ul><li>Your password can&#x27;t be too similar to your other personal information.</li></ul>'
         )
 
     @override_settings(AUTH_PASSWORD_VALIDATORS=[

tests/forms_tests/tests/test_forms.py

@@ -995,7 +995,7 @@ Java</label></li>
         self.assertHTMLEqual(
             f.as_table(),
             """<tr><th>&lt;em&gt;Special&lt;/em&gt; Field:</th><td>
-<ul class="errorlist"><li>Something&#39;s wrong with &#39;Nothing to escape&#39;</li></ul>
+<ul class="errorlist"><li>Something&#x27;s wrong with &#x27;Nothing to escape&#x27;</li></ul>
 <input type="text" name="special_name" value="Nothing to escape" required></td></tr>
 <tr><th><em>Special</em> Field:</th><td>
 <ul class="errorlist"><li>'<b>Nothing to escape</b>' is a safe string</li></ul>
@@ -1008,10 +1008,10 @@ Java</label></li>
         self.assertHTMLEqual(
             f.as_table(),
             """<tr><th>&lt;em&gt;Special&lt;/em&gt; Field:</th><td>
-<ul class="errorlist"><li>Something&#39;s wrong with &#39;Should escape &lt; &amp; &gt; and
-&lt;script&gt;alert(&#39;xss&#39;)&lt;/script&gt;&#39;</li></ul>
+<ul class="errorlist"><li>Something&#x27;s wrong with &#x27;Should escape &lt; &amp; &gt; and
+&lt;script&gt;alert(&#x27;xss&#x27;)&lt;/script&gt;&#x27;</li></ul>
 <input type="text" name="special_name"
-value="Should escape &lt; &amp; &gt; and &lt;script&gt;alert(&#39;xss&#39;)&lt;/script&gt;" required></td></tr>
+value="Should escape &lt; &amp; &gt; and &lt;script&gt;alert(&#x27;xss&#x27;)&lt;/script&gt;" required></td></tr>
 <tr><th><em>Special</em> Field:</th><td>
 <ul class="errorlist"><li>'<b><i>Do not escape</i></b>' is a safe string</li></ul>
 <input type="text" name="special_safe_name" value="&lt;i&gt;Do not escape&lt;/i&gt;" required></td></tr>"""
@@ -2632,7 +2632,7 @@ Password: <input type="password" name="password" required>
             t.render(Context({'form': UserRegistration(auto_id=False)})),
             """<form>
 <p>Username: <input type="text" name="username" maxlength="10" required><br>
-Good luck picking a username that doesn&#39;t already exist.</p>
+Good luck picking a username that doesn&#x27;t already exist.</p>
 <p>Password1: <input type="password" name="password1" required></p>
 <p>Password2: <input type="password" name="password2" required></p>
 <input type="submit" required>

tests/forms_tests/widget_tests/base.py

@@ -22,7 +22,10 @@ class WidgetTest(SimpleTestCase):
         if self.jinja2_renderer:
             output = widget.render(name, value, attrs=attrs, renderer=self.jinja2_renderer, **kwargs)
             # Django escapes quotes with '"' while Jinja2 uses '"'.
-            assertEqual(output.replace('"', '"'), html)
+            output = output.replace('"', '"')
+            # Django escapes single quotes with ''' while Jinja2 uses '''.
+            output = output.replace(''', ''')
+            assertEqual(output, html)
 
         output = widget.render(name, value, attrs=attrs, renderer=self.django_renderer, **kwargs)
         assertEqual(output, html)

tests/forms_tests/widget_tests/test_clearablefileinput.py

@@ -46,7 +46,7 @@ class ClearableFileInputTest(WidgetTest):
         self.check_html(ClearableFileInput(), 'my<div>file', StrangeFieldFile(), html=(
             """
             Currently: <a href="something?chapter=1&amp;sect=2&amp;copy=3&amp;lang=en">
-            something&lt;div onclick=&quot;alert(&#39;oops&#39;)&quot;&gt;.jpg</a>
+            something&lt;div onclick=&quot;alert(&#x27;oops&#x27;)&quot;&gt;.jpg</a>
             <input type="checkbox" name="my&lt;div&gt;file-clear" id="my&lt;div&gt;file-clear_id">
             <label for="my&lt;div&gt;file-clear_id">Clear</label><br>
             Change: <input type="file" name="my&lt;div&gt;file">

tests/model_forms/tests.py

@@ -1197,7 +1197,7 @@ class ModelFormBasicTests(TestCase):
 <li>Article: <textarea rows="10" cols="40" name="article" required></textarea></li>
 <li>Categories: <select multiple name="categories">
 <option value="%s" selected>Entertainment</option>
-<option value="%s" selected>It&#39;s a test</option>
+<option value="%s" selected>It&#x27;s a test</option>
 <option value="%s">Third test</option>
 </select></li>
 <li>Status: <select name="status">
@@ -1239,7 +1239,7 @@ class ModelFormBasicTests(TestCase):
 <li>Article: <textarea rows="10" cols="40" name="article" required>Hello.</textarea></li>
 <li>Categories: <select multiple name="categories">
 <option value="%s">Entertainment</option>
-<option value="%s">It&#39;s a test</option>
+<option value="%s">It&#x27;s a test</option>
 <option value="%s">Third test</option>
 </select></li>
 <li>Status: <select name="status">
@@ -1290,7 +1290,7 @@ class ModelFormBasicTests(TestCase):
 <li><label for="id_categories">Categories:</label>
 <select multiple name="categories" id="id_categories">
 <option value="%d" selected>Entertainment</option>
-<option value="%d" selected>It&39;s a test</option>
+<option value="%d" selected>It&#x27;s a test</option>
 <option value="%d">Third test</option>
 </select></li>"""
             % (self.c1.pk, self.c2.pk, self.c3.pk))
@@ -1361,7 +1361,7 @@ class ModelFormBasicTests(TestCase):
 <tr><th>Article:</th><td><textarea rows="10" cols="40" name="article" required></textarea></td></tr>
 <tr><th>Categories:</th><td><select multiple name="categories">
 <option value="%s">Entertainment</option>
-<option value="%s">It&#39;s a test</option>
+<option value="%s">It&#x27;s a test</option>
 <option value="%s">Third test</option>
 </select></td></tr>
 <tr><th>Status:</th><td><select name="status">
@@ -1391,7 +1391,7 @@ class ModelFormBasicTests(TestCase):
 <li>Article: <textarea rows="10" cols="40" name="article" required>Hello.</textarea></li>
 <li>Categories: <select multiple name="categories">
 <option value="%s" selected>Entertainment</option>
-<option value="%s">It&#39;s a test</option>
+<option value="%s">It&#x27;s a test</option>
 <option value="%s">Third test</option>
 </select></li>
 <li>Status: <select name="status">
@@ -1535,7 +1535,7 @@ class ModelFormBasicTests(TestCase):
 <li>Article: <textarea rows="10" cols="40" name="article" required></textarea></li>
 <li>Categories: <select multiple name="categories">
 <option value="%s">Entertainment</option>
-<option value="%s">It&#39;s a test</option>
+<option value="%s">It&#x27;s a test</option>
 <option value="%s">Third test</option>
 </select> </li>
 <li>Status: <select name="status">
@@ -1561,7 +1561,7 @@ class ModelFormBasicTests(TestCase):
 <li>Article: <textarea rows="10" cols="40" name="article" required></textarea></li>
 <li>Categories: <select multiple name="categories">
 <option value="%s">Entertainment</option>
-<option value="%s">It&#39;s a test</option>
+<option value="%s">It&#x27;s a test</option>
 <option value="%s">Third test</option>
 <option value="%s">Fourth</option>
 </select></li>

tests/template_tests/filter_tests/test_addslashes.py

@@ -15,7 +15,7 @@ class AddslashesTests(SimpleTestCase):
     @setup({'addslashes02': '{{ a|addslashes }} {{ b|addslashes }}'})
     def test_addslashes02(self):
         output = self.engine.render_to_string('addslashes02', {"a": "<a>'", "b": mark_safe("<a>'")})
-        self.assertEqual(output, r"&lt;a&gt;\&#39; <a>\'")
+        self.assertEqual(output, r"&lt;a&gt;\&#x27; <a>\'")
 
 
 class FunctionTests(SimpleTestCase):

tests/template_tests/filter_tests/test_make_list.py

@@ -19,7 +19,7 @@ class MakeListTests(SimpleTestCase):
     @setup({'make_list02': '{{ a|make_list }}'})
     def test_make_list02(self):
         output = self.engine.render_to_string('make_list02', {"a": mark_safe("&")})
-        self.assertEqual(output, "['&']")
+        self.assertEqual(output, '['&']')
 
     @setup({'make_list03': '{% autoescape off %}{{ a|make_list|stringformat:"s"|safe }}{% endautoescape %}'})
     def test_make_list03(self):

tests/template_tests/filter_tests/test_title.py

@@ -9,7 +9,7 @@ class TitleTests(SimpleTestCase):
     @setup({'title1': '{{ a|title }}'})
     def test_title1(self):
         output = self.engine.render_to_string('title1', {'a': 'JOE\'S CRAB SHACK'})
-        self.assertEqual(output, 'Joe's Crab Shack')
+        self.assertEqual(output, 'Joe's Crab Shack')
 
     @setup({'title2': '{{ a|title }}'})
     def test_title2(self):

tests/template_tests/filter_tests/test_urlize.py

@@ -52,7 +52,7 @@ class UrlizeTests(SimpleTestCase):
     @setup({'urlize06': '{{ a|urlize }}'})
     def test_urlize06(self):
         output = self.engine.render_to_string('urlize06', {'a': "<script>alert('foo')</script>"})
-        self.assertEqual(output, '&lt;script&gt;alert(&#39;foo&#39;)&lt;/script&gt;')
+        self.assertEqual(output, '&lt;script&gt;alert(&#x27;foo&#x27;)&lt;/script&gt;')
 
     # mailto: testing for urlize
     @setup({'urlize07': '{{ a|urlize }}'})
@@ -113,7 +113,7 @@ class FunctionTests(SimpleTestCase):
         )
         self.assertEqual(
             urlize('www.server.com\'abc'),
-            '<a href="http://www.server.com" rel="nofollow">www.server.com</a>&#39;abc',
+            '<a href="http://www.server.com" rel="nofollow">www.server.com</a>&#x27;abc',
         )
         self.assertEqual(
             urlize('www.server.com<abc'),
@@ -284,7 +284,7 @@ class FunctionTests(SimpleTestCase):
             ('<>', ('&lt;', '&gt;')),
             ('[]', ('[', ']')),
             ('""', ('&quot;', '&quot;')),
-            ("''", ('&#39;', '&#39;')),
+            ("''", ('&#x27;', '&#x27;')),
         )
         for wrapping_in, (start_out, end_out) in wrapping_chars:
             with self.subTest(wrapping_in=wrapping_in):

tests/template_tests/syntax_tests/test_url.py

@@ -78,7 +78,7 @@ class UrlTagTests(SimpleTestCase):
     @setup({'url12': '{% url "client_action" id=client.id action="!$&\'()*+,;=~:@," %}'})
     def test_url12(self):
         output = self.engine.render_to_string('url12', {'client': {'id': 1}})
-        self.assertEqual(output, '/client/1/!$&'()*+,;=~:@,/')
+        self.assertEqual(output, '/client/1/!$&'()*+,;=~:@,/')
 
     @setup({'url13': '{% url "client_action" id=client.id action=arg|join:"-" %}'})
     def test_url13(self):

tests/utils_tests/test_html.py

@@ -27,7 +27,7 @@ class TestUtilsHtml(SimpleTestCase):
             ('<', '&lt;'),
             ('>', '&gt;'),
             ('"', '&quot;'),
-            ("'", '&#39;'),
+            ("'", '&#x27;'),
         )
         # Substitution patterns for testing the above items.
         patterns = ("%s", "asdf%sfdsa", "%s1", "1%sb")
@@ -70,6 +70,8 @@ class TestUtilsHtml(SimpleTestCase):
         items = (
             ('<p>See: &#39;&eacute; is an apostrophe followed by e acute</p>',
              'See: &#39;&eacute; is an apostrophe followed by e acute'),
+            ('<p>See: &#x27;&eacute; is an apostrophe followed by e acute</p>',
+             'See: &#x27;&eacute; is an apostrophe followed by e acute'),
             ('<adf>a', 'a'),
             ('</adf>a', 'a'),
             ('<asdf><asdf>e', 'e'),

tests/view_tests/tests/test_csrf.py

@@ -44,22 +44,22 @@ class CsrfViewTests(SimpleTestCase):
         self.assertContains(
             response,
             'You are seeing this message because this HTTPS site requires a '
-            ''Referer header' to be sent by your Web browser, but '
+            ''Referer header' to be sent by your Web browser, but '
             'none was sent.',
             status_code=403,
         )
         self.assertContains(
             response,
-            'If you have configured your browser to disable 'Referer' '
+            'If you have configured your browser to disable 'Referer' '
             'headers, please re-enable them, at least for this site, or for '
-            'HTTPS connections, or for 'same-origin' requests.',
+            'HTTPS connections, or for 'same-origin' requests.',
             status_code=403,
         )
         self.assertContains(
             response,
             'If you are using the <meta name="referrer" '
             'content="no-referrer"> tag or including the '
-            ''Referrer-Policy: no-referrer' header, please remove them.',
+            ''Referrer-Policy: no-referrer' header, please remove them.',
             status_code=403,
         )
 

tests/view_tests/tests/test_debug.py

@@ -304,7 +304,7 @@ class ExceptionReporterTests(SimpleTestCase):
         reporter = ExceptionReporter(request, exc_type, exc_value, tb)
         html = reporter.get_traceback_html()
         self.assertInHTML('<h1>ValueError at /test_view/</h1>', html)
-        self.assertIn('<pre class="exception_value">Can&#39;t find my keys</pre>', html)
+        self.assertIn('<pre class="exception_value">Can&#x27;t find my keys</pre>', html)
         self.assertIn('<th>Request Method:</th>', html)
         self.assertIn('<th>Request URL:</th>', html)
         self.assertIn('<h3 id="user-info">USER</h3>', html)
@@ -325,7 +325,7 @@ class ExceptionReporterTests(SimpleTestCase):
         reporter = ExceptionReporter(None, exc_type, exc_value, tb)
         html = reporter.get_traceback_html()
         self.assertInHTML('<h1>ValueError</h1>', html)
-        self.assertIn('<pre class="exception_value">Can&#39;t find my keys</pre>', html)
+        self.assertIn('<pre class="exception_value">Can&#x27;t find my keys</pre>', html)
         self.assertNotIn('<th>Request Method:</th>', html)
         self.assertNotIn('<th>Request URL:</th>', html)
         self.assertNotIn('<h3 id="user-info">USER</h3>', html)
@@ -463,7 +463,7 @@ class ExceptionReporterTests(SimpleTestCase):
         reporter = ExceptionReporter(request, None, "I'm a little teapot", None)
         html = reporter.get_traceback_html()
         self.assertInHTML('<h1>Report at /test_view/</h1>', html)
-        self.assertIn('<pre class="exception_value">I&#39;m a little teapot</pre>', html)
+        self.assertIn('<pre class="exception_value">I&#x27;m a little teapot</pre>', html)
         self.assertIn('<th>Request Method:</th>', html)
         self.assertIn('<th>Request URL:</th>', html)
         self.assertNotIn('<th>Exception Type:</th>', html)
@@ -476,7 +476,7 @@ class ExceptionReporterTests(SimpleTestCase):
         reporter = ExceptionReporter(None, None, "I'm a little teapot", None)
         html = reporter.get_traceback_html()
         self.assertInHTML('<h1>Report</h1>', html)
-        self.assertIn('<pre class="exception_value">I&#39;m a little teapot</pre>', html)
+        self.assertIn('<pre class="exception_value">I&#x27;m a little teapot</pre>', html)
         self.assertNotIn('<th>Request Method:</th>', html)
         self.assertNotIn('<th>Request URL:</th>', html)
         self.assertNotIn('<th>Exception Type:</th>', html)
@@ -508,7 +508,7 @@ class ExceptionReporterTests(SimpleTestCase):
         except Exception:
             exc_type, exc_value, tb = sys.exc_info()
         html = ExceptionReporter(None, exc_type, exc_value, tb).get_traceback_html()
-        self.assertIn('<td class="code"><pre>&#39;&lt;p&gt;Local variable&lt;/p&gt;&#39;</pre></td>', html)
+        self.assertIn('<td class="code"><pre>&#x27;&lt;p&gt;Local variable&lt;/p&gt;&#x27;</pre></td>', html)
 
     def test_unprintable_values_handling(self):
         "Unprintable values should not make the output generation choke."
@@ -607,7 +607,7 @@ class ExceptionReporterTests(SimpleTestCase):
         An exception report can be generated for requests with 'items' in
         request GET, POST, FILES, or COOKIES QueryDicts.
         """
-        value = '<td>items</td><td class="code"><pre>&#39;Oops&#39;</pre></td>'
+        value = '<td>items</td><td class="code"><pre>&#x27;Oops&#x27;</pre></td>'
         # GET
         request = self.rf.get('/test_view/?items=Oops')
         reporter = ExceptionReporter(request, None, None, None)
@@ -634,7 +634,7 @@ class ExceptionReporterTests(SimpleTestCase):
         request = rf.get('/test_view/')
         reporter = ExceptionReporter(request, None, None, None)
         html = reporter.get_traceback_html()
-        self.assertInHTML('<td>items</td><td class="code"><pre>&#39;Oops&#39;</pre></td>', html)
+        self.assertInHTML('<td>items</td><td class="code"><pre>&#x27;Oops&#x27;</pre></td>', html)
 
     def test_exception_fetching_user(self):
         """