forms - remove unidecode and legacy clean name approach

- resolves #7975

docs/releases/3.0.md

@@ -157,3 +157,9 @@ When overriding the `get_form_class` method of a ModelAdmin `CreateView` or `Edi
 `StreamField` now requires a `use_json_field` keyword argument that can be set to `True`/`False`. If set to `True`, the field will use `JSONField` as its internal type instead of `TextField`, which will change the data type used on the database and allow you to use `JSONField` lookups and transforms on the `StreamField`. If set to `False`, the field will keep its previous behaviour and no database changes will be made. If set to `None` (the default), the field will keep its previous behaviour and a warning (`RemovedInWagtail50Warning`) will appear.
 
 After setting the keyword argument, make sure to generate and run the migrations for the models.
+
+### Removal of legacy `clean_name` on `AbstractFormField`
+
+-   If you have a project migrating from pre 2.10 to this release and you are using the Wagtail form builder and you have existing form submissions you must first upgrade to at least 2.11. Then run migrations and run the application with your data to ensure that any existing form fields are correctly migrated.
+-   In Wagtail 2.10 a `clean_name` field was added to form field models that extend `AbstractFormField` and this initially supported legacy migration of the [Unidecode](https://pypi.org/project/Unidecode/) label conversion.
+-   Any new fields created since then will have used the [AnyAscii](https://pypi.org/project/anyascii/) conversion and Unidecode has been removed from the included packages.

setup.py

@@ -49,7 +49,6 @@ testing_extras = [
     "boto3>=1.16,<1.17",
     "freezegun>=0.3.8",
     "openpyxl>=2.6.4",
-    "Unidecode>=0.04.14,<2.0",
     "azure-mgmt-cdn>=5.1,<6.0",
     "azure-mgmt-frontdoor>=0.3,<0.4",
     "django-pattern-library>=0.7,<0.8",

wagtail/contrib/forms/models.py

@@ -3,13 +3,10 @@ import json
 import os
 
 from django.conf import settings
-from django.core.checks import Info
-from django.core.exceptions import FieldError
 from django.core.serializers.json import DjangoJSONEncoder
-from django.db import DatabaseError, models
+from django.db import models
 from django.template.response import TemplateResponse
 from django.utils.formats import date_format
-from django.utils.text import slugify
 from django.utils.translation import gettext_lazy as _
 
 from wagtail.admin.mail import send_mail
@@ -142,45 +139,6 @@ class AbstractFormField(Orderable):
 
         super().save(*args, **kwargs)
 
-    @classmethod
-    def _migrate_legacy_clean_name(cls):
-        """
-        Ensure that existing data stored will be accessible via the legacy clean_name.
-        When checks run, replace any blank clean_name values with the unidecode conversion.
-        """
-
-        try:
-            objects = cls.objects.filter(clean_name__exact="")
-            if objects.count() == 0:
-                return None
-
-        except (FieldError, DatabaseError):
-            # attempting to query on clean_name before field has been added
-            return None
-
-        try:
-            from unidecode import unidecode
-        except ImportError as error:
-            description = "You have form submission data that was created on an older version of Wagtail and requires the unidecode library to retrieve it correctly. Please install the unidecode package."
-            raise Exception(description) from error
-
-        for obj in objects:
-            legacy_clean_name = str(slugify(str(unidecode(obj.label))))
-            obj.clean_name = legacy_clean_name
-            obj.save()
-
-        return Info("Added `clean_name` on %s form field(s)" % objects.count(), obj=cls)
-
-    @classmethod
-    def check(cls, **kwargs):
-        errors = super().check(**kwargs)
-
-        messages = cls._migrate_legacy_clean_name()
-        if messages:
-            errors.append(messages)
-
-        return errors
-
     class Meta:
         abstract = True
         ordering = ["sort_order"]

wagtail/contrib/forms/tests/test_models.py

@@ -4,7 +4,6 @@ import unittest
 
 from django import VERSION as DJANGO_VERSION
 from django.core import mail
-from django.core.checks import Info
 from django.test import TestCase, override_settings
 
 from wagtail.contrib.forms.models import FormSubmission
@@ -19,7 +18,6 @@ from wagtail.test.testapp.models import (
     CustomFormPageSubmission,
     ExtendedFormField,
     FormField,
-    FormFieldWithCustomSubmission,
     FormPageWithCustomFormBuilder,
     JadeFormPage,
 )
@@ -809,75 +807,3 @@ class TestNonHtmlExtension(TestCase):
         self.assertEqual(
             form_page.landing_page_template, "tests/form_page_landing.jade"
         )
-
-
-class TestLegacyFormFieldCleanNameChecks(TestCase, WagtailTestUtils):
-    fixtures = ["test.json"]
-
-    def setUp(self):
-        self.login(username="siteeditor", password="password")
-        self.form_page = Page.objects.get(
-            url_path="/home/contact-us-one-more-time/"
-        ).specific
-
-    def test_form_field_clean_name_update_on_checks(self):
-        fields_before_checks = [
-            (
-                field.label,
-                field.clean_name,
-            )
-            for field in FormFieldWithCustomSubmission.objects.all()
-        ]
-
-        self.assertEqual(
-            fields_before_checks,
-            [
-                ("Your email", ""),
-                ("Your message", ""),
-                ("Your choices", ""),
-            ],
-        )
-
-        # running checks should show an info message AND update blank clean_name values
-
-        messages = FormFieldWithCustomSubmission.check()
-
-        self.assertEqual(
-            messages,
-            [
-                Info(
-                    "Added `clean_name` on 3 form field(s)",
-                    obj=FormFieldWithCustomSubmission,
-                )
-            ],
-        )
-
-        fields_after_checks = [
-            (
-                field.label,
-                field.clean_name,
-            )
-            for field in FormFieldWithCustomSubmission.objects.all()
-        ]
-
-        self.assertEqual(
-            fields_after_checks,
-            [
-                ("Your email", "your-email"),  # kebab case, legacy format
-                ("Your message", "your-message"),
-                ("Your choices", "your-choices"),
-            ],
-        )
-
-        # running checks again should return no messages as fields no longer need changing
-        self.assertEqual(FormFieldWithCustomSubmission.check(), [])
-
-        # creating a new field should use the non-legacy clean_name format
-
-        field = FormFieldWithCustomSubmission.objects.create(
-            page=self.form_page,
-            label="Your FAVOURITE #number",
-            field_type="number",
-        )
-
-        self.assertEqual(field.clean_name, "your_favourite_number")

wagtail/contrib/forms/tests/test_views.py

@@ -5,7 +5,6 @@ from io import BytesIO
 
 from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
-from django.core.checks import Info
 from django.test import RequestFactory, TestCase, override_settings
 from django.urls import reverse
 from openpyxl import load_workbook
@@ -522,49 +521,6 @@ class TestFormsSubmissionsList(TestCase, WagtailTestUtils):
         self.assertIn("this is a really old message", first_row_values)
 
 
-class TestFormsSubmissionsListLegacyFieldName(TestCase, WagtailTestUtils):
-    fixtures = ["test.json"]
-
-    def setUp(self):
-        self.login(username="siteeditor", password="password")
-        self.form_page = Page.objects.get(
-            url_path="/home/contact-us-one-more-time/"
-        ).specific
-
-        # running checks should show an info message AND update blank clean_name values
-
-        messages = FormFieldWithCustomSubmission.check()
-
-        self.assertEqual(
-            messages,
-            [
-                Info(
-                    "Added `clean_name` on 3 form field(s)",
-                    obj=FormFieldWithCustomSubmission,
-                )
-            ],
-        )
-
-        # check clean_name has been updated
-        self.assertEqual(
-            FormFieldWithCustomSubmission.objects.all()[0].clean_name, "your-email"
-        )
-
-    def test_list_submissions(self):
-        response = self.client.get(
-            reverse("wagtailforms:list_submissions", args=(self.form_page.id,))
-        )
-
-        # Check response
-        self.assertEqual(response.status_code, 200)
-        self.assertTemplateUsed(response, "wagtailforms/index_submissions.html")
-        self.assertEqual(len(response.context["data_rows"]), 2)
-
-        # check display of list values within form submissions
-        self.assertContains(response, "old@example.com")
-        self.assertContains(response, "new@example.com")
-
-
 class TestFormsSubmissionsExport(TestCase, WagtailTestUtils):
     def setUp(self):
         # Create a form page

wagtail/images/models.py

@@ -187,7 +187,7 @@ class AbstractImage(ImageFileMixin, CollectionMember, index.Indexed, models.Mode
         folder_name = "original_images"
         filename = self.file.field.storage.get_valid_name(filename)
 
-        # do a unidecode in the filename and then
+        # convert the filename to simple ascii characters and then
         # replace non-ascii characters in filename with _ , to sidestep issues with filesystem encoding
         filename = "".join(
             (i if ord(i) < 128 else "_") for i in string_to_ascii(filename)

wagtail/test/testapp/fixtures/test.json

@@ -487,7 +487,7 @@
     "pk": 1,
     "model": "tests.formfieldwithcustomsubmission",
     "fields": {
-      "clean_name": "",
+      "clean_name": "your_email",
       "sort_order": 1,
       "label": "Your email",
       "field_type": "email",
@@ -502,7 +502,7 @@
     "pk": 2,
     "model": "tests.formfieldwithcustomsubmission",
     "fields": {
-      "clean_name": "",
+      "clean_name": "your_message",
       "sort_order": 2,
       "label": "Your message",
       "field_type": "multiline",
@@ -517,7 +517,7 @@
     "pk": 3,
     "model": "tests.formfieldwithcustomsubmission",
     "fields": {
-      "clean_name": "",
+      "clean_name": "your_choices",
       "sort_order": 3,
       "label": "Your choices",
       "field_type": "checkboxes",