Merge branch 'master' into search-query-api

# Conflicts:
#	wagtail/search/backends/db.py
#	wagtail/search/backends/elasticsearch2.py

CHANGELOG.txt

@@ -23,6 +23,7 @@ Changelog
  * Added `render_landing_page` method to `AbstractForm` to be easily overridden and pass `form_submission` to landing page context (Stein Strindhaug)
  * Added `heading` kwarg to `InlinePanel` to allow heading to be set independently of button label (Adrian Turjak)
  * The value type returned from a `StructBlock` can now be customised (LB (Ben Johnston))
+ * Added `bgcolor` image operation (Karl Hobley)
  * Fix: Do not remove stopwords when generating slugs from non-ASCII titles, to avoid issues with incorrect word boundaries (Sævar Öfjörð Magnússon)
  * Fix: The PostgreSQL search backend now preserves ordering of the `QuerySet` when searching with `order_by_relevance=False` (Bertrand Bordage)
  * Fix: Using `modeladmin_register` as a decorator no longer replaces the decorated class with `None` (Tim Heap)
@@ -43,6 +44,9 @@ Changelog
  * Fix: Fixed error on Elasticsearch backend when passing a queryset as an `__in` filter (Karl Hobley, Matt Westcott)
  * Fix: `__isnull` filters no longer fail on Elasticsearch 5 (Karl Hobley)
  * Fix: Prevented intermittent failures on Postgres search backend when a field is defined as both a `SearchField` and a `FilterField` (Matt Westcott)
+ * Fix: Alt text of images in rich text is no longer truncated on double-quote characters (Matt Westcott)
+ * Fix: Ampersands in embed URLs within rich text are no longer double-escaped (Matt Westcott)
+ * Fix: Using RGBA images no longer crashes with Pillow >= 4.2.0 (Karl Hobley)
 
 
 1.13.1 (17.11.2017)

CONTRIBUTING.md

@@ -17,7 +17,7 @@ for support - use [the 'wagtail' tag on Stack Overflow](http://stackoverflow.com
 Please review the 
 [contributing guidelines](http://docs.wagtail.io/en/latest/contributing/index.html). 
 You might like to start by checking issues with the 
-[difficulty:Easy](https://github.com/wagtail/wagtail/labels/difficulty%3AEasy) label.
+[good first issue](https://github.com/wagtail/wagtail/labels/good%20first%20issue) label.
 
 ## Code reviews
 

CONTRIBUTORS.rst

@@ -265,6 +265,7 @@ Contributors
 * Adrian Turjak
 * Michael Palmer
 * Philipp Bosch
+* misraX
 
 Translators
 ===========

README.rst

@@ -32,7 +32,7 @@ Features
 * Workflow support
 * An extensible `form builder <http://docs.wagtail.io/en/latest/reference/contrib/forms/index.html>`_
 * Multi-site and multi-language support
-* Excellent `test coverage <https://coveralls.io/r/torchbox/wagtail?branch=master>`_
+* Excellent `test coverage <http://codecov.io/github/wagtail/wagtail?branch=master>`_
 
 Find out more at `wagtail.io <http://wagtail.io/>`_.
 
@@ -86,6 +86,6 @@ Contributing
 ~~~~~~~~~~~~
 If you're a Python or Django developer, fork the repo and get stuck in! We run a separate group for developers of Wagtail itself at https://groups.google.com/forum/#!forum/wagtail-developers (please note that this is not for support requests).
 
-You might like to start by reviewing the `contributing guidelines <http://docs.wagtail.io/en/latest/contributing/index.html>`_ and checking issues with the `difficulty:Easy <https://github.com/wagtail/wagtail/labels/difficulty%3AEasy>`_ label.
+You might like to start by reviewing the `contributing guidelines <http://docs.wagtail.io/en/latest/contributing/index.html>`_ and checking issues with the `good first issue <https://github.com/wagtail/wagtail/labels/good%20first%20issue>`_ label.
 
 We also welcome translations for Wagtail's interface. Translation work should be submitted through `Transifex <https://www.transifex.com/projects/p/wagtail/>`_.

docs/advanced_topics/api/v2/configuration.rst

@@ -114,7 +114,7 @@ For example:
     from wagtail.api import APIField
 
     class BlogPageAuthor(Orderable):
-        page = models.ForeignKey('blog.BlogPage', related_name='authors')
+        page = models.ForeignKey('blog.BlogPage', on_delete=models.CASCADE, related_name='authors')
         name = models.CharField(max_length=255)
 
         api_fields = [
@@ -125,7 +125,7 @@ For example:
     class BlogPage(Page):
         published_date = models.DateTimeField()
         body = RichTextField()
-        feed_image = models.ForeignKey('wagtailimages.Image', ...)
+        feed_image = models.ForeignKey('wagtailimages.Image', on_delete=models.CASCADE, ...)
         private_field = models.CharField(max_length=255)
 
         # Export fields over the API

docs/advanced_topics/i18n/index.rst

@@ -90,7 +90,7 @@ This feature is enabled through the project's root URL configuration. Just put t
     from wagtail.admin import urls as wagtailadmin_urls
     from wagtail.documents import urls as wagtaildocs_urls
     from wagtail.core import urls as wagtail_urls
-
+    from search import views as search_views
 
     urlpatterns = [
         url(r'^django-admin/', include(admin.site.urls)),
@@ -103,7 +103,7 @@ This feature is enabled through the project's root URL configuration. Just put t
     urlpatterns += i18n_patterns(
         # These URLs will have /<language_code>/ appended to the beginning
 
-        url(r'^search/$', 'search.views.search', name='search'),
+        url(r'^search/$', search_views.search, name='search'),
 
         url(r'', include(wagtail_urls)),
     )
@@ -143,7 +143,7 @@ For each field you would like to be translatable, duplicate it for every languag
         body_fr = StreamField(...)
 
         # Language-independent fields don't need to be duplicated
-        thumbnail_image = models.ForeignKey('wagtailimages.image', ...)
+        thumbnail_image = models.ForeignKey('wagtailimages.image', on_delete=models.CASCADE, ...)
 
 .. note::
 

docs/advanced_topics/images/custom_image_model.rst

@@ -39,7 +39,7 @@ Here's an example:
 
 
     class CustomRendition(AbstractRendition):
-        image = models.ForeignKey(CustomImage, related_name='renditions')
+        image = models.ForeignKey(CustomImage, on_delete=models.CASCADE, related_name='renditions')
 
         class Meta:
             unique_together = (

docs/getting_started/tutorial.rst

@@ -436,7 +436,7 @@ Add a new ``BlogPageGalleryImage`` model to ``models.py``:
 
 
     class BlogPageGalleryImage(Orderable):
-        page = ParentalKey(BlogPage, related_name='gallery_images')
+        page = ParentalKey(BlogPage, on_delete=models.CASCADE, related_name='gallery_images')
         image = models.ForeignKey(
             'wagtailimages.Image', on_delete=models.CASCADE, related_name='+'
         )

docs/reference/contrib/forms/customisation.rst

@@ -23,7 +23,7 @@ You can do this as shown below.
 
 
     class FormField(AbstractFormField):
-        page = ParentalKey('FormPage', related_name='custom_form_fields')
+        page = ParentalKey('FormPage', on_delete=models.CASCADE, related_name='custom_form_fields')
 
 
     class FormPage(AbstractEmailForm):
@@ -74,7 +74,7 @@ Example:
 
 
     class FormField(AbstractFormField):
-        page = ParentalKey('FormPage', related_name='form_fields')
+        page = ParentalKey('FormPage', on_delete=models.CASCADE, related_name='form_fields')
 
 
     class FormPage(AbstractEmailForm):
@@ -135,7 +135,7 @@ The following example shows how to add a username to the CSV export:
 
 
     class FormField(AbstractFormField):
-        page = ParentalKey('FormPage', related_name='form_fields')
+        page = ParentalKey('FormPage', on_delete=models.CASCADE, related_name='form_fields')
 
 
     class FormPage(AbstractEmailForm):
@@ -213,7 +213,7 @@ Example:
 
 
     class FormField(AbstractFormField):
-        page = ParentalKey('FormPage', related_name='form_fields')
+        page = ParentalKey('FormPage', on_delete=models.CASCADE, related_name='form_fields')
 
 
     class FormPage(AbstractEmailForm):
@@ -309,7 +309,7 @@ The following example shows how to create a multi-step form.
 
 
     class FormField(AbstractFormField):
-        page = ParentalKey('FormPage', related_name='form_fields')
+        page = ParentalKey('FormPage', on_delete=models.CASCADE, related_name='form_fields')
 
 
     class FormPage(AbstractEmailForm):
@@ -455,7 +455,7 @@ First, you need to collect results as shown below:
 
 
     class FormField(AbstractFormField):
-        page = ParentalKey('FormPage', related_name='form_fields')
+        page = ParentalKey('FormPage', on_delete=models.CASCADE, related_name='form_fields')
 
 
     class FormPage(AbstractEmailForm):

docs/reference/contrib/forms/index.rst

@@ -38,7 +38,7 @@ Within the ``models.py`` of one of your apps, create a model that extends ``wagt
 
 
     class FormField(AbstractFormField):
-        page = ParentalKey('FormPage', related_name='form_fields')
+        page = ParentalKey('FormPage', on_delete=models.CASCADE, related_name='form_fields')
 
 
     class FormPage(AbstractEmailForm):

docs/reference/pages/model_recipes.rst

@@ -150,7 +150,7 @@ Using an example from the Wagtail demo site, here's what the tag model and the r
     from taggit.models import TaggedItemBase
 
     class BlogPageTag(TaggedItemBase):
-        content_object = ParentalKey('demo.BlogPage', related_name='tagged_items')
+        content_object = ParentalKey('demo.BlogPage', on_delete=models.CASCADE, related_name='tagged_items')
 
     class BlogPage(Page):
         ...
@@ -166,6 +166,8 @@ Wagtail's admin provides a nice interface for inputting tags into your content,
 Now that we have the many-to-many tag relationship in place, we can fit in a way to render both sides of the relation. Here's more of the Wagtail demo site ``models.py``, where the index model for ``BlogPage`` is extended with logic for filtering the index by tag:
 
 .. code-block:: python
+    
+    from django.shortcuts import render
 
     class BlogIndexPage(Page):
         ...

docs/reference/pages/panels.rst

@@ -318,7 +318,7 @@ Let's look at the example of adding related links to a :class:`~wagtail.core.mod
   # Orderable helper class, and what amounts to a ForeignKey link
   # to the model we want to add related links to (BookPage)
   class BookPageRelatedLinks(Orderable, RelatedLink):
-      page = ParentalKey('demo.BookPage', related_name='related_links')
+      page = ParentalKey('demo.BookPage', on_delete=models.CASCADE, related_name='related_links')
 
   class BookPage(Page):
     # ...

docs/releases/2.0.rst

@@ -36,6 +36,7 @@ Other features
  * Added ``render_landing_page`` method to ``AbstractForm`` to be easily overridden and pass ``form_submission`` to landing page context (Stein Strindhaug)
  * Added ``heading`` kwarg to ``InlinePanel`` to allow heading to be set independently of button label (Adrian Turjak)
  * The value type returned from a ``StructBlock`` can now be customised. See :ref:`custom_value_class_for_structblock` (LB (Ben Johnston))
+ * Added `bgcolor` image operation (Karl Hobley)
 
 Bug fixes
 ~~~~~~~~~
@@ -61,6 +62,9 @@ Bug fixes
  * Fixed error on Elasticsearch backend when passing a queryset as an ``__in`` filter (Karl Hobley, Matt Westcott)
  * ``__isnull`` filters no longer fail on Elasticsearch 5 (Karl Hobley)
  * Prevented intermittent failures on Postgres search backend when a field is defined as both a ``SearchField`` and a ``FilterField`` (Matt Westcott)
+ * Alt text of images in rich text is no longer truncated on double-quote characters (Matt Westcott)
+ * Ampersands in embed URLs within rich text are no longer double-escaped (Matt Westcott)
+ * Using RGBA images no longer crashes with Pillow >= 4.2.0 (Karl Hobley)
 
 
 Upgrade considerations
@@ -72,6 +76,12 @@ Removed support for Python 2.7, Django 1.8 and Django 1.10
 Python 2.7, Django 1.8 and Django 1.10 are no longer supported in this release. You are advised to upgrade your project to Python 3 and Django 1.11 before upgrading to Wagtail 2.0.
 
 
+Added support for Django 2.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Before upgrading to Django 2.0, you are advised to review the `release notes <https://docs.djangoproject.com/en/2.0/releases/2.0/>`_, especially the `backwards incompatible changes <https://docs.djangoproject.com/en/2.0/releases/2.0/#backwards-incompatible-changes-in-2-0>`_ and `removed features <https://docs.djangoproject.com/en/2.0/releases/2.0/#features-removed-in-2-0>`_.
+
+
 Wagtail module path updates
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/topics/images.rst

@@ -253,6 +253,25 @@ For example, to make the tag always convert the image to a JPEG, use ``format-jp
 
 You may also use ``format-png`` or ``format-gif``.
 
+.. _image_background_color
+
+Background color
+----------------
+
+The PNG and GIF image formats both support transparency, but if you want to
+convert images to JPEG format, the transparency will need to be replaced with a
+solid background color.
+
+By default, Wagtail will set the background to white. But if a white background
+doesn't fit your design, you can specify a color using the ``bgcolor`` filter.
+
+This filter takes a single argument, which is a CSS 3 or 6 digit hex code
+representing the color you would like to use:
+
+.. code-block:: html+Django
+
+    {# Sets the image background to black #}
+    {% image page.photo width-400 bgcolor-000 format-jpeg %}
 
 .. _jpeg_image_quality:
 

docs/topics/pages.rst

@@ -77,7 +77,7 @@ This example represents a typical blog post:
 
 
     class BlogPageRelatedLink(Orderable):
-        page = ParentalKey(BlogPage, related_name='related_links')
+        page = ParentalKey(BlogPage, on_delete=models.CASCADE, related_name='related_links')
         name = models.CharField(max_length=255)
         url = models.URLField()
 
@@ -384,7 +384,7 @@ For example, the following inline model can be used to add related links (a list
 
 
     class BlogPageRelatedLink(Orderable):
-        page = ParentalKey(BlogPage, related_name='related_links')
+        page = ParentalKey(BlogPage, on_delete=models.CASCADE, related_name='related_links')
         name = models.CharField(max_length=255)
         url = models.URLField()
 

docs/topics/search/indexing.rst

@@ -219,7 +219,7 @@ To do this, inherit from ``index.Indexed`` and add some ``search_fields`` to the
     class Book(index.Indexed, models.Model):
         title = models.CharField(max_length=255)
         genre = models.CharField(max_length=255, choices=GENRE_CHOICES)
-        author = models.ForeignKey(Author)
+        author = models.ForeignKey(Author, on_delete=models.CASCADE)
         published_date = models.DateTimeField()
 
         search_fields = [

docs/topics/snippets.rst

@@ -137,8 +137,8 @@ To attach multiple adverts to a page, the ``SnippetChooserPanel`` can be placed
   ...
 
   class BookPageAdvertPlacement(Orderable, models.Model):
-      page = ParentalKey('demo.BookPage', related_name='advert_placements')
-      advert = models.ForeignKey('demo.Advert', related_name='+')
+      page = ParentalKey('demo.BookPage', on_delete=models.CASCADE, related_name='advert_placements')
+      advert = models.ForeignKey('demo.Advert', on_delete=models.CASCADE, related_name='+')
 
       class Meta:
           verbose_name = "advert placement"
@@ -218,7 +218,7 @@ Adding tags to snippets is very similar to adding tags to pages. The only differ
     from taggit.managers import TaggableManager
 
     class AdvertTag(TaggedItemBase):
-        content_object = ParentalKey('demo.Advert', related_name='tagged_items')
+        content_object = ParentalKey('demo.Advert', on_delete=models.CASCADE, related_name='tagged_items')
 
     @register_snippet
     class Advert(ClusterableModel):

setup.py

@@ -23,14 +23,14 @@ except ImportError:
 install_requires = [
     "Django>=1.11,<2.1",
     "django-modelcluster>=4.0,<5.0",
-    "django-taggit>=0.20,<1.0",
+    "django-taggit>=0.22.2,<1.0",
     "django-treebeard>=4.2.0,<5.0",
     "djangorestframework>=3.1.3,<4.0",
     "Pillow>=2.6.1,<5.0",
     "beautifulsoup4>=4.5.1,<5.0",
     "html5lib>=0.999,<1",
     "Unidecode>=0.04.14,<1.0",
-    "Willow>=1.0,<1.1",
+    "Willow>=1.1,<1.2",
     "requests>=2.11.1,<3.0",
 ]
 

tox.ini

@@ -42,7 +42,6 @@ deps =
     dj111: Django>=1.11b1,<2.0
     dj111-mssql: django-pyodbc-azure==1.11.0.0
     dj20: Django>=2.0,<2.1
-    dj20: git+https://github.com/jdufresne/django-taggit.git@dj20
     postgres: psycopg2>=2.6
     mysql: mysqlclient==1.3.6
     elasticsearch2: elasticsearch>=2,<3

wagtail/api/v2/filters.py

@@ -6,6 +6,7 @@ from taggit.managers import TaggableManager
 from wagtail.core import hooks
 from wagtail.core.models import Page
 from wagtail.search.backends import get_search_backend
+from wagtail.search.backends.base import FilterFieldError, OrderByFieldError
 
 from .utils import BadRequestError, pages_for_site, parse_boolean
 
@@ -117,7 +118,12 @@ class SearchFilter(BaseFilterBackend):
             order_by_relevance = 'order' not in request.GET
 
             sb = get_search_backend()
-            queryset = sb.search(search_query, queryset, operator=search_operator, order_by_relevance=order_by_relevance)
+            try:
+                queryset = sb.search(search_query, queryset, operator=search_operator, order_by_relevance=order_by_relevance)
+            except FilterFieldError as e:
+                raise BadRequestError("cannot filter by '{}' while searching (field is not indexed)".format(e.field_name))
+            except OrderByFieldError as e:
+                raise BadRequestError("cannot order by '{}' while searching (field is not indexed)".format(e.field_name))
 
         return queryset
 

wagtail/api/v2/tests/test_pages.py

@@ -705,6 +705,23 @@ class TestPageListing(TestCase):
 
         self.assertEqual(set(page_id_list), set([16, 18, 19]))
 
+    def test_search_with_filter(self):
+        response = self.get_response(title="Another blog post", search='blog', order='title')
+        content = json.loads(response.content.decode('UTF-8'))
+
+        page_id_list = self.get_page_id_list(content)
+
+        self.assertEqual(page_id_list, [19])
+
+    def test_search_with_filter_on_non_filterable_field(self):
+        response = self.get_response(type='demosite.BlogEntryPage', body="foo", search='blog', order='title')
+        content = json.loads(response.content.decode('UTF-8'))
+
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(content, {
+            'message': "cannot filter by 'body' while searching (field is not indexed)"
+        })
+
     def test_search_with_order(self):
         response = self.get_response(search='blog', order='title')
         content = json.loads(response.content.decode('UTF-8'))
@@ -713,6 +730,15 @@ class TestPageListing(TestCase):
 
         self.assertEqual(page_id_list, [19, 5, 16, 18])
 
+    def test_search_with_order_on_non_filterable_field(self):
+        response = self.get_response(type='demosite.BlogEntryPage', search='blog', order='body')
+        content = json.loads(response.content.decode('UTF-8'))
+
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(content, {
+            'message': "cannot order by 'body' while searching (field is not indexed)"
+        })
+
     @override_settings(WAGTAILAPI_SEARCH_ENABLED=False)
     def test_search_when_disabled_gives_error(self):
         response = self.get_response(search='blog')

wagtail/contrib/settings/views.py

@@ -1,7 +1,8 @@
+from functools import lru_cache
+
 from django.core.exceptions import PermissionDenied
 from django.http import Http404
 from django.shortcuts import get_object_or_404, redirect, render
-from django.utils.lru_cache import lru_cache
 from django.utils.text import capfirst
 from django.utils.translation import ugettext as _
 

wagtail/core/rich_text.py

@@ -148,10 +148,11 @@ FIND_ATTRS = re.compile(r'([\w-]+)\="([^"]*)"')
 
 def extract_attrs(attr_string):
     """
-    helper method to extract tag attributes as a dict. Does not escape HTML entities!
+    helper method to extract tag attributes, as a dict of un-escaped strings
     """
     attributes = {}
     for name, val in FIND_ATTRS.findall(attr_string):
+        val = val.replace('&lt;', '<').replace('&gt;', '>').replace('&quot;', '"').replace('&amp;', '&')
         attributes[name] = val
     return attributes
 

wagtail/core/tests/test_whitelist.py

@@ -143,3 +143,8 @@ class TestWhitelister(TestCase):
         string = '<b>snowman Yorkshire<!--[if gte mso 10]>MS word junk<![endif]--></b>'
         cleaned_string = Whitelister.clean(string)
         self.assertEqual(cleaned_string, '<b>snowman Yorkshire</b>')
+
+    def test_quoting(self):
+        string = '<img alt="Arthur &quot;two sheds&quot; Jackson" sheds="2">'
+        cleaned_string = Whitelister.clean(string)
+        self.assertEqual(cleaned_string, '<img alt="Arthur &quot;two sheds&quot; Jackson"/>')

wagtail/core/whitelist.py

@@ -5,6 +5,7 @@ specific rules.
 import re
 
 from bs4 import BeautifulSoup, Comment, NavigableString, Tag
+from django.utils.html import escape
 
 ALLOWED_URL_SCHEMES = ['http', 'https', 'ftp', 'mailto', 'tel']
 
@@ -96,7 +97,13 @@ class Whitelister:
         attributes"""
         doc = BeautifulSoup(html, 'html5lib')
         cls.clean_node(doc, doc)
-        return doc.decode()
+
+        # Pass strings through django.utils.html.escape when generating the final HTML.
+        # This differs from BeautifulSoup's default EntitySubstitution.substitute_html formatter
+        # in that it escapes " to &quot; as well as escaping < > & - if we don't do this, then
+        # BeautifulSoup will try to be clever and use single-quotes to wrap attribute values,
+        # which confuses our regexp-based db-HTML-to-real-HTML conversion.
+        return doc.decode(formatter=escape)
 
     @classmethod
     def clean_node(cls, doc, node):

wagtail/embeds/tests.py

@@ -11,6 +11,7 @@ from mock import patch
 
 from wagtail.tests.utils import WagtailTestUtils
 from wagtail.core import blocks
+from wagtail.core.rich_text import expand_db_html
 from wagtail.embeds import oembed_providers
 from wagtail.embeds.blocks import EmbedBlock, EmbedValue
 from wagtail.embeds.embeds import get_embed
@@ -615,3 +616,22 @@ class TestMediaEmbedHandler(TestCase):
         )
 
         self.assertEqual(result, '')
+
+    @patch('wagtail.embeds.embeds.get_embed')
+    def test_expand_html_escaping_end_to_end(self, get_embed):
+        get_embed.return_value = Embed(
+            url='http://www.youtube.com/watch/',
+            max_width=None,
+            type='video',
+            html='test html',
+            title='test title',
+            author_name='test author name',
+            provider_name='test provider name',
+            thumbnail_url='htto://test/thumbnail.url',
+            width=1000,
+            height=1000,
+        )
+
+        result = expand_db_html('<p>1 2 <embed embedtype="media" url="https://www.youtube.com/watch?v=O7D-1RG-VRk&amp;t=25" /> 3 4</p>')
+        self.assertIn('test html', result)
+        get_embed.assert_called_with('https://www.youtube.com/watch?v=O7D-1RG-VRk&t=25')

wagtail/images/checks.py

@@ -1,7 +1,7 @@
+from functools import lru_cache
 import os
 
 from django.core.checks import Warning, register
-from django.utils.lru_cache import lru_cache
 from willow.image import Image
 
 

wagtail/images/formats.py

@@ -19,7 +19,7 @@ class Format:
         when outputting this image within a rich text editor field
         """
         return 'data-embedtype="image" data-id="%d" data-format="%s" data-alt="%s" ' % (
-            image.id, self.name, alt_text
+            image.id, self.name, escape(alt_text)
         )
 
     def image_to_editor_html(self, image, alt_text):
@@ -37,7 +37,7 @@ class Format:
 
         return '<img %s%ssrc="%s" width="%d" height="%d" alt="%s">' % (
             extra_attributes, class_attr,
-            escape(rendition.url), rendition.width, rendition.height, alt_text
+            escape(rendition.url), rendition.width, rendition.height, escape(alt_text)
         )
 
 

wagtail/images/image_operations.py

@@ -2,6 +2,7 @@ import inspect
 
 from wagtail.images.exceptions import InvalidFilterSpecError
 from wagtail.images.rect import Rect
+from wagtail.images.utils import parse_color_string
 
 
 class Operation:
@@ -236,3 +237,11 @@ class FormatOperation(Operation):
 
     def run(self, willow, image, env):
         env['output-format'] = self.format
+
+
+class BackgroundColorOperation(Operation):
+    def construct(self, color_string):
+        self.color = parse_color_string(color_string)
+
+    def run(self, willow, image, env):
+        return willow.set_background_color_rgb(self.color)

wagtail/images/models.py

@@ -395,6 +395,10 @@ class Filter:
                 else:
                     quality = 85
 
+                # If the image has an alpha channel, give it a white background
+                if willow.has_alpha():
+                    willow = willow.set_background_color_rgb((255, 255, 255))
+
                 return willow.save_as_jpeg(output, quality=quality, progressive=True, optimize=True)
             elif output_format == 'png':
                 return willow.save_as_png(output)

wagtail/images/rich_text.py

@@ -37,6 +37,6 @@ class ImageEmbedHandler:
         image_format = get_image_format(attrs['format'])
 
         if for_editor:
-            return image_format.image_to_editor_html(image, attrs['alt'])
+            return image_format.image_to_editor_html(image, attrs.get('alt', ''))
         else:
-            return image_format.image_to_html(image, attrs['alt'])
+            return image_format.image_to_html(image, attrs.get('alt', ''))

wagtail/images/tests/test_admin_views.py

@@ -1,4 +1,5 @@
 import json
+import re
 
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import Group, Permission
@@ -620,6 +621,9 @@ class TestImageChooserSelectFormatView(TestCase, WagtailTestUtils):
     def get(self, params={}):
         return self.client.get(reverse('wagtailimages:chooser_select_format', args=(self.image.id,)), params)
 
+    def post(self, post_data={}):
+        return self.client.post(reverse('wagtailimages:chooser_select_format', args=(self.image.id,)), post_data)
+
     def test_simple(self):
         response = self.get()
         self.assertEqual(response.status_code, 200)
@@ -631,6 +635,23 @@ class TestImageChooserSelectFormatView(TestCase, WagtailTestUtils):
         self.assertEqual(response.status_code, 200)
         self.assertContains(response, 'value=\\"some previous alt text\\"')
 
+    def test_post_response(self):
+        response = self.post({'format': 'left', 'alt_text': 'Arthur "two sheds" Jackson'})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response['Content-Type'], 'text/javascript')
+
+        # extract data as json from the code line: modal.respond('imageChosen', {json});
+        match = re.search(r'modal.respond\(\'imageChosen\', ([^\)]+)\);', response.content.decode())
+        self.assertTrue(match)
+        response_json = json.loads(match.group(1))
+
+        self.assertEqual(response_json['id'], self.image.id)
+        self.assertEqual(response_json['title'], "Test image")
+        self.assertEqual(response_json['format'], 'left')
+        self.assertEqual(response_json['alt'], 'Arthur "two sheds" Jackson')
+        self.assertIn('alt="Arthur "two sheds" Jackson"', response_json['html'])
+
 
 class TestImageChooserUploadView(TestCase, WagtailTestUtils):
     def setUp(self):

wagtail/images/tests/test_image_operations.py

@@ -544,3 +544,44 @@ class TestJPEGQualityFilter(TestCase):
             fil.run(image, f)
 
         save.assert_called_with(f, 'JPEG', quality=40, optimize=True, progressive=True)
+
+
+class TestBackgroundColorFilter(TestCase):
+    def test_original_has_alpha(self):
+        # Checks that the test image we're using has alpha
+        fil = Filter(spec='width-400')
+        image = Image.objects.create(
+            title="Test image",
+            file=get_test_image_file(),
+        )
+        out = fil.run(image, BytesIO())
+
+        self.assertTrue(out.has_alpha())
+
+    def test_3_digit_hex(self):
+        fil = Filter(spec='width-400|bgcolor-fff')
+        image = Image.objects.create(
+            title="Test image",
+            file=get_test_image_file(),
+        )
+        out = fil.run(image, BytesIO())
+
+        self.assertFalse(out.has_alpha())
+
+    def test_6_digit_hex(self):
+        fil = Filter(spec='width-400|bgcolor-ffffff')
+        image = Image.objects.create(
+            title="Test image",
+            file=get_test_image_file(),
+        )
+        out = fil.run(image, BytesIO())
+
+        self.assertFalse(out.has_alpha())
+
+    def test_invalid(self):
+        fil = Filter(spec='width-400|bgcolor-foo')
+        image = Image.objects.create(
+            title="Test image",
+            file=get_test_image_file(),
+        )
+        self.assertRaises(ValueError, fil.run, image, BytesIO())

wagtail/images/tests/test_rich_text.py

@@ -36,6 +36,26 @@ class TestImageEmbedHandler(TestCase):
         )
         self.assertIn('<img class="richtext-image left"', result)
 
+    def test_expand_db_attributes_escapes_alt_text(self):
+        Image.objects.create(id=1, title='Test', file=get_test_image_file())
+        result = ImageEmbedHandler.expand_db_attributes(
+            {'id': 1,
+             'alt': 'Arthur "two sheds" Jackson',
+             'format': 'left'},
+            False
+        )
+        self.assertIn('alt="Arthur &quot;two sheds&quot; Jackson"', result)
+
+    def test_expand_db_attributes_with_missing_alt(self):
+        Image.objects.create(id=1, title='Test', file=get_test_image_file())
+        result = ImageEmbedHandler.expand_db_attributes(
+            {'id': 1,
+             'format': 'left'},
+            False
+        )
+        self.assertIn('<img class="richtext-image left"', result)
+        self.assertIn('alt=""', result)
+
     def test_expand_db_attributes_for_editor(self):
         Image.objects.create(id=1, title='Test', file=get_test_image_file())
         result = ImageEmbedHandler.expand_db_attributes(
@@ -48,3 +68,29 @@ class TestImageEmbedHandler(TestCase):
             '<img data-embedtype="image" data-id="1" data-format="left" '
             'data-alt="test-alt" class="richtext-image left"', result
         )
+
+    def test_expand_db_attributes_for_editor_escapes_alt_text(self):
+        Image.objects.create(id=1, title='Test', file=get_test_image_file())
+        result = ImageEmbedHandler.expand_db_attributes(
+            {'id': 1,
+             'alt': 'Arthur "two sheds" Jackson',
+             'format': 'left'},
+            True
+        )
+        self.assertIn(
+            '<img data-embedtype="image" data-id="1" data-format="left" '
+            'data-alt="Arthur &quot;two sheds&quot; Jackson" class="richtext-image left"', result
+        )
+        self.assertIn('alt="Arthur &quot;two sheds&quot; Jackson"', result)
+
+    def test_expand_db_attributes_for_editor_with_missing_alt(self):
+        Image.objects.create(id=1, title='Test', file=get_test_image_file())
+        result = ImageEmbedHandler.expand_db_attributes(
+            {'id': 1,
+             'format': 'left'},
+            True
+        )
+        self.assertIn(
+            '<img data-embedtype="image" data-id="1" data-format="left" '
+            'data-alt="" class="richtext-image left"', result
+        )

wagtail/images/tests/tests.py

@@ -208,6 +208,17 @@ class TestFormat(TestCase):
             'data-alt="test alt text" class="test classnames" src="[^"]+" width="1" height="1" alt="test alt text">',
         )
 
+    def test_image_to_editor_html_with_quoting(self):
+        result = self.format.image_to_editor_html(
+            self.image,
+            'Arthur "two sheds" Jackson'
+        )
+        self.assertRegex(
+            result,
+            '<img data-embedtype="image" data-id="0" data-format="test name" '
+            'data-alt="Arthur &quot;two sheds&quot; Jackson" class="test classnames" src="[^"]+" width="1" height="1" alt="Arthur &quot;two sheds&quot; Jackson">',
+        )
+
     def test_image_to_html_no_classnames(self):
         self.format.classnames = None
         result = self.format.image_to_html(self.image, 'test alt text')
@@ -217,6 +228,13 @@ class TestFormat(TestCase):
         )
         self.format.classnames = 'test classnames'
 
+    def test_image_to_html_with_quoting(self):
+        result = self.format.image_to_html(self.image, 'Arthur "two sheds" Jackson')
+        self.assertRegex(
+            result,
+            '<img class="test classnames" src="[^"]+" width="1" height="1" alt="Arthur &quot;two sheds&quot; Jackson">'
+        )
+
     def test_get_image_format(self):
         register_image_format(self.format)
         result = get_image_format('test name')

wagtail/images/tests/utils.py

@@ -10,7 +10,7 @@ Image = get_image_model()
 
 def get_test_image_file(filename='test.png', colour='white', size=(640, 480)):
     f = BytesIO()
-    image = PIL.Image.new('RGB', size, colour)
+    image = PIL.Image.new('RGBA', size, colour)
     image.save(f, 'PNG')
     return ImageFile(f, name=filename)
 

wagtail/images/utils.py

@@ -37,3 +37,26 @@ def get_fill_filter_spec_migrations(app_name, rendition_model_name):
                 Rendition.objects.using(db_alias).filter(filter_spec=filter_spec).update(filter=filter)
 
     return (fill_filter_spec_forward, fill_filter_spec_reverse)
+
+
+def parse_color_string(color_string):
+    """
+    Parses a string a user typed into a tuple of 3 integers representing the
+    red, green and blue channels respectively.
+
+    May raise a ValueError if the string cannot be parsed.
+
+    The colour string must be a CSS 3 or 6 digit hex code without the '#' prefix.
+    """
+    if len(color_string) == 3:
+        r = int(color_string[0], 16) * 17
+        g = int(color_string[1], 16) * 17
+        b = int(color_string[2], 16) * 17
+    elif len(color_string) == 6:
+        r = int(color_string[0:2], 16)
+        g = int(color_string[2:4], 16)
+        b = int(color_string[4:6], 16)
+    else:
+        ValueError('Color string must be either 3 or 6 hexadecimal digits long')
+
+    return r, g, b

wagtail/images/wagtail_hooks.py

@@ -86,6 +86,7 @@ def register_image_operations():
         ('height', image_operations.WidthHeightOperation),
         ('jpegquality', image_operations.JPEGQualityOperation),
         ('format', image_operations.FormatOperation),
+        ('bgcolor', image_operations.BackgroundColorOperation),
     ]
 
 

wagtail/search/backends/base.py

@@ -14,6 +14,20 @@ class FilterError(Exception):
 
 
 class FieldError(Exception):
+    def __init__(self, *args, field_name=None, **kwargs):
+        self.field_name = field_name
+        super(FieldError, self).__init__(*args, **kwargs)
+
+
+class SearchFieldError(FieldError):
+    pass
+
+
+class FilterFieldError(FieldError):
+    pass
+
+
+class OrderByFieldError(FieldError):
     pass
 
 
@@ -48,18 +62,20 @@ class BaseSearchQueryCompiler:
     def _connect_filters(self, filters, connector, negated):
         raise NotImplementedError
 
-    def _process_filter(self, field_attname, lookup, value):
+    def _process_filter(self, field_attname, lookup, value, check_only=False):
         # Get the field
         field = self._get_filterable_field(field_attname)
 
         if field is None:
-            raise FieldError(
+            raise FilterFieldError(
                 'Cannot filter search results with field "' + field_attname + '". Please add index.FilterField(\'' +
-                field_attname + '\') to ' + self.queryset.model.__name__ + '.search_fields.'
+                field_attname + '\') to ' + self.queryset.model.__name__ + '.search_fields.',
+                field_name=field_attname
             )
 
         # Process the lookup
-        result = self._process_lookup(field, lookup, value)
+        if not check_only:
+            result = self._process_lookup(field, lookup, value)
 
         if result is None:
             raise FilterError(
@@ -69,7 +85,7 @@ class BaseSearchQueryCompiler:
 
         return result
 
-    def _get_filters_from_where_node(self, where_node):
+    def _get_filters_from_where_node(self, where_node, check_only=False):
         # Check if this is a leaf node
         if isinstance(where_node, Lookup):
             field_attname = where_node.lhs.target.attname
@@ -81,7 +97,7 @@ class BaseSearchQueryCompiler:
                 return
 
             # Process the filter
-            return self._process_filter(field_attname, lookup, value)
+            return self._process_filter(field_attname, lookup, value, check_only=check_only)
 
         elif isinstance(where_node, SubqueryConstraint):
             raise FilterError('Could not apply filter on search results: Subqueries are not allowed.')
@@ -90,9 +106,10 @@ class BaseSearchQueryCompiler:
             # Get child filters
             connector = where_node.connector
             child_filters = [self._get_filters_from_where_node(child) for child in where_node.children]
-            child_filters = [child_filter for child_filter in child_filters if child_filter]
 
-            return self._connect_filters(child_filters, connector, where_node.negated)
+            if not check_only:
+                child_filters = [child_filter for child_filter in child_filters if child_filter]
+                return self._connect_filters(child_filters, connector, where_node.negated)
 
         else:
             raise FilterError('Could not apply filter on search results: Unknown where node: ' + str(type(where_node)))
@@ -114,13 +131,35 @@ class BaseSearchQueryCompiler:
             field = self._get_filterable_field(field_name)
 
             if field is None:
-                raise FieldError(
+                raise OrderByFieldError(
                     'Cannot sort search results with field "' + field_name + '". Please add index.FilterField(\'' +
-                    field_name + '\') to ' + self.queryset.model.__name__ + '.search_fields.'
+                    field_name + '\') to ' + self.queryset.model.__name__ + '.search_fields.',
+                    field_name=field_name
                 )
 
             yield reverse, field
 
+    def check(self):
+        # Check search fields
+        if self.fields:
+            allowed_fields = {field.field_name for field in self.queryset.model.get_searchable_search_fields()}
+
+            for field_name in self.fields:
+                if field_name not in allowed_fields:
+                    raise SearchFieldError(
+                        'Cannot search with field "' + field_name + '". Please add index.SearchField(\'' +
+                        field_name + '\') to ' + self.queryset.model.__name__ + '.search_fields.',
+                        field_name=field_name
+                    )
+
+        # Check where clause
+        # Raises FilterFieldError if an unindexed field is being filtered on
+        self._get_filters_from_where_node(self.queryset.query.where, check_only=True)
+
+        # Check order by
+        # Raises OrderByFieldError if an unindexed field is being used to order by
+        list(self._get_order_by())
+
 
 class BaseSearchResults:
     def __init__(self, backend, query_compiler, prefetch_related=None):
@@ -278,17 +317,6 @@ class BaseSearchBackend:
         if query == "":
             return EmptySearchResults()
 
-        # Only fields that are indexed as a SearchField can be passed in fields
-        if fields:
-            allowed_fields = {field.field_name for field in model.get_searchable_search_fields()}
-
-            for field_name in fields:
-                if field_name not in allowed_fields:
-                    raise FieldError(
-                        'Cannot search with field "' + field_name + '". Please add index.SearchField(\'' +
-                        field_name + '\') to ' + model.__name__ + '.search_fields.'
-                    )
-
         # Apply filters to queryset
         if filters:
             queryset = queryset.filter(**filters)
@@ -302,4 +330,8 @@ class BaseSearchBackend:
         search_query = self.query_compiler_class(
             queryset, query, fields=fields, operator=operator, order_by_relevance=order_by_relevance
         )
+
+        # Check the query
+        search_query.check()
+
         return self.results_class(self, search_query)

wagtail/search/backends/db.py

@@ -94,9 +94,6 @@ class DatabaseSearchResults(BaseSearchResults):
     def _do_search(self):
         queryset = self.get_queryset()
 
-        # Call query._get_order_by so it can raise errors if a non-indexed field is used for ordering
-        list(self.query_compiler._get_order_by())
-
         if self._score_field:
             queryset = queryset.annotate(**{self._score_field: Value(None, output_field=models.FloatField())})
 

wagtail/search/backends/elasticsearch2.py

@@ -277,7 +277,9 @@ class Elasticsearch2SearchQueryCompiler(BaseSearchQueryCompiler):
 
                 fields.append(field_name)
 
-            self.fields = fields
+            self.remapped_fields = fields
+        else:
+            self.remapped_fields = None
 
     def _process_lookup(self, field, lookup, value):
         column_name = self.mapping.get_field_column_name(field)
@@ -379,7 +381,7 @@ class Elasticsearch2SearchQueryCompiler(BaseSearchQueryCompiler):
                 '`%s` is not supported by the Elasticsearch search backend.'
                 % self.query.__class__.__name__)
 
-        fields = self.fields or ['_all', '_partials']
+        fields = self.remapped_fields or ['_all', '_partials']
         operator = self.query.operator
 
         if len(fields) == 1:

wagtail/search/tests/elasticsearch_common_tests.py

@@ -108,7 +108,7 @@ class ElasticsearchCommonSearchBackendTests(BackendTests):
 
     def test_update_index_command_schema_only(self):
         management.call_command(
-            'update_index', backend_name=self.backend_name, schema_only=True, interactive=False, stdout=StringIO()
+            'update_index', backend_name=self.backend_name, schema_only=True, stdout=StringIO()
         )
 
         # This should not give any results

wagtail/search/tests/test_backends.py

@@ -402,15 +402,17 @@ class BackendTests(WagtailTestUtils):
         self.assertSetEqual(results_across_pages, same_rank_objects)
 
     def test_delete(self):
-        # Delete foundation
-        models.Book.objects.filter(title="Foundation").delete()
+        foundation = models.Novel.objects.filter(title="Foundation").first()
 
-        # Refresh the index
-        # Note: The delete signal handler should've removed the book, but we still need to refresh the index manually
-        index = self.backend.get_index_for_model(models.Book)
+        # Delete from the search index
+        index = self.backend.get_index_for_model(models.Novel)
         if index:
+            index.delete_item(foundation)
             index.refresh()
 
+        # Delete from the database
+        foundation.delete()
+
         # To test that the book was deleted from the index as well, we will perform the slicing check from an earlier
         # test where "Foundation" was the first result. We need to test it this way so we can pick up the case where
         # the object still exists in the index but not in the database (in that case, just two objects would be returned
@@ -423,9 +425,10 @@ class BackendTests(WagtailTestUtils):
         results = results[:3]
 
         self.assertEqual(list(r.title for r in results), [
+            # "Foundation"
             "The Hobbit",
             "The Two Towers",
-            "The Fellowship of the Ring"
+            "The Fellowship of the Ring"  # If this item doesn't appear, "Foundation" is still in the index
         ])
 
 

wagtail/search/tests/test_db_backend.py

@@ -18,6 +18,11 @@ class TestDBBackend(QueryAPITestMixin, BackendTests, TestCase):
     def test_search_boosting_on_related_fields(self):
         super().test_search_boosting_on_related_fields()
 
+    # Doesn't support ranking
+    @unittest.expectedFailure
+    def test_same_rank_pages(self):
+        super(TestDBBackend, self).test_same_rank_pages()
+
     # Doesn't support searching specific fields
     @unittest.expectedFailure
     def test_search_child_class_field_from_parent(self):
@@ -32,8 +37,3 @@ class TestDBBackend(QueryAPITestMixin, BackendTests, TestCase):
     @unittest.expectedFailure
     def test_search_callable_field(self):
         super().test_search_callable_field()
-
-    # Doesn't support the index API used in this test
-    @unittest.expectedFailure
-    def test_same_rank_pages(self):
-        super().test_same_rank_pages()

wagtail/wagtailsearch/tests/test_page_search.py

@@ -0,0 +1,49 @@
+from __future__ import absolute_import, unicode_literals
+
+from django.conf import settings
+from django.test import TestCase
+
+from wagtail.wagtailcore.models import Page
+from wagtail.wagtailsearch.backends import get_search_backend
+
+
+class PageSearchTests(object):
+    # A TestCase with this class mixed in will be dynamically created
+    # for each search backend defined in WAGTAILSEARCH_BACKENDS, with the backend name available
+    # as self.backend_name
+
+    fixtures = ['test.json']
+
+    def setUp(self):
+        self.backend = get_search_backend(self.backend_name)
+        self.reset_index()
+        for page in Page.objects.all():
+            self.backend.add(page)
+        self.refresh_index()
+
+    def reset_index(self):
+        if self.backend.rebuilder_class:
+            index = self.backend.get_index_for_model(Page)
+            rebuilder = self.backend.rebuilder_class(index)
+            index = rebuilder.start()
+            index.add_model(Page)
+            rebuilder.finish()
+
+    def refresh_index(self):
+        index = self.backend.get_index_for_model(Page)
+        if index:
+            index.refresh()
+
+    def test_order_by_title(self):
+        list(Page.objects.order_by('title').search('blah', order_by_relevance=False, backend=self.backend_name))
+
+    def test_search_specific_queryset(self):
+        list(Page.objects.specific().search('bread', backend=self.backend_name))
+
+    def test_search_specific_queryset_with_fields(self):
+        list(Page.objects.specific().search('bread', fields=['title'], backend=self.backend_name))
+
+
+for backend_name in settings.WAGTAILSEARCH_BACKENDS.keys():
+    test_name = str("Test%sBackend" % backend_name.title())
+    globals()[test_name] = type(test_name, (PageSearchTests, TestCase,), {'backend_name': backend_name})