Fixed #10762, #17514 -- Prevented the GZip middleware from returning a response longer than the original content, allowed compression of non-200 responses, and added tests (there were none). Thanks cannona for the initial patch.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@17365 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/middleware/gzip.py

@@ -12,8 +12,8 @@ class GZipMiddleware(object):
     on the Accept-Encoding header.
     """
     def process_response(self, request, response):
-        # It's not worth compressing non-OK or really short responses.
-        if response.status_code != 200 or len(response.content) < 200:
+        # It's not worth attempting to compress really short responses.
+        if len(response.content) < 200:
             return response
 
         patch_vary_headers(response, ('Accept-Encoding',))
@@ -32,7 +32,12 @@ class GZipMiddleware(object):
         if not re_accepts_gzip.search(ae):
             return response
 
-        response.content = compress_string(response.content)
+        # Return the compressed content only if it's actually shorter.
+        compressed_content = compress_string(response.content)
+        if len(compressed_content) >= len(response.content):
+            return response
+
+        response.content = compressed_content
         response['Content-Encoding'] = 'gzip'
         response['Content-Length'] = str(len(response.content))
         return response

docs/ref/middleware.txt

@@ -82,22 +82,29 @@ addresses defined in the :setting:`INTERNAL_IPS` setting. This is used by
 Django's :doc:`automatic documentation system </ref/contrib/admin/admindocs>`.
 Depends on :class:`~django.contrib.auth.middleware.AuthenticationMiddleware`.
 
-GZIP middleware
+GZip middleware
 ---------------
 
 .. module:: django.middleware.gzip
-   :synopsis: Middleware to serve gziped content for performance.
+   :synopsis: Middleware to serve GZipped content for performance.
 
 .. class:: GZipMiddleware
 
-Compresses content for browsers that understand gzip compression (all modern
+Compresses content for browsers that understand GZip compression (all modern
 browsers).
 
 It is suggested to place this first in the middleware list, so that the
-compression of the response content is the last thing that happens. Will not
-compress content bodies less than 200 bytes long, when the response code is
-something other than 200, JavaScript files (for IE compatibility), or
-responses that have the ``Content-Encoding`` header already specified.
+compression of the response content is the last thing that happens.
+
+It will not compress content bodies less than 200 bytes long, when the
+``Content-Encoding`` header is already set, or when the browser does not send
+an ``Accept-Encoding`` header containing ``gzip``.
+
+Content will also not be compressed when the browser is Internet Explorer and
+the ``Content-Type`` header contains ``javascript`` or starts with anything
+other than ``text/``. This is done to overcome a bug present in early versions
+of Internet Explorer which caused decompression not to be performed on certain
+content types.
 
 GZip compression can be applied to individual views using the
 :func:`~django.views.decorators.http.gzip_page()` decorator.

tests/regressiontests/middleware/tests.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
 
+import gzip
 import re
+import random
+import StringIO
 
 from django.conf import settings
 from django.core import mail
@@ -9,6 +12,7 @@ from django.http import HttpResponse
 from django.middleware.clickjacking import XFrameOptionsMiddleware
 from django.middleware.common import CommonMiddleware
 from django.middleware.http import ConditionalGetMiddleware
+from django.middleware.gzip import GZipMiddleware
 from django.test import TestCase
 
 
@@ -495,3 +499,86 @@ class XFrameOptionsMiddlewareTest(TestCase):
         r = OtherXFrameOptionsMiddleware().process_response(HttpRequest(),
                                                        HttpResponse())
         self.assertEqual(r['X-Frame-Options'], 'DENY')
+
+
+class GZipMiddlewareTest(TestCase):
+    """
+    Tests the GZip middleware.
+    """
+    short_string = "This string is too short to be worth compressing."
+    compressible_string = 'a' * 500
+    uncompressible_string = ''.join(chr(random.randint(0, 255)) for _ in xrange(500))
+
+    def setUp(self):
+        self.req = HttpRequest()
+        self.req.META = {
+            'SERVER_NAME': 'testserver',
+            'SERVER_PORT': 80,
+        }
+        self.req.path = self.req.path_info = "/"
+        self.req.META['HTTP_ACCEPT_ENCODING'] = 'gzip, deflate'
+        self.req.META['HTTP_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'
+        self.resp = HttpResponse()
+        self.resp.status_code = 200
+        self.resp.content = self.compressible_string
+        self.resp['Content-Type'] = 'text/html; charset=UTF-8'
+
+    @staticmethod
+    def decompress(gzipped_string):
+        return gzip.GzipFile(mode='rb', fileobj=StringIO.StringIO(gzipped_string)).read()
+
+    def test_compress_response(self):
+        """
+        Tests that compression is performed on responses with compressible content.
+        """
+        r = GZipMiddleware().process_response(self.req, self.resp)
+        self.assertEqual(self.decompress(r.content), self.compressible_string)
+        self.assertEqual(r.get('Content-Encoding'), 'gzip')
+        self.assertEqual(r.get('Content-Length'), str(len(r.content)))
+
+    def test_compress_non_200_response(self):
+        """
+        Tests that compression is performed on responses with a status other than 200.
+        See #10762.
+        """
+        self.resp.status_code = 404
+        r = GZipMiddleware().process_response(self.req, self.resp)
+        self.assertEqual(self.decompress(r.content), self.compressible_string)
+        self.assertEqual(r.get('Content-Encoding'), 'gzip')
+
+    def test_no_compress_short_response(self):
+        """
+        Tests that compression isn't performed on responses with short content.
+        """
+        self.resp.content = self.short_string
+        r = GZipMiddleware().process_response(self.req, self.resp)
+        self.assertEqual(r.content, self.short_string)
+        self.assertEqual(r.get('Content-Encoding'), None)
+
+    def test_no_compress_compressed_response(self):
+        """
+        Tests that compression isn't performed on responses that are already compressed.
+        """
+        self.resp['Content-Encoding'] = 'deflate'
+        r = GZipMiddleware().process_response(self.req, self.resp)
+        self.assertEqual(r.content, self.compressible_string)
+        self.assertEqual(r.get('Content-Encoding'), 'deflate')
+
+    def test_no_compress_ie_js_requests(self):
+        """
+        Tests that compression isn't performed on JavaScript requests from Internet Explorer.
+        """
+        self.req.META['HTTP_USER_AGENT'] = 'Mozilla/4.0 (compatible; MSIE 5.00; Windows 98)'
+        self.resp['Content-Type'] = 'application/javascript; charset=UTF-8'
+        r = GZipMiddleware().process_response(self.req, self.resp)
+        self.assertEqual(r.content, self.compressible_string)
+        self.assertEqual(r.get('Content-Encoding'), None)
+
+    def test_no_compress_uncompressible_response(self):
+        """
+        Tests that compression isn't performed on responses with uncompressible content.
+        """
+        self.resp.content = self.uncompressible_string
+        r = GZipMiddleware().process_response(self.req, self.resp)
+        self.assertEqual(r.content, self.uncompressible_string)
+        self.assertEqual(r.get('Content-Encoding'), None)