Chunk Cloudflare frontend cache purging to fit within API limits (#5721)

* Chunk Cloudflare frontend cache purging to fit within API limits

* Add exception to tests to catch if Cloudflare chunking isn't working

wagtail/contrib/frontend_cache/backends.py

@@ -66,6 +66,8 @@ class HTTPBackend(BaseBackend):
 
 
 class CloudflareBackend(BaseBackend):
+    CHUNK_SIZE = 30
+
     def __init__(self, params):
         self.cloudflare_email = params.pop("EMAIL", None)
         self.cloudflare_api_key = (
@@ -84,8 +86,7 @@ class CloudflareBackend(BaseBackend):
                 "The setting 'WAGTAILFRONTENDCACHE' requires both 'EMAIL' and 'API_KEY', or 'BEARER_TOKEN' to be specified."
             )
 
-
-    def purge_batch(self, urls):
+    def _purge_urls(self, urls):
         try:
             purge_url = 'https://api.cloudflare.com/client/v4/zones/{0}/purge_cache'.format(self.cloudflare_zoneid)
 
@@ -125,6 +126,13 @@ class CloudflareBackend(BaseBackend):
                 logger.error("Couldn't purge '%s' from Cloudflare. Cloudflare errors '%s'", url, error_messages)
             return
 
+    def purge_batch(self, urls):
+        # Break the batched URLs in to chunks to fit within Cloudflare's maximum size for
+        # the purge_cache call (https://api.cloudflare.com/#zone-purge-files-by-url)
+        for i in range(0, len(urls), self.CHUNK_SIZE):
+            chunk = urls[i:i + self.CHUNK_SIZE]
+            self._purge_urls(chunk)
+
     def purge(self, url):
         self.purge_batch([url])
 

wagtail/contrib/frontend_cache/tests.py

@@ -198,6 +198,17 @@ class MockBackend(BaseBackend):
         PURGED_URLS.append(url)
 
 
+class MockCloudflareBackend(CloudflareBackend):
+    def __init__(self, config):
+        pass
+
+    def _purge_urls(self, urls):
+        if len(urls) > self.CHUNK_SIZE:
+            raise Exception("Cloudflare backend is not chunking requests as expected")
+
+        PURGED_URLS.extend(urls)
+
+
 @override_settings(WAGTAILFRONTENDCACHE={
     'varnish': {
         'BACKEND': 'wagtail.contrib.frontend_cache.tests.MockBackend',
@@ -238,6 +249,25 @@ class TestCachePurgingFunctions(TestCase):
         self.assertEqual(PURGED_URLS, ['http://localhost/events/', 'http://localhost/events/past/', 'http://localhost/foo'])
 
 
+@override_settings(WAGTAILFRONTENDCACHE={
+    'cloudflare': {
+        'BACKEND': 'wagtail.contrib.frontend_cache.tests.MockCloudflareBackend',
+    },
+})
+class TestCloudflareCachePurgingFunctions(TestCase):
+    def setUp(self):
+        # Reset PURGED_URLS to an empty list
+        PURGED_URLS[:] = []
+
+    def test_cloudflare_purge_batch_chunked(self):
+        batch = PurgeBatch()
+        urls = ['https://localhost/foo{}'.format(i) for i in range(1, 65)]
+        batch.add_urls(urls)
+        batch.purge()
+
+        self.assertCountEqual(PURGED_URLS, urls)
+
+
 @override_settings(WAGTAILFRONTENDCACHE={
     'varnish': {
         'BACKEND': 'wagtail.contrib.frontend_cache.tests.MockBackend',