copy files from fixtures dir to MEDIA_ROOT (or remote storage) during load_initial_data (#255)

Note: This change will require any existing deployments to manually copy
bakerydemo/base/fixtures/media/original_images/ into bakerydemo/media/

.gitignore

@@ -9,7 +9,6 @@
 /data
 .~*
 common/CACHE
-bakerydemo/media/images/*
 bakerydemo/settings/local.py
 bakerydemodb
 __pycache__

.travis.yml

@@ -22,6 +22,9 @@ script:
   - docker-compose run app /bin/sh -c "/venv/bin/pip install flake8 && /venv/bin/flake8 ."
   - docker-compose run app /venv/bin/python /code/manage.py check
   - docker-compose run app /venv/bin/python /code/manage.py makemigrations --check
+  - docker-compose run app /venv/bin/python /code/manage.py load_initial_data
+  # Check for broken image links (wget will have a non-zero exit code if it encounters a 404)
+  - wget -m http://localhost:8000
 
 after_script:
   - docker-compose logs

bakerydemo/base/management/commands/load_initial_data.py

@@ -1,6 +1,7 @@
 import os
 
 from django.conf import settings
+from django.core.files.storage import default_storage, FileSystemStorage
 from django.core.management.base import BaseCommand
 from django.core.management import call_command
 
@@ -8,10 +9,28 @@ from wagtail.core.models import Site, Page
 
 
 class Command(BaseCommand):
+    def _copy_files(self, local_storage, path):
+        """
+        Recursively copy files from local_storage to default_storage. Used
+        to automatically bootstrap the media directory (both locally and on
+        cloud providers) with the images linked from the initial data (and
+        included in MEDIA_ROOT).
+        """
+        directories, file_names = local_storage.listdir(path)
+        for directory in directories:
+            self._copy_files(local_storage, path + directory + '/')
+        for file_name in file_names:
+            with local_storage.open(path + file_name) as file_:
+                default_storage.save(path + file_name, file_)
+
     def handle(self, **options):
         fixtures_dir = os.path.join(settings.PROJECT_DIR, 'base', 'fixtures')
         fixture_file = os.path.join(fixtures_dir, 'bakerydemo.json')
 
+        print("Copying media files to configured storage...")
+        local_storage = FileSystemStorage(os.path.join(fixtures_dir, 'media'))
+        self._copy_files(local_storage, '')  # file storage paths are relative
+
         # Wagtail creates default Site and Page instances during install, but we already have
         # them in the data load. Remove the auto-generated ones.
         if Site.objects.filter(hostname='localhost').exists():

bakerydemo/media/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

docker-compose.yml

@@ -29,6 +29,8 @@ services:
     build:
       context: .
       dockerfile: ./Dockerfile
+    volumes:
+      - media-root:/code/bakerydemo/media
     links:
       - db:db
       - redis:redis
@@ -39,3 +41,5 @@ services:
       - db
       - redis
       - elasticsearch
+volumes:
+  media-root:

readme.md

@@ -188,10 +188,8 @@ environment, set the same environment variables there using the method appropria
 
 Once Heroku restarts your application or your Docker container is refreshed, you should have persistent media storage!
 
-To copy the initial data included with this demo to the S3 bucket (assuming you ran `./manage.py load_initial_data` per
-the above), you can use the AWS CLI included with the requirements:
-
-    heroku run aws s3 sync bakerydemo/media/original_images/ s3://<bucket-name>/original_images/
+Running `./manage.py load_initial_data` will copy local images to S3, but if you set up S3 after you ran it the first
+time you might need to run it again.
 
 # Next steps
 

requirements/production.txt

@@ -15,6 +15,3 @@ botocore==1.12.33
 aws-requests-auth==0.4.0
 django-redis==4.8.0
 django_cache_url==2.0.0
-# For copying initial media to S3 bucket
-awscli==1.16.43
-urllib3>=1.21.1,<1.26