django/tests/admin_scripts/tests.py

"""
A series of tests to establish that the command-line management tools work as
advertised - especially with regards to the handling of the
DJANGO_SETTINGS_MODULE and default settings.py files.
"""
import os
import re
import shutil
import socket
import subprocess
import sys
import tempfile
import unittest
from io import StringIO
from unittest import mock

from django import conf, get_version
from django.conf import settings
from django.core.management import (
    BaseCommand, CommandError, call_command, color,
)
from django.core.management.commands.loaddata import Command as LoaddataCommand
from django.core.management.commands.runserver import (
    Command as RunserverCommand,
)
from django.core.management.commands.testserver import (
    Command as TestserverCommand,
)
from django.db import ConnectionHandler, connection
from django.db.migrations.recorder import MigrationRecorder
from django.test import (
    LiveServerTestCase, SimpleTestCase, TestCase, override_settings,
)

custom_templates_dir = os.path.join(os.path.dirname(__file__), 'custom_templates')

SYSTEM_CHECK_MSG = 'System check identified no issues'


class AdminScriptTestCase(SimpleTestCase):
    def setUp(self):
        tmpdir = tempfile.TemporaryDirectory()
        self.addCleanup(tmpdir.cleanup)
        # os.path.realpath() is required for temporary directories on macOS,
        # where `/var` is a symlink to `/private/var`.
        self.test_dir = os.path.realpath(os.path.join(tmpdir.name, 'test_project'))
        os.mkdir(self.test_dir)

    def write_settings(self, filename, apps=None, is_dir=False, sdict=None, extra=None):
        if is_dir:
            settings_dir = os.path.join(self.test_dir, filename)
            os.mkdir(settings_dir)
            settings_file_path = os.path.join(settings_dir, '__init__.py')
        else:
            settings_file_path = os.path.join(self.test_dir, filename)

        with open(settings_file_path, 'w') as settings_file:
            settings_file.write('# Settings file automatically generated by admin_scripts test case\n')
            if extra:
                settings_file.write("%s\n" % extra)
            exports = [
                'DATABASES',
                'ROOT_URLCONF',
                'SECRET_KEY',
            ]
            for s in exports:
                if hasattr(settings, s):
                    o = getattr(settings, s)
                    if not isinstance(o, (dict, tuple, list)):
                        o = "'%s'" % o
                    settings_file.write("%s = %s\n" % (s, o))

            if apps is None:
                apps = ['django.contrib.auth', 'django.contrib.contenttypes', 'admin_scripts']

            settings_file.write("INSTALLED_APPS = %s\n" % apps)

            if sdict:
                for k, v in sdict.items():
                    settings_file.write("%s = %s\n" % (k, v))

    def _ext_backend_paths(self):
        """
        Returns the paths for any external backend packages.
        """
        paths = []
        for backend in settings.DATABASES.values():
            package = backend['ENGINE'].split('.')[0]
            if package != 'django':
                backend_pkg = __import__(package)
                backend_dir = os.path.dirname(backend_pkg.__file__)
                paths.append(os.path.dirname(backend_dir))
        return paths

    def run_test(self, args, settings_file=None, apps=None):
        base_dir = os.path.dirname(self.test_dir)
        # The base dir for Django's tests is one level up.
        tests_dir = os.path.dirname(os.path.dirname(__file__))
        # The base dir for Django is one level above the test dir. We don't use
        # `import django` to figure that out, so we don't pick up a Django
        # from site-packages or similar.
        django_dir = os.path.dirname(tests_dir)
        ext_backend_base_dirs = self._ext_backend_paths()

        # Define a temporary environment for the subprocess
        test_environ = os.environ.copy()

        # Set the test environment
        if settings_file:
            test_environ['DJANGO_SETTINGS_MODULE'] = settings_file
        elif 'DJANGO_SETTINGS_MODULE' in test_environ:
            del test_environ['DJANGO_SETTINGS_MODULE']
        python_path = [base_dir, django_dir, tests_dir]
        python_path.extend(ext_backend_base_dirs)
        test_environ['PYTHONPATH'] = os.pathsep.join(python_path)
        test_environ['PYTHONWARNINGS'] = ''

        p = subprocess.run(
            [sys.executable, *args],
            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            cwd=self.test_dir,
            env=test_environ, universal_newlines=True,
        )
        return p.stdout, p.stderr

    def run_django_admin(self, args, settings_file=None):
        return self.run_test(['-m', 'django', *args], settings_file)

    def run_manage(self, args, settings_file=None, manage_py=None):
        template_manage_py = (
            os.path.join(os.path.dirname(__file__), manage_py)
            if manage_py else
            os.path.join(os.path.dirname(conf.__file__), 'project_template', 'manage.py-tpl')
        )
        test_manage_py = os.path.join(self.test_dir, 'manage.py')
        shutil.copyfile(template_manage_py, test_manage_py)

        with open(test_manage_py) as fp:
            manage_py_contents = fp.read()
        manage_py_contents = manage_py_contents.replace(
            "{{ project_name }}", "test_project")
        with open(test_manage_py, 'w') as fp:
            fp.write(manage_py_contents)

        return self.run_test(['./manage.py', *args], settings_file)

    def assertNoOutput(self, stream):
        "Utility assertion: assert that the given stream is empty"
        self.assertEqual(len(stream), 0, "Stream should be empty: actually contains '%s'" % stream)

    def assertOutput(self, stream, msg, regex=False):
        "Utility assertion: assert that the given message exists in the output"
        if regex:
            self.assertIsNotNone(
                re.search(msg, stream),
                "'%s' does not match actual output text '%s'" % (msg, stream)
            )
        else:
            self.assertIn(msg, stream, "'%s' does not match actual output text '%s'" % (msg, stream))

    def assertNotInOutput(self, stream, msg):
        "Utility assertion: assert that the given message doesn't exist in the output"
        self.assertNotIn(msg, stream, "'%s' matches actual output text '%s'" % (msg, stream))

##########################################################################
# DJANGO ADMIN TESTS
# This first series of test classes checks the environment processing
# of the django-admin.py script
##########################################################################


class DjangoAdminNoSettings(AdminScriptTestCase):
    "A series of tests for django-admin.py when there is no settings.py file."

    def test_builtin_command(self):
        "no settings: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_bad_settings(self):
        "no settings: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "no settings: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_commands_with_invalid_settings(self):
        """"
        Commands that don't require settings succeed if the settings file
        doesn't exist.
        """
        args = ['startproject']
        out, err = self.run_django_admin(args, settings_file='bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "You must provide a project name", regex=True)


class DjangoAdminDefaultSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    contains the test application.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py')

    def test_builtin_command(self):
        "default: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "default: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "default: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "default: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "default: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "default: django-admin can't execute user commands if it isn't provided settings"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "default: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "default: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminFullPathDefaultSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    contains the test application specified using a full path.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py', ['django.contrib.auth', 'django.contrib.contenttypes',
                                            'admin_scripts', 'admin_scripts.complex_app'])

    def test_builtin_command(self):
        "fulldefault: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "fulldefault: django-admin builtin commands succeed if a settings file is provided"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "fulldefault: django-admin builtin commands succeed if the environment contains settings"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "fulldefault: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "fulldefault: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "fulldefault: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "fulldefault: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "fulldefault: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminMinimalSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    doesn't contain the test application.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])

    def test_builtin_command(self):
        "minimal: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "minimal: django-admin builtin commands fail if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_environment(self):
        "minimal: django-admin builtin commands fail if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_bad_settings(self):
        "minimal: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "minimal: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "minimal: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "minimal: django-admin can't execute user commands, even if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_environment(self):
        "minimal: django-admin can't execute user commands, even if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")


class DjangoAdminAlternateSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings file
    with a name other than 'settings.py'.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "alternate: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.alternate_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "alternate: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "alternate: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "alternate: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "alternate: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "alternate: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.alternate_settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "alternate: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminMultipleSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when multiple settings files
    (including the default 'settings.py') are available. The default settings
    file is insufficient for performing the operations described, so the
    alternate settings must be used by the running script.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "alternate: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.alternate_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "alternate: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "alternate: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "alternate: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "alternate: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "alternate: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.alternate_settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "alternate: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminSettingsDirectory(AdminScriptTestCase):
    """
    A series of tests for django-admin.py when the settings file is in a
    directory. (see #9751).
    """

    def setUp(self):
        super().setUp()
        self.write_settings('settings', is_dir=True)

    def test_setup_environ(self):
        "directory: startapp creates the correct directory"
        args = ['startapp', 'settings_test']
        app_path = os.path.join(self.test_dir, 'settings_test')
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(app_path))
        with open(os.path.join(app_path, 'apps.py')) as f:
            content = f.read()
            self.assertIn("class SettingsTestConfig(AppConfig)", content)
            self.assertIn("name = 'settings_test'", content)

    def test_setup_environ_custom_template(self):
        "directory: startapp creates the correct directory with a custom template"
        template_path = os.path.join(custom_templates_dir, 'app_template')
        args = ['startapp', '--template', template_path, 'custom_settings_test']
        app_path = os.path.join(self.test_dir, 'custom_settings_test')
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(app_path))
        self.assertTrue(os.path.exists(os.path.join(app_path, 'api.py')))

    def test_startapp_unicode_name(self):
        """startapp creates the correct directory with Unicode characters."""
        args = ['startapp', 'こんにちは']
        app_path = os.path.join(self.test_dir, 'こんにちは')
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(app_path))
        with open(os.path.join(app_path, 'apps.py'), encoding='utf8') as f:
            content = f.read()
            self.assertIn("class こんにちはConfig(AppConfig)", content)
            self.assertIn("name = 'こんにちは'", content)

    def test_builtin_command(self):
        "directory: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_bad_settings(self):
        "directory: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "directory: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "directory: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_builtin_with_settings(self):
        "directory: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "directory: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)


##########################################################################
# MANAGE.PY TESTS
# This next series of test classes checks the environment processing
# of the generated manage.py script
##########################################################################

class ManageManuallyConfiguredSettings(AdminScriptTestCase):
    """Customized manage.py calling settings.configure()."""
    def test_non_existent_command_output(self):
        out, err = self.run_manage(['invalid_command'], manage_py='configured_settings_manage.py')
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'invalid_command'")
        self.assertNotInOutput(err, 'No Django settings specified')


class ManageNoSettings(AdminScriptTestCase):
    "A series of tests for manage.py when there is no settings.py file."

    def test_builtin_command(self):
        "no settings: manage.py builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, r"No module named '?(test_project\.)?settings'?", regex=True)

    def test_builtin_with_bad_settings(self):
        "no settings: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "no settings: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)


class ManageDefaultSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    contains the test application.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py')

    def test_builtin_command(self):
        "default: manage.py builtin commands succeed when default settings are appropriate"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_settings(self):
        "default: manage.py builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "default: manage.py builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "default: manage.py builtin commands succeed if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "default: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "default: manage.py can execute user commands when default settings are appropriate"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_settings(self):
        "default: manage.py can execute user commands when settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "default: manage.py can execute user commands when settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class ManageFullPathDefaultSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    contains the test application specified using a full path.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py', ['django.contrib.auth', 'django.contrib.contenttypes', 'admin_scripts'])

    def test_builtin_command(self):
        "fulldefault: manage.py builtin commands succeed when default settings are appropriate"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_settings(self):
        "fulldefault: manage.py builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "fulldefault: manage.py builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "fulldefault: manage.py builtin commands succeed if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "fulldefault: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "fulldefault: manage.py can execute user commands when default settings are appropriate"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_settings(self):
        "fulldefault: manage.py can execute user commands when settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "fulldefault: manage.py can execute user commands when settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class ManageMinimalSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    doesn't contain the test application.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])

    def test_builtin_command(self):
        "minimal: manage.py builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_settings(self):
        "minimal: manage.py builtin commands fail if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_environment(self):
        "minimal: manage.py builtin commands fail if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_bad_settings(self):
        "minimal: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "minimal: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "minimal: manage.py can't execute user commands without appropriate settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "minimal: manage.py can't execute user commands, even if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_environment(self):
        "minimal: manage.py can't execute user commands, even if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")


class ManageAlternateSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings file
    with a name other than 'settings.py'.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: manage.py builtin commands fail with an error when no default settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, r"No module named '?(test_project\.)?settings'?", regex=True)

    def test_builtin_with_settings(self):
        "alternate: manage.py builtin commands work with settings provided as argument"
        args = ['check', '--settings=alternate_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertOutput(out, SYSTEM_CHECK_MSG)
        self.assertNoOutput(err)

    def test_builtin_with_environment(self):
        "alternate: manage.py builtin commands work if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertOutput(out, SYSTEM_CHECK_MSG)
        self.assertNoOutput(err)

    def test_builtin_with_bad_settings(self):
        "alternate: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "alternate: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "alternate: manage.py can't execute user commands without settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, r"No module named '?(test_project\.)?settings'?", regex=True)

    def test_custom_command_with_settings(self):
        "alternate: manage.py can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('force_color', False), "
            "('no_color', False), ('pythonpath', None), ('settings', "
            "'alternate_settings'), ('traceback', False), ('verbosity', 1)]"
        )
        self.assertNoOutput(err)

    def test_custom_command_with_environment(self):
        "alternate: manage.py can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('force_color', False), "
            "('no_color', False), ('pythonpath', None), ('settings', None), "
            "('traceback', False), ('verbosity', 1)]"
        )
        self.assertNoOutput(err)

    def test_custom_command_output_color(self):
        "alternate: manage.py output syntax color can be deactivated with the `--no-color` option"
        args = ['noargs_command', '--no-color', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('force_color', False), "
            "('no_color', True), ('pythonpath', None), ('settings', "
            "'alternate_settings'), ('traceback', False), ('verbosity', 1)]"
        )
        self.assertNoOutput(err)


class ManageMultipleSettings(AdminScriptTestCase):
    """A series of tests for manage.py when multiple settings files
    (including the default 'settings.py') are available. The default settings
    file is insufficient for performing the operations described, so the
    alternate settings must be used by the running script.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def test_builtin_command(self):
        "multiple: manage.py builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_settings(self):
        "multiple: manage.py builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=alternate_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "multiple: manage.py can execute builtin commands if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "multiple: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "multiple: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "multiple: manage.py can't execute user commands using default settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "multiple: manage.py can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "multiple: manage.py can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class ManageSettingsWithSettingsErrors(AdminScriptTestCase):
    """
    Tests for manage.py when using the default settings.py file containing
    runtime errors.
    """
    def write_settings_with_import_error(self, filename):
        settings_file_path = os.path.join(self.test_dir, filename)
        with open(settings_file_path, 'w') as settings_file:
            settings_file.write('# Settings file automatically generated by admin_scripts test case\n')
            settings_file.write('# The next line will cause an import error:\nimport foo42bar\n')

    def test_import_error(self):
        """
        import error: manage.py builtin commands shows useful diagnostic info
        when settings with import errors is provided (#14130).
        """
        self.write_settings_with_import_error('settings.py')
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named")
        self.assertOutput(err, "foo42bar")

    def test_attribute_error(self):
        """
        manage.py builtin commands does not swallow attribute error due to bad
        settings (#18845).
        """
        self.write_settings('settings.py', sdict={'BAD_VAR': 'INSTALLED_APPS.crash'})
        args = ['collectstatic', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "AttributeError: 'list' object has no attribute 'crash'")

    def test_key_error(self):
        self.write_settings('settings.py', sdict={'BAD_VAR': 'DATABASES["blah"]'})
        args = ['collectstatic', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "KeyError: 'blah'")

    def test_help(self):
        """
        Test listing available commands output note when only core commands are
        available.
        """
        self.write_settings(
            'settings.py',
            extra='from django.core.exceptions import ImproperlyConfigured\n'
                  'raise ImproperlyConfigured()',
        )
        args = ['help']
        out, err = self.run_manage(args)
        self.assertOutput(out, 'only Django core commands are listed')
        self.assertNoOutput(err)


class ManageCheck(AdminScriptTestCase):
    def test_nonexistent_app(self):
        """check reports an error on a nonexistent app in INSTALLED_APPS."""
        self.write_settings(
            'settings.py',
            apps=['admin_scriptz.broken_app'],
            sdict={'USE_I18N': False},
        )
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'ModuleNotFoundError')
        self.assertOutput(err, 'No module named')
        self.assertOutput(err, 'admin_scriptz')

    def test_broken_app(self):
        """ manage.py check reports an ImportError if an app's models.py
        raises one on import """

        self.write_settings('settings.py', apps=['admin_scripts.broken_app'])
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'ImportError')

    def test_complex_app(self):
        """ manage.py check does not raise an ImportError validating a
        complex app with nested calls to load_app """

        self.write_settings(
            'settings.py',
            apps=[
                'admin_scripts.complex_app',
                'admin_scripts.simple_app',
                'django.contrib.admin.apps.SimpleAdminConfig',
                'django.contrib.auth',
                'django.contrib.contenttypes',
                'django.contrib.messages',
            ],
            sdict={
                'DEBUG': True,
                'MIDDLEWARE': [
                    'django.contrib.messages.middleware.MessageMiddleware',
                    'django.contrib.auth.middleware.AuthenticationMiddleware',
                    'django.contrib.sessions.middleware.SessionMiddleware',
                ],
                'TEMPLATES': [
                    {
                        'BACKEND': 'django.template.backends.django.DjangoTemplates',
                        'DIRS': [],
                        'APP_DIRS': True,
                        'OPTIONS': {
                            'context_processors': [
                                'django.contrib.auth.context_processors.auth',
                                'django.contrib.messages.context_processors.messages',
                            ],
                        },
                    },
                ],
            }
        )
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertEqual(out, 'System check identified no issues (0 silenced).\n')

    def test_app_with_import(self):
        """ manage.py check does not raise errors when an app imports a base
        class that itself has an abstract base. """

        self.write_settings(
            'settings.py',
            apps=[
                'admin_scripts.app_with_import',
                'django.contrib.auth',
                'django.contrib.contenttypes',
                'django.contrib.sites',
            ],
            sdict={'DEBUG': True},
        )
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertEqual(out, 'System check identified no issues (0 silenced).\n')

    def test_output_format(self):
        """ All errors/warnings should be sorted by level and by message. """

        self.write_settings(
            'settings.py',
            apps=[
                'admin_scripts.app_raising_messages',
                'django.contrib.auth',
                'django.contrib.contenttypes',
            ],
            sdict={'DEBUG': True},
        )
        args = ['check']
        out, err = self.run_manage(args)
        expected_err = (
            "SystemCheckError: System check identified some issues:\n"
            "\n"
            "ERRORS:\n"
            "?: An error\n"
            "\tHINT: Error hint\n"
            "\n"
            "WARNINGS:\n"
            "a: Second warning\n"
            "obj: First warning\n"
            "\tHINT: Hint\n"
            "\n"
            "System check identified 3 issues (0 silenced).\n"
        )
        self.assertEqual(err, expected_err)
        self.assertNoOutput(out)

    def test_warning_does_not_halt(self):
        """
        When there are only warnings or less serious messages, then Django
        should not prevent user from launching their project, so `check`
        command should not raise `CommandError` exception.

        In this test we also test output format.
        """

        self.write_settings(
            'settings.py',
            apps=[
                'admin_scripts.app_raising_warning',
                'django.contrib.auth',
                'django.contrib.contenttypes',
            ],
            sdict={'DEBUG': True},
        )
        args = ['check']
        out, err = self.run_manage(args)
        expected_err = (
            "System check identified some issues:\n"  # No "CommandError: " part
            "\n"
            "WARNINGS:\n"
            "?: A warning\n"
            "\n"
            "System check identified 1 issue (0 silenced).\n"
        )
        self.assertEqual(err, expected_err)
        self.assertNoOutput(out)


class ManageRunserver(SimpleTestCase):
    def setUp(self):
        def monkey_run(*args, **options):
            return

        self.output = StringIO()
        self.cmd = RunserverCommand(stdout=self.output)
        self.cmd.run = monkey_run

    def assertServerSettings(self, addr, port, ipv6=False, raw_ipv6=False):
        self.assertEqual(self.cmd.addr, addr)
        self.assertEqual(self.cmd.port, port)
        self.assertEqual(self.cmd.use_ipv6, ipv6)
        self.assertEqual(self.cmd._raw_ipv6, raw_ipv6)

    def test_runserver_addrport(self):
        call_command(self.cmd)
        self.assertServerSettings('127.0.0.1', '8000')

        call_command(self.cmd, addrport="1.2.3.4:8000")
        self.assertServerSettings('1.2.3.4', '8000')

        call_command(self.cmd, addrport="7000")
        self.assertServerSettings('127.0.0.1', '7000')

    @unittest.skipUnless(socket.has_ipv6, "platform doesn't support IPv6")
    def test_runner_addrport_ipv6(self):
        call_command(self.cmd, addrport="", use_ipv6=True)
        self.assertServerSettings('::1', '8000', ipv6=True, raw_ipv6=True)

        call_command(self.cmd, addrport="7000", use_ipv6=True)
        self.assertServerSettings('::1', '7000', ipv6=True, raw_ipv6=True)

        call_command(self.cmd, addrport="[2001:0db8:1234:5678::9]:7000")
        self.assertServerSettings('2001:0db8:1234:5678::9', '7000', ipv6=True, raw_ipv6=True)

    def test_runner_hostname(self):
        call_command(self.cmd, addrport="localhost:8000")
        self.assertServerSettings('localhost', '8000')

        call_command(self.cmd, addrport="test.domain.local:7000")
        self.assertServerSettings('test.domain.local', '7000')

    @unittest.skipUnless(socket.has_ipv6, "platform doesn't support IPv6")
    def test_runner_hostname_ipv6(self):
        call_command(self.cmd, addrport="test.domain.local:7000", use_ipv6=True)
        self.assertServerSettings('test.domain.local', '7000', ipv6=True)

    def test_runner_custom_defaults(self):
        self.cmd.default_addr = '0.0.0.0'
        self.cmd.default_port = '5000'
        call_command(self.cmd)
        self.assertServerSettings('0.0.0.0', '5000')

    @unittest.skipUnless(socket.has_ipv6, "platform doesn't support IPv6")
    def test_runner_custom_defaults_ipv6(self):
        self.cmd.default_addr_ipv6 = '::'
        call_command(self.cmd, use_ipv6=True)
        self.assertServerSettings('::', '8000', ipv6=True, raw_ipv6=True)

    def test_runner_ambiguous(self):
        # Only 4 characters, all of which could be in an ipv6 address
        call_command(self.cmd, addrport="beef:7654")
        self.assertServerSettings('beef', '7654')

        # Uses only characters that could be in an ipv6 address
        call_command(self.cmd, addrport="deadbeef:7654")
        self.assertServerSettings('deadbeef', '7654')

    def test_no_database(self):
        """
        Ensure runserver.check_migrations doesn't choke on empty DATABASES.
        """
        tested_connections = ConnectionHandler({})
        with mock.patch('django.core.management.base.connections', new=tested_connections):
            self.cmd.check_migrations()

    def test_readonly_database(self):
        """
        runserver.check_migrations() doesn't choke when a database is read-only.
        """
        with mock.patch.object(MigrationRecorder, 'has_table', return_value=False):
            self.cmd.check_migrations()
        # You have # ...
        self.assertIn('unapplied migration(s)', self.output.getvalue())


class ManageRunserverMigrationWarning(TestCase):

    def setUp(self):
        self.stdout = StringIO()
        self.runserver_command = RunserverCommand(stdout=self.stdout)

    @override_settings(INSTALLED_APPS=["admin_scripts.app_waiting_migration"])
    def test_migration_warning_one_app(self):
        self.runserver_command.check_migrations()
        output = self.stdout.getvalue()
        self.assertIn('You have 1 unapplied migration(s)', output)
        self.assertIn('apply the migrations for app(s): app_waiting_migration.', output)

    @override_settings(
        INSTALLED_APPS=[
            "admin_scripts.app_waiting_migration",
            "admin_scripts.another_app_waiting_migration",
        ],
    )
    def test_migration_warning_multiple_apps(self):
        self.runserver_command.check_migrations()
        output = self.stdout.getvalue()
        self.assertIn('You have 2 unapplied migration(s)', output)
        self.assertIn(
            'apply the migrations for app(s): another_app_waiting_migration, '
            'app_waiting_migration.', output
        )


class ManageRunserverEmptyAllowedHosts(AdminScriptTestCase):
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py', sdict={
            'ALLOWED_HOSTS': [],
            'DEBUG': False,
        })

    def test_empty_allowed_hosts_error(self):
        out, err = self.run_manage(['runserver'])
        self.assertNoOutput(out)
        self.assertOutput(err, 'CommandError: You must set settings.ALLOWED_HOSTS if DEBUG is False.')


class ManageTestserver(SimpleTestCase):

    @mock.patch.object(TestserverCommand, 'handle', return_value='')
    def test_testserver_handle_params(self, mock_handle):
        out = StringIO()
        call_command('testserver', 'blah.json', stdout=out)
        mock_handle.assert_called_with(
            'blah.json',
            stdout=out, settings=None, pythonpath=None, verbosity=1,
            traceback=False, addrport='', no_color=False, use_ipv6=False,
            skip_checks=True, interactive=True, force_color=False,
        )

    @mock.patch('django.db.connection.creation.create_test_db', return_value='test_db')
    @mock.patch.object(LoaddataCommand, 'handle', return_value='')
    @mock.patch.object(RunserverCommand, 'handle', return_value='')
    def test_params_to_runserver(self, mock_runserver_handle, mock_loaddata_handle, mock_create_test_db):
        out = StringIO()
        call_command('testserver', 'blah.json', stdout=out)
        mock_runserver_handle.assert_called_with(
            addrport='',
            force_color=False,
            insecure_serving=False,
            no_color=False,
            pythonpath=None,
            settings=None,
            shutdown_message=(
                "\nServer stopped.\nNote that the test database, 'test_db', "
                "has not been deleted. You can explore it on your own."
            ),
            skip_checks=True,
            traceback=False,
            use_ipv6=False,
            use_reloader=False,
            use_static_handler=True,
            use_threading=connection.features.test_db_allows_multiple_connections,
            verbosity=1,
        )


##########################################################################
# COMMAND PROCESSING TESTS
# user-space commands are correctly handled - in particular, arguments to
# the commands are correctly parsed and processed.
##########################################################################
class ColorCommand(BaseCommand):
    requires_system_checks = False

    def handle(self, *args, **options):
        self.stdout.write('Hello, world!', self.style.ERROR)
        self.stderr.write('Hello, world!', self.style.ERROR)


class CommandTypes(AdminScriptTestCase):
    "Tests for the various types of base command types that can be defined."
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py')

    def test_version(self):
        "version is handled as a special case"
        args = ['version']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, get_version())

    def test_version_alternative(self):
        "--version is equivalent to version"
        args1, args2 = ['version'], ['--version']
        # It's possible one outputs on stderr and the other on stdout, hence the set
        self.assertEqual(set(self.run_manage(args1)), set(self.run_manage(args2)))

    def test_help(self):
        "help is handled as a special case"
        args = ['help']
        out, err = self.run_manage(args)
        self.assertOutput(out, "Type 'manage.py help <subcommand>' for help on a specific subcommand.")
        self.assertOutput(out, '[django]')
        self.assertOutput(out, 'startapp')
        self.assertOutput(out, 'startproject')

    def test_help_commands(self):
        "help --commands shows the list of all available commands"
        args = ['help', '--commands']
        out, err = self.run_manage(args)
        self.assertNotInOutput(out, 'usage:')
        self.assertNotInOutput(out, 'Options:')
        self.assertNotInOutput(out, '[django]')
        self.assertOutput(out, 'startapp')
        self.assertOutput(out, 'startproject')
        self.assertNotInOutput(out, '\n\n')

    def test_help_alternative(self):
        "--help is equivalent to help"
        args1, args2 = ['help'], ['--help']
        self.assertEqual(self.run_manage(args1), self.run_manage(args2))

    def test_help_short_altert(self):
        "-h is handled as a short form of --help"
        args1, args2 = ['--help'], ['-h']
        self.assertEqual(self.run_manage(args1), self.run_manage(args2))

    def test_specific_help(self):
        "--help can be used on a specific command"
        args = ['check', '--help']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        # Command-specific options like --tag appear before options common to
        # all commands like --version.
        tag_location = out.find('--tag')
        version_location = out.find('--version')
        self.assertNotEqual(tag_location, -1)
        self.assertNotEqual(version_location, -1)
        self.assertLess(tag_location, version_location)
        self.assertOutput(out, "Checks the entire Django project for potential problems.")

    def test_color_style(self):
        style = color.no_style()
        self.assertEqual(style.ERROR('Hello, world!'), 'Hello, world!')

        style = color.make_style('nocolor')
        self.assertEqual(style.ERROR('Hello, world!'), 'Hello, world!')

        style = color.make_style('dark')
        self.assertIn('Hello, world!', style.ERROR('Hello, world!'))
        self.assertNotEqual(style.ERROR('Hello, world!'), 'Hello, world!')

        # Default palette has color.
        style = color.make_style('')
        self.assertIn('Hello, world!', style.ERROR('Hello, world!'))
        self.assertNotEqual(style.ERROR('Hello, world!'), 'Hello, world!')

    def test_command_color(self):
        out = StringIO()
        err = StringIO()
        command = ColorCommand(stdout=out, stderr=err)
        call_command(command)
        if color.supports_color():
            self.assertIn('Hello, world!\n', out.getvalue())
            self.assertIn('Hello, world!\n', err.getvalue())
            self.assertNotEqual(out.getvalue(), 'Hello, world!\n')
            self.assertNotEqual(err.getvalue(), 'Hello, world!\n')
        else:
            self.assertEqual(out.getvalue(), 'Hello, world!\n')
            self.assertEqual(err.getvalue(), 'Hello, world!\n')

    def test_command_no_color(self):
        "--no-color prevent colorization of the output"
        out = StringIO()
        err = StringIO()
        command = ColorCommand(stdout=out, stderr=err, no_color=True)
        call_command(command)
        self.assertEqual(out.getvalue(), 'Hello, world!\n')
        self.assertEqual(err.getvalue(), 'Hello, world!\n')

        out = StringIO()
        err = StringIO()
        command = ColorCommand(stdout=out, stderr=err)
        call_command(command, no_color=True)
        self.assertEqual(out.getvalue(), 'Hello, world!\n')
        self.assertEqual(err.getvalue(), 'Hello, world!\n')

    def test_force_color_execute(self):
        out = StringIO()
        err = StringIO()
        with mock.patch.object(sys.stdout, 'isatty', lambda: False):
            command = ColorCommand(stdout=out, stderr=err)
            call_command(command, force_color=True)
        self.assertEqual(out.getvalue(), '\x1b[31;1mHello, world!\n\x1b[0m')
        self.assertEqual(err.getvalue(), '\x1b[31;1mHello, world!\n\x1b[0m')

    def test_force_color_command_init(self):
        out = StringIO()
        err = StringIO()
        with mock.patch.object(sys.stdout, 'isatty', lambda: False):
            command = ColorCommand(stdout=out, stderr=err, force_color=True)
            call_command(command)
        self.assertEqual(out.getvalue(), '\x1b[31;1mHello, world!\n\x1b[0m')
        self.assertEqual(err.getvalue(), '\x1b[31;1mHello, world!\n\x1b[0m')

    def test_no_color_force_color_mutually_exclusive_execute(self):
        msg = "The --no-color and --force-color options can't be used together."
        with self.assertRaisesMessage(CommandError, msg):
            call_command(BaseCommand(), no_color=True, force_color=True)

    def test_no_color_force_color_mutually_exclusive_command_init(self):
        msg = "'no_color' and 'force_color' can't be used together."
        with self.assertRaisesMessage(CommandError, msg):
            call_command(BaseCommand(no_color=True, force_color=True))

    def test_custom_stdout(self):
        class Command(BaseCommand):
            requires_system_checks = False

            def handle(self, *args, **options):
                self.stdout.write("Hello, World!")

        out = StringIO()
        command = Command(stdout=out)
        call_command(command)
        self.assertEqual(out.getvalue(), "Hello, World!\n")
        out.truncate(0)
        new_out = StringIO()
        call_command(command, stdout=new_out)
        self.assertEqual(out.getvalue(), "")
        self.assertEqual(new_out.getvalue(), "Hello, World!\n")

    def test_custom_stderr(self):
        class Command(BaseCommand):
            requires_system_checks = False

            def handle(self, *args, **options):
                self.stderr.write("Hello, World!")

        err = StringIO()
        command = Command(stderr=err)
        call_command(command)
        self.assertEqual(err.getvalue(), "Hello, World!\n")
        err.truncate(0)
        new_err = StringIO()
        call_command(command, stderr=new_err)
        self.assertEqual(err.getvalue(), "")
        self.assertEqual(new_err.getvalue(), "Hello, World!\n")

    def test_base_command(self):
        "User BaseCommands can execute when a label is provided"
        args = ['base_command', 'testlabel']
        expected_labels = "('testlabel',)"
        self._test_base_command(args, expected_labels)

    def test_base_command_no_label(self):
        "User BaseCommands can execute when no labels are provided"
        args = ['base_command']
        expected_labels = "()"
        self._test_base_command(args, expected_labels)

    def test_base_command_multiple_label(self):
        "User BaseCommands can execute when no labels are provided"
        args = ['base_command', 'testlabel', 'anotherlabel']
        expected_labels = "('testlabel', 'anotherlabel')"
        self._test_base_command(args, expected_labels)

    def test_base_command_with_option(self):
        "User BaseCommands can execute with options when a label is provided"
        args = ['base_command', 'testlabel', '--option_a=x']
        expected_labels = "('testlabel',)"
        self._test_base_command(args, expected_labels, option_a="'x'")

    def test_base_command_with_options(self):
        "User BaseCommands can execute with multiple options when a label is provided"
        args = ['base_command', 'testlabel', '-a', 'x', '--option_b=y']
        expected_labels = "('testlabel',)"
        self._test_base_command(args, expected_labels, option_a="'x'", option_b="'y'")

    def test_base_command_with_wrong_option(self):
        "User BaseCommands outputs command usage when wrong option is specified"
        args = ['base_command', '--invalid']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "usage: manage.py base_command")
        self.assertOutput(err, "error: unrecognized arguments: --invalid")

    def _test_base_command(self, args, labels, option_a="'1'", option_b="'2'"):
        out, err = self.run_manage(args)

        expected_out = (
            "EXECUTE:BaseCommand labels=%s, "
            "options=[('force_color', False), ('no_color', False), "
            "('option_a', %s), ('option_b', %s), ('option_c', '3'), "
            "('pythonpath', None), ('settings', None), ('traceback', False), "
            "('verbosity', 1)]") % (labels, option_a, option_b)
        self.assertNoOutput(err)
        self.assertOutput(out, expected_out)

    def test_base_run_from_argv(self):
        """
        Test run_from_argv properly terminates even with custom execute() (#19665)
        Also test proper traceback display.
        """
        err = StringIO()
        command = BaseCommand(stderr=err)

        def raise_command_error(*args, **kwargs):
            raise CommandError("Custom error")

        command.execute = lambda args: args  # This will trigger TypeError

        # If the Exception is not CommandError it should always
        # raise the original exception.
        with self.assertRaises(TypeError):
            command.run_from_argv(['', ''])

        # If the Exception is CommandError and --traceback is not present
        # this command should raise a SystemExit and don't print any
        # traceback to the stderr.
        command.execute = raise_command_error
        err.truncate(0)
        with self.assertRaises(SystemExit):
            command.run_from_argv(['', ''])
        err_message = err.getvalue()
        self.assertNotIn("Traceback", err_message)
        self.assertIn("CommandError", err_message)

        # If the Exception is CommandError and --traceback is present
        # this command should raise the original CommandError as if it
        # were not a CommandError.
        err.truncate(0)
        with self.assertRaises(CommandError):
            command.run_from_argv(['', '', '--traceback'])

    def test_run_from_argv_non_ascii_error(self):
        """
        Non-ASCII message of CommandError does not raise any
        UnicodeDecodeError in run_from_argv.
        """
        def raise_command_error(*args, **kwargs):
            raise CommandError("Erreur personnalisée")

        command = BaseCommand(stderr=StringIO())
        command.execute = raise_command_error

        with self.assertRaises(SystemExit):
            command.run_from_argv(['', ''])

    def test_run_from_argv_closes_connections(self):
        """
        A command called from the command line should close connections after
        being executed (#21255).
        """
        command = BaseCommand(stderr=StringIO())
        command.check = lambda: []
        command.handle = lambda *args, **kwargs: args
        with mock.patch('django.core.management.base.connections') as mock_connections:
            command.run_from_argv(['', ''])
        # Test connections have been closed
        self.assertTrue(mock_connections.close_all.called)

    def test_noargs(self):
        "NoArg Commands can be executed"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('force_color', False), "
            "('no_color', False), ('pythonpath', None), ('settings', None), "
            "('traceback', False), ('verbosity', 1)]"
        )

    def test_noargs_with_args(self):
        "NoArg Commands raise an error if an argument is provided"
        args = ['noargs_command', 'argument']
        out, err = self.run_manage(args)
        self.assertOutput(err, "error: unrecognized arguments: argument")

    def test_app_command(self):
        "User AppCommands can execute when a single app name is provided"
        args = ['app_command', 'auth']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:AppCommand name=django.contrib.auth, options=")
        self.assertOutput(
            out,
            ", options=[('force_color', False), ('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), "
            "('verbosity', 1)]"
        )

    def test_app_command_no_apps(self):
        "User AppCommands raise an error when no app name is provided"
        args = ['app_command']
        out, err = self.run_manage(args)
        self.assertOutput(err, 'error: Enter at least one application label.')

    def test_app_command_multiple_apps(self):
        "User AppCommands raise an error when multiple app names are provided"
        args = ['app_command', 'auth', 'contenttypes']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:AppCommand name=django.contrib.auth, options=")
        self.assertOutput(
            out,
            ", options=[('force_color', False), ('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), "
            "('verbosity', 1)]"
        )
        self.assertOutput(out, "EXECUTE:AppCommand name=django.contrib.contenttypes, options=")
        self.assertOutput(
            out,
            ", options=[('force_color', False), ('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), "
            "('verbosity', 1)]"
        )

    def test_app_command_invalid_app_label(self):
        "User AppCommands can execute when a single app name is provided"
        args = ['app_command', 'NOT_AN_APP']
        out, err = self.run_manage(args)
        self.assertOutput(err, "No installed app with label 'NOT_AN_APP'.")

    def test_app_command_some_invalid_app_labels(self):
        "User AppCommands can execute when some of the provided app names are invalid"
        args = ['app_command', 'auth', 'NOT_AN_APP']
        out, err = self.run_manage(args)
        self.assertOutput(err, "No installed app with label 'NOT_AN_APP'.")

    def test_label_command(self):
        "User LabelCommands can execute when a label is provided"
        args = ['label_command', 'testlabel']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE:LabelCommand label=testlabel, options=[('force_color', "
            "False), ('no_color', False), ('pythonpath', None), ('settings', "
            "None), ('traceback', False), ('verbosity', 1)]"
        )

    def test_label_command_no_label(self):
        "User LabelCommands raise an error if no label is provided"
        args = ['label_command']
        out, err = self.run_manage(args)
        self.assertOutput(err, 'Enter at least one label')

    def test_label_command_multiple_label(self):
        "User LabelCommands are executed multiple times if multiple labels are provided"
        args = ['label_command', 'testlabel', 'anotherlabel']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE:LabelCommand label=testlabel, options=[('force_color', "
            "False), ('no_color', False), ('pythonpath', None), "
            "('settings', None), ('traceback', False), ('verbosity', 1)]"
        )
        self.assertOutput(
            out,
            "EXECUTE:LabelCommand label=anotherlabel, options=[('force_color', "
            "False), ('no_color', False), ('pythonpath', None), "
            "('settings', None), ('traceback', False), ('verbosity', 1)]"
        )


class Discovery(SimpleTestCase):

    def test_precedence(self):
        """
        Apps listed first in INSTALLED_APPS have precedence.
        """
        with self.settings(INSTALLED_APPS=['admin_scripts.complex_app',
                                           'admin_scripts.simple_app',
                                           'django.contrib.auth',
                                           'django.contrib.contenttypes']):
            out = StringIO()
            call_command('duplicate', stdout=out)
            self.assertEqual(out.getvalue().strip(), 'complex_app')
        with self.settings(INSTALLED_APPS=['admin_scripts.simple_app',
                                           'admin_scripts.complex_app',
                                           'django.contrib.auth',
                                           'django.contrib.contenttypes']):
            out = StringIO()
            call_command('duplicate', stdout=out)
            self.assertEqual(out.getvalue().strip(), 'simple_app')


class ArgumentOrder(AdminScriptTestCase):
    """Tests for 2-stage argument parsing scheme.

    django-admin command arguments are parsed in 2 parts; the core arguments
    (--settings, --traceback and --pythonpath) are parsed using a basic parser,
    ignoring any unknown options. Then the full settings are
    passed to the command parser, which extracts commands of interest to the
    individual command.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def test_setting_then_option(self):
        """ Options passed after settings are correctly handled. """
        args = ['base_command', 'testlabel', '--settings=alternate_settings', '--option_a=x']
        self._test(args)

    def test_setting_then_short_option(self):
        """ Short options passed after settings are correctly handled. """
        args = ['base_command', 'testlabel', '--settings=alternate_settings', '-a', 'x']
        self._test(args)

    def test_option_then_setting(self):
        """ Options passed before settings are correctly handled. """
        args = ['base_command', 'testlabel', '--option_a=x', '--settings=alternate_settings']
        self._test(args)

    def test_short_option_then_setting(self):
        """ Short options passed before settings are correctly handled. """
        args = ['base_command', 'testlabel', '-a', 'x', '--settings=alternate_settings']
        self._test(args)

    def test_option_then_setting_then_option(self):
        """ Options are correctly handled when they are passed before and after
        a setting. """
        args = ['base_command', 'testlabel', '--option_a=x', '--settings=alternate_settings', '--option_b=y']
        self._test(args, option_b="'y'")

    def _test(self, args, option_b="'2'"):
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE:BaseCommand labels=('testlabel',), options=["
            "('force_color', False), ('no_color', False), ('option_a', 'x'), "
            "('option_b', %s), ('option_c', '3'), ('pythonpath', None), "
            "('settings', 'alternate_settings'), ('traceback', False), "
            "('verbosity', 1)]" % option_b
        )


@override_settings(ROOT_URLCONF='admin_scripts.urls')
class StartProject(LiveServerTestCase, AdminScriptTestCase):

    available_apps = [
        'admin_scripts',
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
    ]

    def test_wrong_args(self):
        "Make sure passing the wrong kinds of arguments outputs an error and prints usage"
        out, err = self.run_django_admin(['startproject'])
        self.assertNoOutput(out)
        self.assertOutput(err, "usage:")
        self.assertOutput(err, "You must provide a project name.")

    def test_simple_project(self):
        "Make sure the startproject management command creates a project"
        args = ['startproject', 'testproject']
        testproject_dir = os.path.join(self.test_dir, 'testproject')

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))

        # running again..
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(
            err,
            "CommandError: 'testproject' conflicts with the name of an "
            "existing Python module and cannot be used as a project name. "
            "Please try another name.",
        )

    def test_invalid_project_name(self):
        "Make sure the startproject management command validates a project name"
        for bad_name in ('7testproject', '../testproject'):
            with self.subTest(project_name=bad_name):
                args = ['startproject', bad_name]
                testproject_dir = os.path.join(self.test_dir, bad_name)

                out, err = self.run_django_admin(args)
                self.assertOutput(
                    err,
                    "Error: '%s' is not a valid project name. Please make "
                    "sure the name is a valid identifier." % bad_name
                )
                self.assertFalse(os.path.exists(testproject_dir))

    def test_importable_project_name(self):
        """
        startproject validates that project name doesn't clash with existing
        Python modules.
        """
        bad_name = 'os'
        args = ['startproject', bad_name]
        testproject_dir = os.path.join(self.test_dir, bad_name)

        out, err = self.run_django_admin(args)
        self.assertOutput(
            err,
            "CommandError: 'os' conflicts with the name of an existing "
            "Python module and cannot be used as a project name. Please try "
            "another name."
        )
        self.assertFalse(os.path.exists(testproject_dir))

    def test_simple_project_different_directory(self):
        "Make sure the startproject management command creates a project in a specific directory"
        args = ['startproject', 'testproject', 'othertestproject']
        testproject_dir = os.path.join(self.test_dir, 'othertestproject')
        os.mkdir(testproject_dir)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'manage.py')))

        # running again..
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(
            err,
            "already exists. Overlaying a project into an existing directory "
            "won't replace conflicting files."
        )

    def test_custom_project_template(self):
        "Make sure the startproject management command is able to use a different project template"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'customtestproject']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))

    def test_template_dir_with_trailing_slash(self):
        "Ticket 17475: Template dir passed has a trailing path separator"
        template_path = os.path.join(custom_templates_dir, 'project_template' + os.sep)
        args = ['startproject', '--template', template_path, 'customtestproject']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))

    def test_custom_project_template_from_tarball_by_path(self):
        "Make sure the startproject management command is able to use a different project template from a tarball"
        template_path = os.path.join(custom_templates_dir, 'project_template.tgz')
        args = ['startproject', '--template', template_path, 'tarballtestproject']
        testproject_dir = os.path.join(self.test_dir, 'tarballtestproject')

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_custom_project_template_from_tarball_to_alternative_location(self):
        "Startproject can use a project template from a tarball and create it in a specified location"
        template_path = os.path.join(custom_templates_dir, 'project_template.tgz')
        args = ['startproject', '--template', template_path, 'tarballtestproject', 'altlocation']
        testproject_dir = os.path.join(self.test_dir, 'altlocation')
        os.mkdir(testproject_dir)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_custom_project_template_from_tarball_by_url(self):
        """
        The startproject management command is able to use a different project
        template from a tarball via a URL.
        """
        template_url = '%s/custom_templates/project_template.tgz' % self.live_server_url

        args = ['startproject', '--template', template_url, 'urltestproject']
        testproject_dir = os.path.join(self.test_dir, 'urltestproject')

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_project_template_tarball_url(self):
        "Startproject management command handles project template tar/zip balls from non-canonical urls"
        template_url = '%s/custom_templates/project_template.tgz/' % self.live_server_url

        args = ['startproject', '--template', template_url, 'urltestproject']
        testproject_dir = os.path.join(self.test_dir, 'urltestproject')

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_file_without_extension(self):
        "Make sure the startproject management command is able to render custom files"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'customtestproject', '-e', 'txt', '-n', 'Procfile']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))
        base_path = os.path.join(testproject_dir, 'additional_dir')
        for f in ('Procfile', 'additional_file.py', 'requirements.txt'):
            self.assertTrue(os.path.exists(os.path.join(base_path, f)))
            with open(os.path.join(base_path, f)) as fh:
                self.assertEqual(fh.read().strip(), '# some file for customtestproject test project')

    def test_custom_project_template_context_variables(self):
        "Make sure template context variables are rendered with proper values"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'another_project', 'project_dir']
        testproject_dir = os.path.join(self.test_dir, 'project_dir')
        os.mkdir(testproject_dir)
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        test_manage_py = os.path.join(testproject_dir, 'manage.py')
        with open(test_manage_py) as fp:
            content = fp.read()
            self.assertIn("project_name = 'another_project'", content)
            self.assertIn("project_directory = '%s'" % testproject_dir, content)

    def test_no_escaping_of_project_variables(self):
        "Make sure template context variables are not html escaped"
        # We're using a custom command so we need the alternate settings
        self.write_settings('alternate_settings.py')
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = [
            'custom_startproject', '--template', template_path,
            'another_project', 'project_dir', '--extra', '<&>',
            '--settings=alternate_settings',
        ]
        testproject_dir = os.path.join(self.test_dir, 'project_dir')
        os.mkdir(testproject_dir)
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        test_manage_py = os.path.join(testproject_dir, 'additional_dir', 'extra.py')
        with open(test_manage_py) as fp:
            content = fp.read()
            self.assertIn("<&>", content)

    def test_custom_project_destination_missing(self):
        """
        Make sure an exception is raised when the provided
        destination directory doesn't exist
        """
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'yet_another_project', 'project_dir2']
        testproject_dir = os.path.join(self.test_dir, 'project_dir2')
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Destination directory '%s' does not exist, please create it first." % testproject_dir)
        self.assertFalse(os.path.exists(testproject_dir))

    def test_custom_project_template_with_non_ascii_templates(self):
        """
        The startproject management command is able to render templates with
        non-ASCII content.
        """
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, '--extension=txt', 'customtestproject']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        path = os.path.join(testproject_dir, 'ticket-18091-non-ascii-template.txt')
        with open(path, encoding='utf-8') as f:
            self.assertEqual(f.read().splitlines(False), [
                'Some non-ASCII text for testing ticket #18091:',
                'üäö €'])


class StartApp(AdminScriptTestCase):

    def test_invalid_name(self):
        """startapp validates that app name is a valid Python identifier."""
        for bad_name in ('7testproject', '../testproject'):
            with self.subTest(app_name=bad_name):
                args = ['startapp', bad_name]
                testproject_dir = os.path.join(self.test_dir, bad_name)

                out, err = self.run_django_admin(args)
                self.assertOutput(
                    err,
                    "CommandError: '{}' is not a valid app name. Please make "
                    "sure the name is a valid identifier.".format(bad_name)
                )
                self.assertFalse(os.path.exists(testproject_dir))

    def test_importable_name(self):
        """
        startapp validates that app name doesn't clash with existing Python
        modules.
        """
        bad_name = 'os'
        args = ['startapp', bad_name]
        testproject_dir = os.path.join(self.test_dir, bad_name)

        out, err = self.run_django_admin(args)
        self.assertOutput(
            err,
            "CommandError: 'os' conflicts with the name of an existing "
            "Python module and cannot be used as an app name. Please try "
            "another name."
        )
        self.assertFalse(os.path.exists(testproject_dir))

    def test_invalid_target_name(self):
        for bad_target in ('invalid.dir_name', '7invalid_dir_name', '.invalid_dir_name'):
            with self.subTest(bad_target):
                _, err = self.run_django_admin(['startapp', 'app', bad_target])
                self.assertOutput(
                    err,
                    "CommandError: '%s' is not a valid app directory. Please "
                    "make sure the directory is a valid identifier." % bad_target
                )

    def test_importable_target_name(self):
        _, err = self.run_django_admin(['startapp', 'app', 'os'])
        self.assertOutput(
            err,
            "CommandError: 'os' conflicts with the name of an existing Python "
            "module and cannot be used as an app directory. Please try "
            "another directory."
        )

    def test_overlaying_app(self):
        # Use a subdirectory so it is outside the PYTHONPATH.
        os.makedirs(os.path.join(self.test_dir, 'apps/app1'))
        self.run_django_admin(['startapp', 'app1', 'apps/app1'])
        out, err = self.run_django_admin(['startapp', 'app2', 'apps/app1'])
        self.assertOutput(
            err,
            "already exists. Overlaying an app into an existing directory "
            "won't replace conflicting files."
        )


class DiffSettings(AdminScriptTestCase):
    """Tests for diffsettings management command."""

    def test_basic(self):
        """Runs without error and emits settings diff."""
        self.write_settings('settings_to_diff.py', sdict={'FOO': '"bar"'})
        args = ['diffsettings', '--settings=settings_to_diff']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "FOO = 'bar'  ###")
        # Attributes from django.conf.Settings don't appear.
        self.assertNotInOutput(out, 'is_overridden = ')

    def test_settings_configured(self):
        out, err = self.run_manage(['diffsettings'], manage_py='configured_settings_manage.py')
        self.assertNoOutput(err)
        self.assertOutput(out, 'CUSTOM = 1  ###\nDEBUG = True')
        # Attributes from django.conf.UserSettingsHolder don't appear.
        self.assertNotInOutput(out, 'default_settings = ')

    def test_dynamic_settings_configured(self):
        # Custom default settings appear.
        out, err = self.run_manage(['diffsettings'], manage_py='configured_dynamic_settings_manage.py')
        self.assertNoOutput(err)
        self.assertOutput(out, "FOO = 'bar'  ###")

    def test_all(self):
        """The all option also shows settings with the default value."""
        self.write_settings('settings_to_diff.py', sdict={'STATIC_URL': 'None'})
        args = ['diffsettings', '--settings=settings_to_diff', '--all']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "### STATIC_URL = None")

    def test_custom_default(self):
        """
        The --default option specifies an alternate settings module for
        comparison.
        """
        self.write_settings('settings_default.py', sdict={'FOO': '"foo"', 'BAR': '"bar1"'})
        self.write_settings('settings_to_diff.py', sdict={'FOO': '"foo"', 'BAR': '"bar2"'})
        out, err = self.run_manage(['diffsettings', '--settings=settings_to_diff', '--default=settings_default'])
        self.assertNoOutput(err)
        self.assertNotInOutput(out, "FOO")
        self.assertOutput(out, "BAR = 'bar2'")

    def test_unified(self):
        """--output=unified emits settings diff in unified mode."""
        self.write_settings('settings_to_diff.py', sdict={'FOO': '"bar"'})
        args = ['diffsettings', '--settings=settings_to_diff', '--output=unified']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "+ FOO = 'bar'")
        self.assertOutput(out, "- SECRET_KEY = ''")
        self.assertOutput(out, "+ SECRET_KEY = 'django_tests_secret_key'")
        self.assertNotInOutput(out, "  APPEND_SLASH = True")

    def test_unified_all(self):
        """
        --output=unified --all emits settings diff in unified mode and includes
        settings with the default value.
        """
        self.write_settings('settings_to_diff.py', sdict={'FOO': '"bar"'})
        args = ['diffsettings', '--settings=settings_to_diff', '--output=unified', '--all']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "  APPEND_SLASH = True")
        self.assertOutput(out, "+ FOO = 'bar'")
        self.assertOutput(out, "- SECRET_KEY = ''")


class Dumpdata(AdminScriptTestCase):
    """Tests for dumpdata management command."""

    def setUp(self):
        super().setUp()
        self.write_settings('settings.py')

    def test_pks_parsing(self):
        """Regression for #20509

        Test would raise an exception rather than printing an error message.
        """
        args = ['dumpdata', '--pks=1']
        out, err = self.run_manage(args)
        self.assertOutput(err, "You can only use --pks option with one model")
        self.assertNoOutput(out)


class MainModule(AdminScriptTestCase):
    """python -m django works like django-admin."""

    def test_program_name_in_help(self):
        out, err = self.run_test(['-m', 'django', 'help'])
        self.assertOutput(out, "Type 'python -m django help <subcommand>' for help on a specific subcommand.")


class DjangoAdminSuggestions(AdminScriptTestCase):
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py')

    def test_suggestions(self):
        args = ['rnserver', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'rnserver'. Did you mean runserver?")

    def test_no_suggestions(self):
        args = ['abcdef', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertNotInOutput(err, 'Did you mean')