django/tests/admin_scripts/tests.py

# -*- coding: utf-8 -*-
"""
A series of tests to establish that the command-line managment tools work as
advertised - especially with regards to the handling of the DJANGO_SETTINGS_MODULE
and default settings.py files.
"""
from __future__ import unicode_literals

import os
import re
import shutil
import socket
import subprocess
import sys
import codecs

import django
from django import conf, get_version
from django.conf import settings
from django.core.management import BaseCommand, CommandError
from django.db import connection
from django.test.runner import DiscoverRunner
from django.utils import unittest
from django.utils.encoding import force_text
from django.utils._os import upath
from django.utils.six import StringIO
from django.test import LiveServerTestCase


test_dir = os.path.realpath(os.path.join(os.environ['DJANGO_TEST_TEMP_DIR'], 'test_project'))
if not os.path.exists(test_dir):
    os.mkdir(test_dir)
    open(os.path.join(test_dir, '__init__.py'), 'w').close()

custom_templates_dir = os.path.join(os.path.dirname(__file__), 'custom_templates')


class AdminScriptTestCase(unittest.TestCase):
    def write_settings(self, filename, apps=None, is_dir=False, sdict=None):
        if is_dir:
            settings_dir = os.path.join(test_dir, filename)
            os.mkdir(settings_dir)
            settings_file_path = os.path.join(settings_dir, '__init__.py')
        else:
            settings_file_path = os.path.join(test_dir, filename)

        with open(settings_file_path, 'w') as settings_file:
            settings_file.write('# Settings file automatically generated by admin_scripts test case\n')
            exports = [
                'DATABASES',
                'ROOT_URLCONF',
                'SECRET_KEY',
            ]
            for s in exports:
                if hasattr(settings, s):
                    o = getattr(settings, s)
                    if not isinstance(o, dict):
                        o = "'%s'" % o
                    settings_file.write("%s = %s\n" % (s, o))

            if apps is None:
                apps = ['django.contrib.auth', 'django.contrib.contenttypes', 'admin_scripts']

            settings_file.write("INSTALLED_APPS = %s\n" % apps)

            if sdict:
                for k, v in sdict.items():
                    settings_file.write("%s = %s\n" % (k, v))

    def remove_settings(self, filename, is_dir=False):
        full_name = os.path.join(test_dir, filename)
        if is_dir:
            shutil.rmtree(full_name)
        else:
            os.remove(full_name)

        # Also try to remove the compiled file; if it exists, it could
        # mess up later tests that depend upon the .py file not existing
        try:
            if sys.platform.startswith('java'):
                # Jython produces module$py.class files
                os.remove(re.sub(r'\.py$', '$py.class', full_name))
            else:
                # CPython produces module.pyc files
                os.remove(full_name + 'c')
        except OSError:
            pass
        # Also remove a __pycache__ directory, if it exists
        cache_name = os.path.join(test_dir, '__pycache__')
        if os.path.isdir(cache_name):
            shutil.rmtree(cache_name)

    def _ext_backend_paths(self):
        """
        Returns the paths for any external backend packages.
        """
        paths = []
        first_package_re = re.compile(r'(^[^\.]+)\.')
        for backend in settings.DATABASES.values():
            result = first_package_re.findall(backend['ENGINE'])
            if result and result != ['django']:
                backend_pkg = __import__(result[0])
                backend_dir = os.path.dirname(backend_pkg.__file__)
                paths.append(os.path.dirname(backend_dir))
        return paths

    def run_test(self, script, args, settings_file=None, apps=None):
        project_dir = test_dir
        base_dir = os.path.dirname(test_dir)
        # The base dir for Django's tests is one level up.
        tests_dir = os.path.dirname(os.path.dirname(__file__))
        # The base dir for Django is one level above the test dir. We don't use
        # `import django` to figure that out, so we don't pick up a Django
        # from site-packages or similar.
        django_dir = os.path.dirname(tests_dir)
        ext_backend_base_dirs = self._ext_backend_paths()

        # Remember the old environment
        old_django_settings_module = os.environ.get('DJANGO_SETTINGS_MODULE', None)
        if sys.platform.startswith('java'):
            python_path_var_name = 'JYTHONPATH'
        else:
            python_path_var_name = 'PYTHONPATH'

        old_python_path = os.environ.get(python_path_var_name, None)
        old_cwd = os.getcwd()

        # Set the test environment
        if settings_file:
            os.environ['DJANGO_SETTINGS_MODULE'] = settings_file
        elif 'DJANGO_SETTINGS_MODULE' in os.environ:
            del os.environ['DJANGO_SETTINGS_MODULE']
        python_path = [base_dir, django_dir, tests_dir]
        python_path.extend(ext_backend_base_dirs)
        os.environ[python_path_var_name] = os.pathsep.join(python_path)

        # Move to the test directory and run
        os.chdir(test_dir)
        out, err = subprocess.Popen([sys.executable, script] + args,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                universal_newlines=True).communicate()
        # Restore the old environment
        if old_django_settings_module:
            os.environ['DJANGO_SETTINGS_MODULE'] = old_django_settings_module
        if old_python_path:
            os.environ[python_path_var_name] = old_python_path
        # Move back to the old working directory
        os.chdir(old_cwd)

        return out, err

    def run_django_admin(self, args, settings_file=None):
        script_dir = os.path.abspath(os.path.join(os.path.dirname(upath(django.__file__)), 'bin'))
        return self.run_test(os.path.join(script_dir, 'django-admin.py'), args, settings_file)

    def run_manage(self, args, settings_file=None):
        def safe_remove(path):
            try:
                os.remove(path)
            except OSError:
                pass

        conf_dir = os.path.dirname(upath(conf.__file__))
        template_manage_py = os.path.join(conf_dir, 'project_template', 'manage.py')

        test_manage_py = os.path.join(test_dir, 'manage.py')
        shutil.copyfile(template_manage_py, test_manage_py)

        with open(test_manage_py, 'r') as fp:
            manage_py_contents = fp.read()
        manage_py_contents = manage_py_contents.replace(
            "{{ project_name }}", "test_project")
        with open(test_manage_py, 'w') as fp:
            fp.write(manage_py_contents)
        self.addCleanup(safe_remove, test_manage_py)

        return self.run_test('./manage.py', args, settings_file)

    def assertNoOutput(self, stream):
        "Utility assertion: assert that the given stream is empty"
        self.assertEqual(len(stream), 0, "Stream should be empty: actually contains '%s'" % stream)

    def assertOutput(self, stream, msg):
        "Utility assertion: assert that the given message exists in the output"
        stream = force_text(stream)
        self.assertTrue(msg in stream, "'%s' does not match actual output text '%s'" % (msg, stream))

    def assertNotInOutput(self, stream, msg):
        "Utility assertion: assert that the given message doesn't exist in the output"
        stream = force_text(stream)
        self.assertFalse(msg in stream, "'%s' matches actual output text '%s'" % (msg, stream))

##########################################################################
# DJANGO ADMIN TESTS
# This first series of test classes checks the environment processing
# of the django-admin.py script
##########################################################################


class DjangoAdminNoSettings(AdminScriptTestCase):
    "A series of tests for django-admin.py when there is no settings.py file."

    def test_builtin_command(self):
        "no settings: django-admin builtin commands fail with an error when no settings provided"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_bad_settings(self):
        "no settings: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "no settings: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")


class DjangoAdminDefaultSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    contains the test application.
    """
    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "default: django-admin builtin commands fail with an error when no settings provided"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "default: django-admin builtin commands succeed if settings are provided as argument"
        args = ['sqlall', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_environment(self):
        "default: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_bad_settings(self):
        "default: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "default: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_custom_command(self):
        "default: django-admin can't execute user commands if it isn't provided settings"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "default: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

    def test_custom_command_with_environment(self):
        "default: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

class DjangoAdminFullPathDefaultSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    contains the test application specified using a full path.
    """
    def setUp(self):
        self.write_settings('settings.py', ['django.contrib.auth', 'django.contrib.contenttypes', 'admin_scripts'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "fulldefault: django-admin builtin commands fail with an error when no settings provided"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "fulldefault: django-admin builtin commands succeed if a settings file is provided"
        args = ['sqlall', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_environment(self):
        "fulldefault: django-admin builtin commands succeed if the environment contains settings"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_bad_settings(self):
        "fulldefault: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "fulldefault: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_custom_command(self):
        "fulldefault: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "fulldefault: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

    def test_custom_command_with_environment(self):
        "fulldefault: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

class DjangoAdminMinimalSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    doesn't contain the test application.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "minimal: django-admin builtin commands fail with an error when no settings provided"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "minimal: django-admin builtin commands fail if settings are provided as argument"
        args = ['sqlall', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'App with label admin_scripts could not be found')

    def test_builtin_with_environment(self):
        "minimal: django-admin builtin commands fail if settings are provided in the environment"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, 'App with label admin_scripts could not be found')

    def test_builtin_with_bad_settings(self):
        "minimal: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "minimal: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_custom_command(self):
        "minimal: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "minimal: django-admin can't execute user commands, even if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_environment(self):
        "minimal: django-admin can't execute user commands, even if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

class DjangoAdminAlternateSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings file
    with a name other than 'settings.py'.
    """
    def setUp(self):
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: django-admin builtin commands fail with an error when no settings provided"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "alternate: django-admin builtin commands succeed if settings are provided as argument"
        args = ['sqlall', '--settings=test_project.alternate_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_environment(self):
        "alternate: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_bad_settings(self):
        "alternate: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "alternate: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_custom_command(self):
        "alternate: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "alternate: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.alternate_settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

    def test_custom_command_with_environment(self):
        "alternate: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")


class DjangoAdminMultipleSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when multiple settings files
    (including the default 'settings.py') are available. The default settings
    file is insufficient for performing the operations described, so the
    alternate settings must be used by the running script.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: django-admin builtin commands fail with an error when no settings provided"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "alternate: django-admin builtin commands succeed if settings are provided as argument"
        args = ['sqlall', '--settings=test_project.alternate_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_environment(self):
        "alternate: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_bad_settings(self):
        "alternate: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "alternate: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_custom_command(self):
        "alternate: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "alternate: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.alternate_settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

    def test_custom_command_with_environment(self):
        "alternate: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")


class DjangoAdminSettingsDirectory(AdminScriptTestCase):
    """
    A series of tests for django-admin.py when the settings file is in a
    directory. (see #9751).
    """

    def setUp(self):
        self.write_settings('settings', is_dir=True)

    def tearDown(self):
        self.remove_settings('settings', is_dir=True)

    def test_setup_environ(self):
        "directory: startapp creates the correct directory"
        args = ['startapp', 'settings_test']
        app_path = os.path.join(test_dir, 'settings_test')
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.addCleanup(shutil.rmtree, app_path)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(app_path))

    def test_setup_environ_custom_template(self):
        "directory: startapp creates the correct directory with a custom template"
        template_path = os.path.join(custom_templates_dir, 'app_template')
        args = ['startapp', '--template', template_path, 'custom_settings_test']
        app_path = os.path.join(test_dir, 'custom_settings_test')
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.addCleanup(shutil.rmtree, app_path)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(app_path))
        self.assertTrue(os.path.exists(os.path.join(app_path, 'api.py')))

    def test_builtin_command(self):
        "directory: django-admin builtin commands fail with an error when no settings provided"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_bad_settings(self):
        "directory: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "directory: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_custom_command(self):
        "directory: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_builtin_with_settings(self):
        "directory: django-admin builtin commands succeed if settings are provided as argument"
        args = ['sqlall', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_environment(self):
        "directory: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')


##########################################################################
# MANAGE.PY TESTS
# This next series of test classes checks the environment processing
# of the generated manage.py script
##########################################################################

class ManageNoSettings(AdminScriptTestCase):
    "A series of tests for manage.py when there is no settings.py file."

    def test_builtin_command(self):
        "no settings: manage.py builtin commands fail with an error when no settings provided"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'test_project.settings'")

    def test_builtin_with_bad_settings(self):
        "no settings: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "no settings: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")


class ManageDefaultSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    contains the test application.
    """
    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "default: manage.py builtin commands succeed when default settings are appropriate"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_settings(self):
        "default: manage.py builtin commands succeed if settings are provided as argument"
        args = ['sqlall', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_environment(self):
        "default: manage.py builtin commands succeed if settings are provided in the environment"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_bad_settings(self):
        "default: manage.py builtin commands succeed if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "default: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_custom_command(self):
        "default: manage.py can execute user commands when default settings are appropriate"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

    def test_custom_command_with_settings(self):
        "default: manage.py can execute user commands when settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

    def test_custom_command_with_environment(self):
        "default: manage.py can execute user commands when settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")


class ManageFullPathDefaultSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    contains the test application specified using a full path.
    """
    def setUp(self):
        self.write_settings('settings.py', ['django.contrib.auth', 'django.contrib.contenttypes', 'admin_scripts'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "fulldefault: manage.py builtin commands succeed when default settings are appropriate"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_settings(self):
        "fulldefault: manage.py builtin commands succeed if settings are provided as argument"
        args = ['sqlall', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_environment(self):
        "fulldefault: manage.py builtin commands succeed if settings are provided in the environment"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_bad_settings(self):
        "fulldefault: manage.py builtin commands succeed if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "fulldefault: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_custom_command(self):
        "fulldefault: manage.py can execute user commands when default settings are appropriate"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

    def test_custom_command_with_settings(self):
        "fulldefault: manage.py can execute user commands when settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

    def test_custom_command_with_environment(self):
        "fulldefault: manage.py can execute user commands when settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

class ManageMinimalSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    doesn't contain the test application.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "minimal: manage.py builtin commands fail with an error when no settings provided"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'App with label admin_scripts could not be found')

    def test_builtin_with_settings(self):
        "minimal: manage.py builtin commands fail if settings are provided as argument"
        args = ['sqlall', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'App with label admin_scripts could not be found')

    def test_builtin_with_environment(self):
        "minimal: manage.py builtin commands fail if settings are provided in the environment"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, 'App with label admin_scripts could not be found')

    def test_builtin_with_bad_settings(self):
        "minimal: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "minimal: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_custom_command(self):
        "minimal: manage.py can't execute user commands without appropriate settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "minimal: manage.py can't execute user commands, even if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_environment(self):
        "minimal: manage.py can't execute user commands, even if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

class ManageAlternateSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings file
    with a name other than 'settings.py'.
    """
    def setUp(self):
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: manage.py builtin commands fail with an error when no default settings provided"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'test_project.settings'")

    def test_builtin_with_settings(self):
        "alternate: manage.py builtin commands work with settings provided as argument"
        args = ['sqlall', '--settings=alternate_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        expected = ('create table %s'
                    % connection.ops.quote_name('admin_scripts_article'))
        self.assertTrue(expected.lower() in out.lower())
        self.assertNoOutput(err)

    def test_builtin_with_environment(self):
        "alternate: manage.py builtin commands work if settings are provided in the environment"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args, 'alternate_settings')
        expected = ('create table %s'
                    % connection.ops.quote_name('admin_scripts_article'))
        self.assertTrue(expected.lower() in out.lower())
        self.assertNoOutput(err)

    def test_builtin_with_bad_settings(self):
        "alternate: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "alternate: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_custom_command(self):
        "alternate: manage.py can't execute user commands without settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'test_project.settings'")

    def test_custom_command_with_settings(self):
        "alternate: manage.py can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertOutput(out, "EXECUTE:NoArgsCommand options=[('pythonpath', None), ('settings', 'alternate_settings'), ('traceback', None), ('verbosity', '1')]")
        self.assertNoOutput(err)

    def test_custom_command_with_environment(self):
        "alternate: manage.py can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertOutput(out, "EXECUTE:NoArgsCommand options=[('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")
        self.assertNoOutput(err)


class ManageMultipleSettings(AdminScriptTestCase):
    """A series of tests for manage.py when multiple settings files
    (including the default 'settings.py') are available. The default settings
    file is insufficient for performing the operations described, so the
    alternate settings must be used by the running script.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "multiple: manage.py builtin commands fail with an error when no settings provided"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'App with label admin_scripts could not be found.')

    def test_builtin_with_settings(self):
        "multiple: manage.py builtin commands succeed if settings are provided as argument"
        args = ['sqlall', '--settings=alternate_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_environment(self):
        "multiple: manage.py can execute builtin commands if settings are provided in the environment"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, 'CREATE TABLE')

    def test_builtin_with_bad_settings(self):
        "multiple: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['sqlall', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_builtin_with_bad_environment(self):
        "multiple: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Could not import settings 'bad_settings'")

    def test_custom_command(self):
        "multiple: manage.py can't execute user commands using default settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "multiple: manage.py can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")

    def test_custom_command_with_environment(self):
        "multiple: manage.py can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand")


class ManageSettingsWithImportError(AdminScriptTestCase):
    """Tests for manage.py when using the default settings.py file
    with an import error. Ticket #14130.
    """
    def tearDown(self):
        self.remove_settings('settings.py')

    def write_settings_with_import_error(self, filename, apps=None, is_dir=False, sdict=None):
        if is_dir:
            settings_dir = os.path.join(test_dir, filename)
            os.mkdir(settings_dir)
            settings_file_path = os.path.join(settings_dir, '__init__.py')
        else:
            settings_file_path = os.path.join(test_dir, filename)
        with open(settings_file_path, 'w') as settings_file:
            settings_file.write('# Settings file automatically generated by admin_scripts test case\n')
            settings_file.write('# The next line will cause an import error:\nimport foo42bar\n')

    def test_builtin_command(self):
        """
        import error: manage.py builtin commands shows useful diagnostic info
        when settings with import errors is provided
        """
        self.write_settings_with_import_error('settings.py')
        args = ['sqlall', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named")
        self.assertOutput(err, "foo42bar")

    def test_builtin_command_with_attribute_error(self):
        """
        manage.py builtin commands does not swallow attribute errors from bad settings (#18845)
        """
        self.write_settings('settings.py', sdict={'BAD_VAR': 'INSTALLED_APPS.crash'})
        args = ['collectstatic', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "AttributeError: 'list' object has no attribute 'crash'")


class ManageValidate(AdminScriptTestCase):
    def tearDown(self):
        self.remove_settings('settings.py')

    def test_nonexistent_app(self):
        "manage.py validate reports an error on a non-existent app in INSTALLED_APPS"
        self.write_settings('settings.py', apps=['admin_scriptz.broken_app'], sdict={'USE_I18N': False})
        args = ['validate']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'No module named')
        self.assertOutput(err, 'admin_scriptz')

    def test_broken_app(self):
        "manage.py validate reports an ImportError if an app's models.py raises one on import"
        self.write_settings('settings.py', apps=['admin_scripts.broken_app'])
        args = ['validate']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'ImportError')

    def test_complex_app(self):
        "manage.py validate does not raise an ImportError validating a complex app with nested calls to load_app"
        self.write_settings('settings.py',
            apps=['admin_scripts.complex_app', 'admin_scripts.simple_app'],
            sdict={'DEBUG': True})
        args = ['validate']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, '0 errors found')

    def test_app_with_import(self):
        "manage.py validate does not raise errors when an app imports a base class that itself has an abstract base"
        self.write_settings('settings.py',
            apps=['admin_scripts.app_with_import',
                  'django.contrib.comments',
                  'django.contrib.auth',
                  'django.contrib.contenttypes',
                  'django.contrib.sites'],
            sdict={'DEBUG': True})
        args = ['validate']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, '0 errors found')


class CustomTestRunner(DiscoverRunner):

    def __init__(self, *args, **kwargs):
        assert 'liveserver' not in kwargs
        super(CustomTestRunner, self).__init__(*args, **kwargs)

    def run_tests(self, test_labels, extra_tests=None, **kwargs):
        pass

class ManageTestCommand(AdminScriptTestCase):
    def setUp(self):
        from django.core.management.commands.test import Command as TestCommand
        self.cmd = TestCommand()

    def test_liveserver(self):
        """
        Ensure that the --liveserver option sets the environment variable
        correctly.
        Refs #2879.
        """

        # Backup original state
        address_predefined = 'DJANGO_LIVE_TEST_SERVER_ADDRESS' in os.environ
        old_address = os.environ.get('DJANGO_LIVE_TEST_SERVER_ADDRESS')

        self.cmd.handle(verbosity=0, testrunner='admin_scripts.tests.CustomTestRunner')

        # Original state hasn't changed
        self.assertEqual('DJANGO_LIVE_TEST_SERVER_ADDRESS' in os.environ, address_predefined)
        self.assertEqual(os.environ.get('DJANGO_LIVE_TEST_SERVER_ADDRESS'), old_address)

        self.cmd.handle(verbosity=0, testrunner='admin_scripts.tests.CustomTestRunner',
                        liveserver='blah')

        # Variable was correctly set
        self.assertEqual(os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'], 'blah')

        # Restore original state
        if address_predefined:
            os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = old_address
        else:
            del os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS']


class ManageRunserver(AdminScriptTestCase):
    def setUp(self):
        from django.core.management.commands.runserver import Command

        def monkey_run(*args, **options):
            return

        self.cmd = Command()
        self.cmd.run = monkey_run

    def assertServerSettings(self, addr, port, ipv6=None, raw_ipv6=False):
        self.assertEqual(self.cmd.addr, addr)
        self.assertEqual(self.cmd.port, port)
        self.assertEqual(self.cmd.use_ipv6, ipv6)
        self.assertEqual(self.cmd._raw_ipv6, raw_ipv6)

    def test_runserver_addrport(self):
        self.cmd.handle()
        self.assertServerSettings('127.0.0.1', '8000')

        self.cmd.handle(addrport="1.2.3.4:8000")
        self.assertServerSettings('1.2.3.4', '8000')

        self.cmd.handle(addrport="7000")
        self.assertServerSettings('127.0.0.1', '7000')

    @unittest.skipUnless(socket.has_ipv6, "platform doesn't support IPv6")
    def test_runner_addrport_ipv6(self):
        self.cmd.handle(addrport="", use_ipv6=True)
        self.assertServerSettings('::1', '8000', ipv6=True, raw_ipv6=True)

        self.cmd.handle(addrport="7000", use_ipv6=True)
        self.assertServerSettings('::1', '7000', ipv6=True, raw_ipv6=True)

        self.cmd.handle(addrport="[2001:0db8:1234:5678::9]:7000")
        self.assertServerSettings('2001:0db8:1234:5678::9', '7000', ipv6=True, raw_ipv6=True)

    def test_runner_hostname(self):
        self.cmd.handle(addrport="localhost:8000")
        self.assertServerSettings('localhost', '8000')

        self.cmd.handle(addrport="test.domain.local:7000")
        self.assertServerSettings('test.domain.local', '7000')

    @unittest.skipUnless(socket.has_ipv6, "platform doesn't support IPv6")
    def test_runner_hostname_ipv6(self):
        self.cmd.handle(addrport="test.domain.local:7000", use_ipv6=True)
        self.assertServerSettings('test.domain.local', '7000', ipv6=True)

    def test_runner_ambiguous(self):
        # Only 4 characters, all of which could be in an ipv6 address
        self.cmd.handle(addrport="beef:7654")
        self.assertServerSettings('beef', '7654')

        # Uses only characters that could be in an ipv6 address
        self.cmd.handle(addrport="deadbeef:7654")
        self.assertServerSettings('deadbeef', '7654')


##########################################################################
# COMMAND PROCESSING TESTS
# Check that user-space commands are correctly handled - in particular,
# that arguments to the commands are correctly parsed and processed.
##########################################################################

class CommandTypes(AdminScriptTestCase):
    "Tests for the various types of base command types that can be defined."
    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_version(self):
        "version is handled as a special case"
        args = ['version']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, get_version())

    def test_version_alternative(self):
        "--version is equivalent to version"
        args1, args2 = ['version'], ['--version']
        self.assertEqual(self.run_manage(args1), self.run_manage(args2))

    def test_help(self):
        "help is handled as a special case"
        args = ['help']
        out, err = self.run_manage(args)
        self.assertOutput(out, "Usage: manage.py subcommand [options] [args]")
        self.assertOutput(out, "Type 'manage.py help <subcommand>' for help on a specific subcommand.")
        self.assertOutput(out, '[django]')
        self.assertOutput(out, 'startapp')
        self.assertOutput(out, 'startproject')

    def test_help_commands(self):
        "help --commands shows the list of all available commands"
        args = ['help', '--commands']
        out, err = self.run_manage(args)
        self.assertNotInOutput(out, 'Usage:')
        self.assertNotInOutput(out, 'Options:')
        self.assertNotInOutput(out, '[django]')
        self.assertOutput(out, 'startapp')
        self.assertOutput(out, 'startproject')
        self.assertNotInOutput(out, '\n\n')

    def test_help_alternative(self):
        "--help is equivalent to help"
        args1, args2 = ['help'], ['--help']
        self.assertEqual(self.run_manage(args1), self.run_manage(args2))

    def test_help_short_altert(self):
        "-h is handled as a short form of --help"
        args1, args2 = ['--help'], ['-h']
        self.assertEqual(self.run_manage(args1), self.run_manage(args2))

    def test_specific_help(self):
        "--help can be used on a specific command"
        args = ['sqlall', '--help']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "Prints the CREATE TABLE, custom SQL and CREATE INDEX SQL statements for the given model module name(s).")

    def test_base_command(self):
        "User BaseCommands can execute when a label is provided"
        args = ['base_command', 'testlabel']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:BaseCommand labels=('testlabel',), options=[('option_a', '1'), ('option_b', '2'), ('option_c', '3'), ('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")

    def test_base_command_no_label(self):
        "User BaseCommands can execute when no labels are provided"
        args = ['base_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:BaseCommand labels=(), options=[('option_a', '1'), ('option_b', '2'), ('option_c', '3'), ('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")

    def test_base_command_multiple_label(self):
        "User BaseCommands can execute when no labels are provided"
        args = ['base_command', 'testlabel', 'anotherlabel']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:BaseCommand labels=('testlabel', 'anotherlabel'), options=[('option_a', '1'), ('option_b', '2'), ('option_c', '3'), ('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")

    def test_base_command_with_option(self):
        "User BaseCommands can execute with options when a label is provided"
        args = ['base_command', 'testlabel', '--option_a=x']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:BaseCommand labels=('testlabel',), options=[('option_a', 'x'), ('option_b', '2'), ('option_c', '3'), ('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")

    def test_base_command_with_options(self):
        "User BaseCommands can execute with multiple options when a label is provided"
        args = ['base_command', 'testlabel', '-a', 'x', '--option_b=y']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:BaseCommand labels=('testlabel',), options=[('option_a', 'x'), ('option_b', 'y'), ('option_c', '3'), ('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")

    def test_base_run_from_argv(self):
        """
        Test run_from_argv properly terminates even with custom execute() (#19665)
        Also test proper traceback display.
        """
        command = BaseCommand()
        def raise_command_error(*args, **kwargs):
            raise CommandError("Custom error")

        old_stderr = sys.stderr
        sys.stderr = err = StringIO()
        try:
            command.execute = lambda args: args  # This will trigger TypeError
            with self.assertRaises(SystemExit):
                command.run_from_argv(['', ''])
            err_message = err.getvalue()
            # Exceptions other than CommandError automatically output the traceback
            self.assertIn("Traceback", err_message)
            self.assertIn("TypeError", err_message)

            command.execute = raise_command_error
            err.truncate(0)
            with self.assertRaises(SystemExit):
                command.run_from_argv(['', ''])
            err_message = err.getvalue()
            self.assertNotIn("Traceback", err_message)
            self.assertIn("CommandError", err_message)

            err.truncate(0)
            with self.assertRaises(SystemExit):
                command.run_from_argv(['', '', '--traceback'])
            err_message = err.getvalue()
            self.assertIn("Traceback (most recent call last)", err_message)
            self.assertIn("CommandError", err_message)
        finally:
            sys.stderr = old_stderr

    def test_noargs(self):
        "NoArg Commands can be executed"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:NoArgsCommand options=[('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")

    def test_noargs_with_args(self):
        "NoArg Commands raise an error if an argument is provided"
        args = ['noargs_command', 'argument']
        out, err = self.run_manage(args)
        self.assertOutput(err, "Error: Command doesn't accept any arguments")

    def test_app_command(self):
        "User AppCommands can execute when a single app name is provided"
        args = ['app_command', 'auth']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:AppCommand app=<module 'django.contrib.auth.models'")
        self.assertOutput(out, os.sep.join(['django', 'contrib', 'auth', 'models.py']))
        self.assertOutput(out, "'>, options=[('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")

    def test_app_command_no_apps(self):
        "User AppCommands raise an error when no app name is provided"
        args = ['app_command']
        out, err = self.run_manage(args)
        self.assertOutput(err, 'Error: Enter at least one appname.')

    def test_app_command_multiple_apps(self):
        "User AppCommands raise an error when multiple app names are provided"
        args = ['app_command', 'auth', 'contenttypes']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:AppCommand app=<module 'django.contrib.auth.models'")
        self.assertOutput(out, os.sep.join(['django', 'contrib', 'auth', 'models.py']))
        self.assertOutput(out, "'>, options=[('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")
        self.assertOutput(out, "EXECUTE:AppCommand app=<module 'django.contrib.contenttypes.models'")
        self.assertOutput(out, os.sep.join(['django', 'contrib', 'contenttypes', 'models.py']))
        self.assertOutput(out, "'>, options=[('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")

    def test_app_command_invalid_appname(self):
        "User AppCommands can execute when a single app name is provided"
        args = ['app_command', 'NOT_AN_APP']
        out, err = self.run_manage(args)
        self.assertOutput(err, "App with label NOT_AN_APP could not be found")

    def test_app_command_some_invalid_appnames(self):
        "User AppCommands can execute when some of the provided app names are invalid"
        args = ['app_command', 'auth', 'NOT_AN_APP']
        out, err = self.run_manage(args)
        self.assertOutput(err, "App with label NOT_AN_APP could not be found")

    def test_label_command(self):
        "User LabelCommands can execute when a label is provided"
        args = ['label_command', 'testlabel']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:LabelCommand label=testlabel, options=[('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")

    def test_label_command_no_label(self):
        "User LabelCommands raise an error if no label is provided"
        args = ['label_command']
        out, err = self.run_manage(args)
        self.assertOutput(err, 'Enter at least one label')

    def test_label_command_multiple_label(self):
        "User LabelCommands are executed multiple times if multiple labels are provided"
        args = ['label_command', 'testlabel', 'anotherlabel']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:LabelCommand label=testlabel, options=[('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")
        self.assertOutput(out, "EXECUTE:LabelCommand label=anotherlabel, options=[('pythonpath', None), ('settings', None), ('traceback', None), ('verbosity', '1')]")

class ArgumentOrder(AdminScriptTestCase):
    """Tests for 2-stage argument parsing scheme.

    django-admin command arguments are parsed in 2 parts; the core arguments
    (--settings, --traceback and --pythonpath) are parsed using a Lax parser.
    This Lax parser ignores any unknown options. Then the full settings are
    passed to the command parser, which extracts commands of interest to the
    individual command.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')
        self.remove_settings('alternate_settings.py')

    def test_setting_then_option(self):
        "Options passed after settings are correctly handled"
        args = ['base_command', 'testlabel', '--settings=alternate_settings', '--option_a=x']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:BaseCommand labels=('testlabel',), options=[('option_a', 'x'), ('option_b', '2'), ('option_c', '3'), ('pythonpath', None), ('settings', 'alternate_settings'), ('traceback', None), ('verbosity', '1')]")

    def test_setting_then_short_option(self):
        "Short options passed after settings are correctly handled"
        args = ['base_command', 'testlabel', '--settings=alternate_settings', '--option_a=x']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:BaseCommand labels=('testlabel',), options=[('option_a', 'x'), ('option_b', '2'), ('option_c', '3'), ('pythonpath', None), ('settings', 'alternate_settings'), ('traceback', None), ('verbosity', '1')]")

    def test_option_then_setting(self):
        "Options passed before settings are correctly handled"
        args = ['base_command', 'testlabel', '--option_a=x', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:BaseCommand labels=('testlabel',), options=[('option_a', 'x'), ('option_b', '2'), ('option_c', '3'), ('pythonpath', None), ('settings', 'alternate_settings'), ('traceback', None), ('verbosity', '1')]")

    def test_short_option_then_setting(self):
        "Short options passed before settings are correctly handled"
        args = ['base_command', 'testlabel', '-a', 'x', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:BaseCommand labels=('testlabel',), options=[('option_a', 'x'), ('option_b', '2'), ('option_c', '3'), ('pythonpath', None), ('settings', 'alternate_settings'), ('traceback', None), ('verbosity', '1')]")

    def test_option_then_setting_then_option(self):
        "Options are correctly handled when they are passed before and after a setting"
        args = ['base_command', 'testlabel', '--option_a=x', '--settings=alternate_settings', '--option_b=y']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:BaseCommand labels=('testlabel',), options=[('option_a', 'x'), ('option_b', 'y'), ('option_c', '3'), ('pythonpath', None), ('settings', 'alternate_settings'), ('traceback', None), ('verbosity', '1')]")


class StartProject(LiveServerTestCase, AdminScriptTestCase):

    def test_wrong_args(self):
        "Make sure passing the wrong kinds of arguments raises a CommandError"
        out, err = self.run_django_admin(['startproject'])
        self.assertNoOutput(out)
        self.assertOutput(err, "you must provide a project name")

    def test_simple_project(self):
        "Make sure the startproject management command creates a project"
        args = ['startproject', 'testproject']
        testproject_dir = os.path.join(test_dir, 'testproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))

        # running again..
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "already exists")

    def test_invalid_project_name(self):
        "Make sure the startproject management command validates a project name"
        for bad_name in ('7testproject', '../testproject'):
            args = ['startproject', bad_name]
            testproject_dir = os.path.join(test_dir, bad_name)
            self.addCleanup(shutil.rmtree, testproject_dir, True)

            out, err = self.run_django_admin(args)
            self.assertOutput(err, "Error: '%s' is not a valid project name. "
                "Please make sure the name begins with a letter or underscore." % bad_name)
            self.assertFalse(os.path.exists(testproject_dir))

    def test_simple_project_different_directory(self):
        "Make sure the startproject management command creates a project in a specific directory"
        args = ['startproject', 'testproject', 'othertestproject']
        testproject_dir = os.path.join(test_dir, 'othertestproject')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'manage.py')))

        # running again..
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "already exists")

    def test_custom_project_template(self):
        "Make sure the startproject management command is able to use a different project template"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'customtestproject']
        testproject_dir = os.path.join(test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))

    def test_template_dir_with_trailing_slash(self):
        "Ticket 17475: Template dir passed has a trailing path separator"
        template_path = os.path.join(custom_templates_dir, 'project_template' + os.sep)
        args = ['startproject', '--template', template_path, 'customtestproject']
        testproject_dir = os.path.join(test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))

    def test_custom_project_template_from_tarball_by_path(self):
        "Make sure the startproject management command is able to use a different project template from a tarball"
        template_path = os.path.join(custom_templates_dir, 'project_template.tgz')
        args = ['startproject', '--template', template_path, 'tarballtestproject']
        testproject_dir = os.path.join(test_dir, 'tarballtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_custom_project_template_from_tarball_to_alternative_location(self):
        "Startproject can use a project template from a tarball and create it in a specified location"
        template_path = os.path.join(custom_templates_dir, 'project_template.tgz')
        args = ['startproject', '--template', template_path, 'tarballtestproject', 'altlocation']
        testproject_dir = os.path.join(test_dir, 'altlocation')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_custom_project_template_from_tarball_by_url(self):
        "Make sure the startproject management command is able to use a different project template from a tarball via a url"
        template_url = '%s/admin_scripts/custom_templates/project_template.tgz' % self.live_server_url

        args = ['startproject', '--template', template_url, 'urltestproject']
        testproject_dir = os.path.join(test_dir, 'urltestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_project_template_tarball_url(self):
        "Startproject management command handles project template tar/zip balls from non-canonical urls"
        template_url = '%s/admin_scripts/custom_templates/project_template.tgz/' % self.live_server_url

        args = ['startproject', '--template', template_url, 'urltestproject']
        testproject_dir = os.path.join(test_dir, 'urltestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_file_without_extension(self):
        "Make sure the startproject management command is able to render custom files"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'customtestproject', '-e', 'txt', '-n', 'Procfile']
        testproject_dir = os.path.join(test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))
        base_path = os.path.join(testproject_dir, 'additional_dir')
        for f in ('Procfile', 'additional_file.py', 'requirements.txt'):
            self.assertTrue(os.path.exists(os.path.join(base_path, f)))
            with open(os.path.join(base_path, f)) as fh:
                self.assertEqual(fh.read(),
                    '# some file for customtestproject test project')

    def test_custom_project_template_context_variables(self):
        "Make sure template context variables are rendered with proper values"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'another_project', 'project_dir']
        testproject_dir = os.path.join(test_dir, 'project_dir')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        test_manage_py = os.path.join(testproject_dir, 'manage.py')
        with open(test_manage_py, 'r') as fp:
            content = force_text(fp.read())
            self.assertIn("project_name = 'another_project'", content)
            self.assertIn("project_directory = '%s'" % testproject_dir, content)

    def test_no_escaping_of_project_variables(self):
        "Make sure template context variables are not html escaped"
        # We're using a custom command so we need the alternate settings
        self.write_settings('alternate_settings.py')
        self.addCleanup(self.remove_settings, 'alternate_settings.py')
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['custom_startproject', '--template', template_path, 'another_project', 'project_dir', '--extra', '<&>', '--settings=alternate_settings']
        testproject_dir = os.path.join(test_dir, 'project_dir')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        test_manage_py = os.path.join(testproject_dir, 'additional_dir', 'extra.py')
        with open(test_manage_py, 'r') as fp:
            content = fp.read()
            self.assertIn("<&>", content)

    def test_custom_project_destination_missing(self):
        """
        Make sure an exception is raised when the provided
        destination directory doesn't exist
        """
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'yet_another_project', 'project_dir2']
        testproject_dir = os.path.join(test_dir, 'project_dir2')
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Destination directory '%s' does not exist, please create it first." % testproject_dir)
        self.assertFalse(os.path.exists(testproject_dir))

    def test_custom_project_template_with_non_ascii_templates(self):
        "Ticket 18091: Make sure the startproject management command is able to render templates with non-ASCII content"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, '--extension=txt', 'customtestproject']
        testproject_dir = os.path.join(test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        path = os.path.join(testproject_dir, 'ticket-18091-non-ascii-template.txt')
        with codecs.open(path, 'r', 'utf-8') as f:
            self.assertEqual(f.read(),
                'Some non-ASCII text for testing ticket #18091:\nüäö €\n')


class DiffSettings(AdminScriptTestCase):
    """Tests for diffsettings management command."""

    def test_basic(self):
        """Runs without error and emits settings diff."""
        self.write_settings('settings_to_diff.py', sdict={'FOO': '"bar"'})
        self.addCleanup(self.remove_settings, 'settings_to_diff.py')
        args = ['diffsettings', '--settings=settings_to_diff']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "FOO = 'bar'  ###")

    def test_all(self):
        """The all option also shows settings with the default value."""
        self.write_settings('settings_to_diff.py', sdict={'STATIC_URL': 'None'})
        self.addCleanup(self.remove_settings, 'settings_to_diff.py')
        args = ['diffsettings', '--settings=settings_to_diff', '--all']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "### STATIC_URL = None")