runtests.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484
  1. #!/usr/bin/env python
  2. import argparse
  3. import atexit
  4. import copy
  5. import os
  6. import shutil
  7. import subprocess
  8. import sys
  9. import tempfile
  10. import warnings
  11. import django
  12. from django.apps import apps
  13. from django.conf import settings
  14. from django.db import connection, connections
  15. from django.test import TestCase, TransactionTestCase
  16. from django.test.runner import default_test_processes
  17. from django.test.selenium import SeleniumTestCaseBase
  18. from django.test.utils import get_runner
  19. from django.utils import six
  20. from django.utils._os import upath
  21. from django.utils.deprecation import (
  22. RemovedInDjango20Warning, RemovedInDjango21Warning,
  23. )
  24. from django.utils.log import DEFAULT_LOGGING
  25. # Make deprecation warnings errors to ensure no usage of deprecated features.
  26. warnings.simplefilter("error", RemovedInDjango20Warning)
  27. warnings.simplefilter("error", RemovedInDjango21Warning)
  28. # Make runtime warning errors to ensure no usage of error prone patterns.
  29. warnings.simplefilter("error", RuntimeWarning)
  30. # Ignore known warnings in test dependencies.
  31. warnings.filterwarnings("ignore", "'U' mode is deprecated", DeprecationWarning, module='docutils.io')
  32. RUNTESTS_DIR = os.path.abspath(os.path.dirname(upath(__file__)))
  33. TEMPLATE_DIR = os.path.join(RUNTESTS_DIR, 'templates')
  34. # Create a specific subdirectory for the duration of the test suite.
  35. TMPDIR = tempfile.mkdtemp(prefix='django_')
  36. # Set the TMPDIR environment variable in addition to tempfile.tempdir
  37. # so that children processes inherit it.
  38. tempfile.tempdir = os.environ['TMPDIR'] = TMPDIR
  39. # Removing the temporary TMPDIR. Ensure we pass in unicode so that it will
  40. # successfully remove temp trees containing non-ASCII filenames on Windows.
  41. # (We're assuming the temp dir name itself only contains ASCII characters.)
  42. atexit.register(shutil.rmtree, six.text_type(TMPDIR))
  43. SUBDIRS_TO_SKIP = [
  44. 'data',
  45. 'import_error_package',
  46. 'test_discovery_sample',
  47. 'test_discovery_sample2',
  48. ]
  49. ALWAYS_INSTALLED_APPS = [
  50. 'django.contrib.contenttypes',
  51. 'django.contrib.auth',
  52. 'django.contrib.sites',
  53. 'django.contrib.sessions',
  54. 'django.contrib.messages',
  55. 'django.contrib.admin.apps.SimpleAdminConfig',
  56. 'django.contrib.staticfiles',
  57. ]
  58. ALWAYS_MIDDLEWARE = [
  59. 'django.contrib.sessions.middleware.SessionMiddleware',
  60. 'django.middleware.common.CommonMiddleware',
  61. 'django.middleware.csrf.CsrfViewMiddleware',
  62. 'django.contrib.auth.middleware.AuthenticationMiddleware',
  63. 'django.contrib.messages.middleware.MessageMiddleware',
  64. ]
  65. # Need to add the associated contrib app to INSTALLED_APPS in some cases to
  66. # avoid "RuntimeError: Model class X doesn't declare an explicit app_label
  67. # and isn't in an application in INSTALLED_APPS."
  68. CONTRIB_TESTS_TO_APPS = {
  69. 'flatpages_tests': 'django.contrib.flatpages',
  70. 'redirects_tests': 'django.contrib.redirects',
  71. }
  72. def get_test_modules():
  73. modules = []
  74. discovery_paths = [
  75. (None, RUNTESTS_DIR),
  76. # GIS tests are in nested apps
  77. ('gis_tests', os.path.join(RUNTESTS_DIR, 'gis_tests')),
  78. ]
  79. for modpath, dirpath in discovery_paths:
  80. for f in os.listdir(dirpath):
  81. if ('.' in f or
  82. os.path.basename(f) in SUBDIRS_TO_SKIP or
  83. os.path.isfile(f) or
  84. not os.path.exists(os.path.join(dirpath, f, '__init__.py'))):
  85. continue
  86. modules.append((modpath, f))
  87. return modules
  88. def get_installed():
  89. return [app_config.name for app_config in apps.get_app_configs()]
  90. def setup(verbosity, test_labels, parallel):
  91. if verbosity >= 1:
  92. msg = "Testing against Django installed in '%s'" % os.path.dirname(django.__file__)
  93. max_parallel = default_test_processes() if parallel == 0 else parallel
  94. if max_parallel > 1:
  95. msg += " with up to %d processes" % max_parallel
  96. print(msg)
  97. # Force declaring available_apps in TransactionTestCase for faster tests.
  98. def no_available_apps(self):
  99. raise Exception("Please define available_apps in TransactionTestCase "
  100. "and its subclasses.")
  101. TransactionTestCase.available_apps = property(no_available_apps)
  102. TestCase.available_apps = None
  103. state = {
  104. 'INSTALLED_APPS': settings.INSTALLED_APPS,
  105. 'ROOT_URLCONF': getattr(settings, "ROOT_URLCONF", ""),
  106. 'TEMPLATES': settings.TEMPLATES,
  107. 'LANGUAGE_CODE': settings.LANGUAGE_CODE,
  108. 'STATIC_URL': settings.STATIC_URL,
  109. 'STATIC_ROOT': settings.STATIC_ROOT,
  110. 'MIDDLEWARE': settings.MIDDLEWARE,
  111. }
  112. # Redirect some settings for the duration of these tests.
  113. settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS
  114. settings.ROOT_URLCONF = 'urls'
  115. settings.STATIC_URL = '/static/'
  116. settings.STATIC_ROOT = os.path.join(TMPDIR, 'static')
  117. settings.TEMPLATES = [{
  118. 'BACKEND': 'django.template.backends.django.DjangoTemplates',
  119. 'DIRS': [TEMPLATE_DIR],
  120. 'APP_DIRS': True,
  121. 'OPTIONS': {
  122. 'context_processors': [
  123. 'django.template.context_processors.debug',
  124. 'django.template.context_processors.request',
  125. 'django.contrib.auth.context_processors.auth',
  126. 'django.contrib.messages.context_processors.messages',
  127. ],
  128. },
  129. }]
  130. settings.LANGUAGE_CODE = 'en'
  131. settings.SITE_ID = 1
  132. settings.MIDDLEWARE = ALWAYS_MIDDLEWARE
  133. settings.MIGRATION_MODULES = {
  134. # This lets us skip creating migrations for the test models as many of
  135. # them depend on one of the following contrib applications.
  136. 'auth': None,
  137. 'contenttypes': None,
  138. 'sessions': None,
  139. }
  140. log_config = copy.deepcopy(DEFAULT_LOGGING)
  141. # Filter out non-error logging so we don't have to capture it in lots of
  142. # tests.
  143. log_config['loggers']['django']['level'] = 'ERROR'
  144. settings.LOGGING = log_config
  145. settings.SILENCED_SYSTEM_CHECKS = [
  146. 'fields.W342', # ForeignKey(unique=True) -> OneToOneField
  147. 'fields.W901', # CommaSeparatedIntegerField deprecated
  148. ]
  149. # Load all the ALWAYS_INSTALLED_APPS.
  150. django.setup()
  151. # Load all the test model apps.
  152. test_modules = get_test_modules()
  153. # Reduce given test labels to just the app module path
  154. test_labels_set = set()
  155. for label in test_labels:
  156. bits = label.split('.')[:1]
  157. test_labels_set.add('.'.join(bits))
  158. installed_app_names = set(get_installed())
  159. for modpath, module_name in test_modules:
  160. if modpath:
  161. module_label = '.'.join([modpath, module_name])
  162. else:
  163. module_label = module_name
  164. # if the module (or an ancestor) was named on the command line, or
  165. # no modules were named (i.e., run all), import
  166. # this module and add it to INSTALLED_APPS.
  167. if not test_labels:
  168. module_found_in_labels = True
  169. else:
  170. module_found_in_labels = any(
  171. # exact match or ancestor match
  172. module_label == label or module_label.startswith(label + '.')
  173. for label in test_labels_set)
  174. if module_name in CONTRIB_TESTS_TO_APPS and module_found_in_labels:
  175. settings.INSTALLED_APPS.append(CONTRIB_TESTS_TO_APPS[module_name])
  176. if module_found_in_labels and module_label not in installed_app_names:
  177. if verbosity >= 2:
  178. print("Importing application %s" % module_name)
  179. settings.INSTALLED_APPS.append(module_label)
  180. # Add contrib.gis to INSTALLED_APPS if needed (rather than requiring
  181. # @override_settings(INSTALLED_APPS=...) on all test cases.
  182. gis = 'django.contrib.gis'
  183. if connection.features.gis_enabled and gis not in settings.INSTALLED_APPS:
  184. if verbosity >= 2:
  185. print("Importing application %s" % gis)
  186. settings.INSTALLED_APPS.append(gis)
  187. apps.set_installed_apps(settings.INSTALLED_APPS)
  188. return state
  189. def teardown(state):
  190. # Restore the old settings.
  191. for key, value in state.items():
  192. setattr(settings, key, value)
  193. def actual_test_processes(parallel):
  194. if parallel == 0:
  195. # This doesn't work before django.setup() on some databases.
  196. if all(conn.features.can_clone_databases for conn in connections.all()):
  197. return default_test_processes()
  198. else:
  199. return 1
  200. else:
  201. return parallel
  202. class ActionSelenium(argparse.Action):
  203. """
  204. Validate the comma-separated list of requested browsers.
  205. """
  206. def __call__(self, parser, namespace, values, option_string=None):
  207. browsers = values.split(',')
  208. for browser in browsers:
  209. try:
  210. SeleniumTestCaseBase.import_webdriver(browser)
  211. except ImportError:
  212. raise argparse.ArgumentError(self, "Selenium browser specification '%s' is not valid." % browser)
  213. setattr(namespace, self.dest, browsers)
  214. def django_tests(verbosity, interactive, failfast, keepdb, reverse,
  215. test_labels, debug_sql, parallel, tags, exclude_tags):
  216. state = setup(verbosity, test_labels, parallel)
  217. extra_tests = []
  218. # Run the test suite, including the extra validation tests.
  219. if not hasattr(settings, 'TEST_RUNNER'):
  220. settings.TEST_RUNNER = 'django.test.runner.DiscoverRunner'
  221. TestRunner = get_runner(settings)
  222. test_runner = TestRunner(
  223. verbosity=verbosity,
  224. interactive=interactive,
  225. failfast=failfast,
  226. keepdb=keepdb,
  227. reverse=reverse,
  228. debug_sql=debug_sql,
  229. parallel=actual_test_processes(parallel),
  230. tags=tags,
  231. exclude_tags=exclude_tags,
  232. )
  233. failures = test_runner.run_tests(
  234. test_labels or get_installed(),
  235. extra_tests=extra_tests,
  236. )
  237. teardown(state)
  238. return failures
  239. def get_subprocess_args(options):
  240. subprocess_args = [
  241. sys.executable, upath(__file__), '--settings=%s' % options.settings
  242. ]
  243. if options.failfast:
  244. subprocess_args.append('--failfast')
  245. if options.verbosity:
  246. subprocess_args.append('--verbosity=%s' % options.verbosity)
  247. if not options.interactive:
  248. subprocess_args.append('--noinput')
  249. if options.tags:
  250. subprocess_args.append('--tag=%s' % options.tags)
  251. if options.exclude_tags:
  252. subprocess_args.append('--exclude_tag=%s' % options.exclude_tags)
  253. return subprocess_args
  254. def bisect_tests(bisection_label, options, test_labels, parallel):
  255. state = setup(options.verbosity, test_labels, parallel)
  256. test_labels = test_labels or get_installed()
  257. print('***** Bisecting test suite: %s' % ' '.join(test_labels))
  258. # Make sure the bisection point isn't in the test list
  259. # Also remove tests that need to be run in specific combinations
  260. for label in [bisection_label, 'model_inheritance_same_model_name']:
  261. try:
  262. test_labels.remove(label)
  263. except ValueError:
  264. pass
  265. subprocess_args = get_subprocess_args(options)
  266. iteration = 1
  267. while len(test_labels) > 1:
  268. midpoint = len(test_labels) // 2
  269. test_labels_a = test_labels[:midpoint] + [bisection_label]
  270. test_labels_b = test_labels[midpoint:] + [bisection_label]
  271. print('***** Pass %da: Running the first half of the test suite' % iteration)
  272. print('***** Test labels: %s' % ' '.join(test_labels_a))
  273. failures_a = subprocess.call(subprocess_args + test_labels_a)
  274. print('***** Pass %db: Running the second half of the test suite' % iteration)
  275. print('***** Test labels: %s' % ' '.join(test_labels_b))
  276. print('')
  277. failures_b = subprocess.call(subprocess_args + test_labels_b)
  278. if failures_a and not failures_b:
  279. print("***** Problem found in first half. Bisecting again...")
  280. iteration += 1
  281. test_labels = test_labels_a[:-1]
  282. elif failures_b and not failures_a:
  283. print("***** Problem found in second half. Bisecting again...")
  284. iteration += 1
  285. test_labels = test_labels_b[:-1]
  286. elif failures_a and failures_b:
  287. print("***** Multiple sources of failure found")
  288. break
  289. else:
  290. print("***** No source of failure found... try pair execution (--pair)")
  291. break
  292. if len(test_labels) == 1:
  293. print("***** Source of error: %s" % test_labels[0])
  294. teardown(state)
  295. def paired_tests(paired_test, options, test_labels, parallel):
  296. state = setup(options.verbosity, test_labels, parallel)
  297. test_labels = test_labels or get_installed()
  298. print('***** Trying paired execution')
  299. # Make sure the constant member of the pair isn't in the test list
  300. # Also remove tests that need to be run in specific combinations
  301. for label in [paired_test, 'model_inheritance_same_model_name']:
  302. try:
  303. test_labels.remove(label)
  304. except ValueError:
  305. pass
  306. subprocess_args = get_subprocess_args(options)
  307. for i, label in enumerate(test_labels):
  308. print('***** %d of %d: Check test pairing with %s' % (
  309. i + 1, len(test_labels), label))
  310. failures = subprocess.call(subprocess_args + [label, paired_test])
  311. if failures:
  312. print('***** Found problem pair with %s' % label)
  313. return
  314. print('***** No problem pair found')
  315. teardown(state)
  316. if __name__ == "__main__":
  317. parser = argparse.ArgumentParser(description="Run the Django test suite.")
  318. parser.add_argument(
  319. 'modules', nargs='*', metavar='module',
  320. help='Optional path(s) to test modules; e.g. "i18n" or '
  321. '"i18n.tests.TranslationTests.test_lazy_objects".',
  322. )
  323. parser.add_argument(
  324. '-v', '--verbosity', default=1, type=int, choices=[0, 1, 2, 3],
  325. help='Verbosity level; 0=minimal output, 1=normal output, 2=all output',
  326. )
  327. parser.add_argument(
  328. '--noinput', action='store_false', dest='interactive', default=True,
  329. help='Tells Django to NOT prompt the user for input of any kind.',
  330. )
  331. parser.add_argument(
  332. '--failfast', action='store_true', dest='failfast', default=False,
  333. help='Tells Django to stop running the test suite after first failed test.',
  334. )
  335. parser.add_argument(
  336. '-k', '--keepdb', action='store_true', dest='keepdb', default=False,
  337. help='Tells Django to preserve the test database between runs.',
  338. )
  339. parser.add_argument(
  340. '--settings',
  341. help='Python path to settings module, e.g. "myproject.settings". If '
  342. 'this isn\'t provided, either the DJANGO_SETTINGS_MODULE '
  343. 'environment variable or "test_sqlite" will be used.',
  344. )
  345. parser.add_argument(
  346. '--bisect',
  347. help='Bisect the test suite to discover a test that causes a test '
  348. 'failure when combined with the named test.',
  349. )
  350. parser.add_argument(
  351. '--pair',
  352. help='Run the test suite in pairs with the named test to find problem pairs.',
  353. )
  354. parser.add_argument(
  355. '--reverse', action='store_true', default=False,
  356. help='Sort test suites and test cases in opposite order to debug '
  357. 'test side effects not apparent with normal execution lineup.',
  358. )
  359. parser.add_argument(
  360. '--selenium', dest='selenium', action=ActionSelenium, metavar='BROWSERS',
  361. help='A comma-separated list of browsers to run the Selenium tests against.',
  362. )
  363. parser.add_argument(
  364. '--debug-sql', action='store_true', dest='debug_sql', default=False,
  365. help='Turn on the SQL query logger within tests.',
  366. )
  367. parser.add_argument(
  368. '--parallel', dest='parallel', nargs='?', default=0, type=int,
  369. const=default_test_processes(), metavar='N',
  370. help='Run tests using up to N parallel processes.',
  371. )
  372. parser.add_argument(
  373. '--tag', dest='tags', action='append',
  374. help='Run only tests with the specified tags. Can be used multiple times.',
  375. )
  376. parser.add_argument(
  377. '--exclude-tag', dest='exclude_tags', action='append',
  378. help='Do not run tests with the specified tag. Can be used multiple times.',
  379. )
  380. options = parser.parse_args()
  381. # mock is a required dependency
  382. try:
  383. from django.test import mock # NOQA
  384. except ImportError:
  385. print(
  386. "Please install test dependencies first: \n"
  387. "$ pip install -r requirements/py%s.txt" % sys.version_info.major
  388. )
  389. sys.exit(1)
  390. # Allow including a trailing slash on app_labels for tab completion convenience
  391. options.modules = [os.path.normpath(labels) for labels in options.modules]
  392. if options.settings:
  393. os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
  394. else:
  395. if "DJANGO_SETTINGS_MODULE" not in os.environ:
  396. os.environ['DJANGO_SETTINGS_MODULE'] = 'test_sqlite'
  397. options.settings = os.environ['DJANGO_SETTINGS_MODULE']
  398. if options.selenium:
  399. if not options.tags:
  400. options.tags = ['selenium']
  401. elif 'selenium' not in options.tags:
  402. options.tags.append('selenium')
  403. SeleniumTestCaseBase.browsers = options.selenium
  404. if options.bisect:
  405. bisect_tests(options.bisect, options, options.modules, options.parallel)
  406. elif options.pair:
  407. paired_tests(options.pair, options, options.modules, options.parallel)
  408. else:
  409. failures = django_tests(
  410. options.verbosity, options.interactive, options.failfast,
  411. options.keepdb, options.reverse, options.modules,
  412. options.debug_sql, options.parallel, options.tags,
  413. options.exclude_tags,
  414. )
  415. if failures:
  416. sys.exit(1)