Fixed #12672 -- Added the ability to configure which applications are available on which database.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@12290 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/core/management/commands/loaddata.py

@@ -8,15 +8,10 @@ from django.conf import settings
 from django.core import serializers
 from django.core.management.base import BaseCommand
 from django.core.management.color import no_style
-from django.db import connections, transaction, DEFAULT_DB_ALIAS
+from django.db import connections, router, transaction, DEFAULT_DB_ALIAS
 from django.db.models import get_apps
 from django.utils.itercompat import product
 
-try:
-    set
-except NameError:
-    from sets import Set as set   # Python 2.3 fallback
-
 try:
     import bz2
     has_bz2 = True
@@ -31,13 +26,10 @@ class Command(BaseCommand):
         make_option('--database', action='store', dest='database',
             default=DEFAULT_DB_ALIAS, help='Nominates a specific database to load '
                 'fixtures into. Defaults to the "default" database.'),
-        make_option('-e', '--exclude', dest='exclude',action='append', default=[],
-            help='App to exclude (use multiple --exclude to exclude multiple apps).'),
     )
 
     def handle(self, *fixture_labels, **options):
         using = options.get('database', DEFAULT_DB_ALIAS)
-        excluded_apps = options.get('exclude', [])
 
         connection = connections[using]
         self.style = no_style()
@@ -171,7 +163,7 @@ class Command(BaseCommand):
                             try:
                                 objects = serializers.deserialize(format, fixture, using=using)
                                 for obj in objects:
-                                    if obj.object._meta.app_label not in excluded_apps:
+                                    if router.allow_syncdb(using, obj.object.__class__):
                                         objects_in_fixture += 1
                                         models.add(obj.object.__class__)
                                         obj.save(using=using)

django/core/management/commands/syncdb.py

@@ -5,7 +5,7 @@ from django.conf import settings
 from django.core.management.base import NoArgsCommand
 from django.core.management.color import no_style
 from django.core.management.sql import custom_sql_for_model, emit_post_sync_signal
-from django.db import connections, transaction, models, DEFAULT_DB_ALIAS
+from django.db import connections, router, transaction, models, DEFAULT_DB_ALIAS
 from django.utils.importlib import import_module
 
 
@@ -16,8 +16,6 @@ class Command(NoArgsCommand):
         make_option('--database', action='store', dest='database',
             default=DEFAULT_DB_ALIAS, help='Nominates a database to synchronize. '
                 'Defaults to the "default" database.'),
-        make_option('-e', '--exclude', dest='exclude',action='append', default=[],
-            help='App to exclude (use multiple --exclude to exclude multiple apps).'),
     )
     help = "Create the database tables for all apps in INSTALLED_APPS whose tables haven't already been created."
 
@@ -26,7 +24,6 @@ class Command(NoArgsCommand):
         verbosity = int(options.get('verbosity', 1))
         interactive = options.get('interactive')
         show_traceback = options.get('traceback', False)
-        exclude = options.get('exclude', [])
 
         self.style = no_style()
 
@@ -59,13 +56,16 @@ class Command(NoArgsCommand):
         created_models = set()
         pending_references = {}
 
-        excluded_apps = set(models.get_app(app_label) for app_label in exclude)
-        included_apps = set(app for app in models.get_apps() if app not in excluded_apps)
+        # Build the manifest of apps and models that are to be synchronized
+        manifest = dict(
+            (app.__name__.split('.')[-2],
+                [m for m in models.get_models(app, include_auto_created=True)
+                if router.allow_syncdb(db, m)])
+            for app in models.get_apps()
+        )
 
         # Create the tables for each model
-        for app in included_apps:
-            app_name = app.__name__.split('.')[-2]
-            model_list = models.get_models(app, include_auto_created=True)
+        for app_name, model_list in manifest.items():
             for model in model_list:
                 # Create the model's database table, if it doesn't already exist.
                 if verbosity >= 2:
@@ -101,9 +101,8 @@ class Command(NoArgsCommand):
 
         # Install custom SQL for the app (but only if this
         # is a model we've just created)
-        for app in included_apps:
-            app_name = app.__name__.split('.')[-2]
-            for model in models.get_models(app):
+        for app_name, model_list in manifest.items():
+            for model in model_list:
                 if model in created_models:
                     custom_sql = custom_sql_for_model(model, self.style, connection)
                     if custom_sql:
@@ -126,9 +125,8 @@ class Command(NoArgsCommand):
                             print "No custom SQL for %s.%s model" % (app_name, model._meta.object_name)
 
         # Install SQL indicies for all newly created models
-        for app in included_apps:
-            app_name = app.__name__.split('.')[-2]
-            for model in models.get_models(app):
+        for app_name, model_list in manifest.items():
+            for model in model_list:
                 if model in created_models:
                     index_sql = connection.creation.sql_indexes_for_model(model, self.style)
                     if index_sql:
@@ -145,4 +143,4 @@ class Command(NoArgsCommand):
                             transaction.commit_unless_managed(using=db)
 
         from django.core.management import call_command
-        call_command('loaddata', 'initial_data', verbosity=verbosity, exclude=exclude, database=db)
+        call_command('loaddata', 'initial_data', verbosity=verbosity, database=db)

django/db/utils.py

@@ -121,3 +121,10 @@ class ConnectionRouter(object):
             if allow is not None:
                 return allow
         return obj1._state.db == obj2._state.db
+
+    def allow_syncdb(self, db, model):
+        for router in self.routers:
+            allow = router.allow_syncdb(db, model)
+            if allow is not None:
+                return allow
+        return True

django/test/simple.py

@@ -284,17 +284,11 @@ class DjangoTestSuiteRunner(object):
         Returns the number of tests that failed.
         """
         self.setup_test_environment()
-
         suite = self.build_suite(test_labels, extra_tests)
-
         old_config = self.setup_databases()
-
         result = self.run_suite(suite)
-
         self.teardown_databases(old_config)
-
         self.teardown_test_environment()
-
         return self.suite_result(result)
 
 def run_tests(test_labels, verbosity=1, interactive=True, failfast=False, extra_tests=None):

docs/ref/django-admin.txt

@@ -423,25 +423,6 @@ define the fixture ``mydata.master.json`` or
 have specified that you want to load data onto the ``master``
 database.
 
-Excluding applications from loading
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. versionadded:: 1.2
-
-The :djadminopt:`--exclude` option may be provided to prevent specific
-applications from being loaded.
-
-For example, if you wanted to exclude models from ``django.contrib.auth``
-from being loaded into your database, you would call::
-
-    django-admin.py loaddata mydata.json --exclude auth
-
-This will look for for a JSON fixture called ``mydata`` in all the
-usual locations - including the ``fixtures`` directory of the
-``django.contrib.auth`` application. However, any fixture object that
-identifies itself as belonging to the ``auth`` application (e.g.,
-instance of ``auth.User``) would be ignored by loaddata.
-
 makemessages
 ------------
 

docs/topics/db/multi-db.txt

@@ -66,13 +66,9 @@ all databases in our example, you would need to call::
     $ ./manage.py syncdb --database=users
 
 If you don't want every application to be synchronized onto a
-particular database. you can specify the :djadminopt:`--exclude`
-argument to :djadmin:`syncdb`. The :djadminopt:`--exclude` option lets
-you prevent a specific application or applications from being
-synchronized. For example, if you don't want the ``sales`` application
-to be in the ``users`` database, you could run::
-
-    $ ./manage.py syncdb --database=users --exclude=sales
+particular database, you can define a :ref:`database
+router<topics-db-multi-db-routing>` that implements a policy
+constraining the availability of particular models.
 
 Alternatively, if you want fine-grained control of synchronization,
 you can pipe all or part of the output of :djadmin:`sqlall` for a
@@ -103,7 +99,7 @@ routing scheme.
 Database routers
 ----------------
 
-A database Router is a class that provides three methods:
+A database Router is a class that provides four methods:
 
 .. method:: db_for_read(model, **hints)
 
@@ -137,6 +133,14 @@ A database Router is a class that provides three methods:
     used by foreign key and many to many operations to determine if a
     relation should be allowed between two objects.
 
+.. method:: allow_syncdb(db, model)
+
+    Determine if the ``model`` should be synchronized onto the
+    database with alias ``db``. Return True if the model should be
+    synchronized, False if it should not be synchronized, or None if
+    the router has no opinion. This method can be used to determine
+    the availability of a model on a given database.
+
 .. _topics-db-multi-db-hints:
 
 Hints
@@ -221,6 +225,13 @@ master/slave relationship between the databases 'master', 'slave1' and
                 return True
             return None
 
+        def allow_syncdb(self, db, model):
+            "Make sure the auth app only appears on the 'credentials' db"
+            if db == 'credentials':
+                return model._meta.app_label == 'auth'
+            elif model._meta.app_label == 'auth':
+                return False
+            return None
 
      class MasterSlaveRouter(object):
         """A router that sets up a simple master/slave configuration"""
@@ -240,11 +251,26 @@ master/slave relationship between the databases 'master', 'slave1' and
                 return True
             return None
 
+        def allow_syncdb(self, db, model):
+            "Explicitly put all models on all databases."
+            return True
+
 Then, in your settings file, add the following (substituting ``path.to.`` with
 the actual python path to the module where you define the routers)::
 
     DATABASE_ROUTERS = ['path.to.AuthRouter', 'path.to.MasterSlaveRouter']
 
+The order in which routers are processed is significant. Routers will
+be queried in the order the are listed in the
+:setting:`DATABASE_ROUTERS` setting . In this example, the
+``AuthRouter`` is processed before the ``MasterSlaveRouter``, and as a
+result, decisions concerning the models in ``auth`` are processed
+before any other decision is made. If the :setting:`DATABASE_ROUTERS`
+setting listed the two routers in the other order,
+``MasterSlaveRouter.allow_syncdb()`` would be processed first. The
+catch-all nature of the MasterSlaveRouter implementation would mean
+that all models would be available on all databases.
+
 With this setup installed, lets run some Django code::
 
     >>> # This retrieval will be performed on the 'credentials' database
@@ -270,6 +296,7 @@ With this setup installed, lets run some Django code::
     >>> # ... but if we re-retrieve the object, it will come back on a slave
     >>> mh = Book.objects.get(title='Mostly Harmless')
 
+
 Manually selecting a database
 =============================
 

tests/modeltests/fixtures/models.py

@@ -289,14 +289,6 @@ Multiple fixtures named 'fixture5' in '...fixtures'. Aborting.
 
 >>> management.call_command('flush', verbosity=0, interactive=False)
 
-# Try to load fixture 1, but this time, exclude the 'fixtures' app.
->>> management.call_command('loaddata', 'fixture1', verbosity=0, exclude='fixtures')
->>> Article.objects.all()
-[<Article: Python program becomes self aware>]
-
->>> Category.objects.all()
-[]
-
 # Load back in fixture 1, we need the articles from it
 >>> management.call_command('loaddata', 'fixture1', verbosity=0)
 

tests/regressiontests/multiple_database/tests.py

@@ -655,6 +655,25 @@ class TestRouter(object):
     def allow_relation(self, obj1, obj2, **hints):
         return obj1._state.db in ('default', 'other') and obj2._state.db in ('default', 'other')
 
+    def allow_syncdb(self, db, model):
+        return True
+
+class AuthRouter(object):
+    # Another test router. This one doesn't do anything interesting
+    # other than validate syncdb behavior
+    def db_for_read(self, model, **hints):
+        return None
+    def db_for_write(self, model, **hints):
+        return None
+    def allow_relation(self, obj1, obj2, **hints):
+        return None
+    def allow_syncdb(self, db, model):
+        if db == 'other':
+            return model._meta.app_label == 'auth'
+        elif model._meta.app_label == 'auth':
+            return False
+        return None
+
 class RouterTestCase(TestCase):
     multi_db = True
 
@@ -677,6 +696,35 @@ class RouterTestCase(TestCase):
         self.assertEquals(Book.objects.db_manager('default').db, 'default')
         self.assertEquals(Book.objects.db_manager('default').all().db, 'default')
 
+    def test_syncdb_selection(self):
+        "Synchronization behaviour is predicatable"
+
+        self.assertTrue(router.allow_syncdb('default', User))
+        self.assertTrue(router.allow_syncdb('default', Book))
+
+        self.assertTrue(router.allow_syncdb('other', User))
+        self.assertTrue(router.allow_syncdb('other', Book))
+
+        # Add the auth router to the chain.
+        # TestRouter is a universal synchronizer, so it should have no effect.
+        router.routers = [TestRouter(), AuthRouter()]
+
+        self.assertTrue(router.allow_syncdb('default', User))
+        self.assertTrue(router.allow_syncdb('default', Book))
+
+        self.assertTrue(router.allow_syncdb('other', User))
+        self.assertTrue(router.allow_syncdb('other', Book))
+
+        # Now check what happens if the router order is the other way around
+        router.routers = [AuthRouter(), TestRouter()]
+
+        self.assertFalse(router.allow_syncdb('default', User))
+        self.assertTrue(router.allow_syncdb('default', Book))
+
+        self.assertTrue(router.allow_syncdb('other', User))
+        self.assertFalse(router.allow_syncdb('other', Book))
+
+
     def test_database_routing(self):
         marty = Person.objects.using('default').create(name="Marty Alchin")
         pro = Book.objects.using('default').create(title="Pro Django",
@@ -1046,6 +1094,7 @@ class UserProfileTestCase(TestCase):
         self.assertEquals(alice.get_profile().flavor, 'chocolate')
         self.assertEquals(bob.get_profile().flavor, 'crunchy frog')
 
+
 class FixtureTestCase(TestCase):
     multi_db = True
     fixtures = ['multidb-common', 'multidb']