Fixed #13252 -- Added ability to serialize with natural primary keys.

Added ``--natural-foreign`` and ``--natural-primary`` options and
deprecated the ``--natural`` option to the ``dumpdata`` management
command.

Added ``use_natural_foreign_keys`` and ``use_natural_primary_keys``
arguments and deprecated the ``use_natural_keys`` argument to
``django.core.serializers.Serializer.serialize()``.

Thanks SmileyChris for the suggestion.

django/core/management/commands/dumpdata.py

@@ -1,3 +1,5 @@
+import warnings
+
 from collections import OrderedDict
 from optparse import make_option
 
@@ -20,6 +22,10 @@ class Command(BaseCommand):
             help='An appname or appname.ModelName to exclude (use multiple --exclude to exclude multiple apps/models).'),
         make_option('-n', '--natural', action='store_true', dest='use_natural_keys', default=False,
             help='Use natural keys if they are available.'),
+        make_option('--natural-foreign', action='store_true', dest='use_natural_foreign_keys', default=False,
+            help='Use natural foreign keys if they are available.'),
+        make_option('--natural-primary', action='store_true', dest='use_natural_primary_keys', default=False,
+            help='Use natural primary keys if they are available.'),
         make_option('-a', '--all', action='store_true', dest='use_base_manager', default=False,
             help="Use Django's base manager to dump all models stored in the database, including those that would otherwise be filtered or modified by a custom manager."),
         make_option('--pks', dest='primary_keys', help="Only dump objects with "
@@ -40,6 +46,11 @@ class Command(BaseCommand):
         excludes = options.get('exclude')
         show_traceback = options.get('traceback')
         use_natural_keys = options.get('use_natural_keys')
+        if use_natural_keys:
+            warnings.warn("``--natural`` is deprecated; use ``--natural-foreign`` instead.",
+                PendingDeprecationWarning)
+        use_natural_foreign_keys = options.get('use_natural_foreign_keys') or use_natural_keys
+        use_natural_primary_keys = options.get('use_natural_primary_keys')
         use_base_manager = options.get('use_base_manager')
         pks = options.get('primary_keys')
 
@@ -133,7 +144,9 @@ class Command(BaseCommand):
         try:
             self.stdout.ending = None
             serializers.serialize(format, get_objects(), indent=indent,
-                    use_natural_keys=use_natural_keys, stream=self.stdout)
+                    use_natural_foreign_keys=use_natural_foreign_keys,
+                    use_natural_primary_keys=use_natural_primary_keys,
+                    stream=self.stdout)
         except Exception as e:
             if show_traceback:
                 raise

django/core/serializers/base.py

@@ -1,6 +1,7 @@
 """
 Module for abstract serializer/unserializer base classes.
 """
+import warnings
 
 from django.db import models
 from django.utils import six
@@ -35,6 +36,11 @@ class Serializer(object):
         self.stream = options.pop("stream", six.StringIO())
         self.selected_fields = options.pop("fields", None)
         self.use_natural_keys = options.pop("use_natural_keys", False)
+        if self.use_natural_keys:
+            warnings.warn("``use_natural_keys`` is deprecated; use ``use_natural_foreign_keys`` instead.",
+                PendingDeprecationWarning)
+        self.use_natural_foreign_keys = options.pop('use_natural_foreign_keys', False) or self.use_natural_keys
+        self.use_natural_primary_keys = options.pop('use_natural_primary_keys', False)
 
         self.start_serialization()
         self.first = True
@@ -169,3 +175,20 @@ class DeserializedObject(object):
         # prevent a second (possibly accidental) call to save() from saving
         # the m2m data twice.
         self.m2m_data = None
+
+def build_instance(Model, data, db):
+    """
+    Build a model instance.
+
+    If the model instance doesn't have a primary key and the model supports
+    natural keys, try to retrieve it from the database.
+    """
+    obj = Model(**data)
+    if (obj.pk is None and hasattr(Model, 'natural_key') and
+            hasattr(Model._default_manager, 'get_by_natural_key')):
+        natural_key = obj.natural_key()
+        try:
+            obj.pk = Model._default_manager.db_manager(db).get_by_natural_key(*natural_key).pk
+        except Model.DoesNotExist:
+            pass
+    return obj

django/core/serializers/python.py

@@ -34,11 +34,14 @@ class Serializer(base.Serializer):
         self._current = None
 
     def get_dump_object(self, obj):
-        return {
-            "pk": smart_text(obj._get_pk_val(), strings_only=True),
+        data = {
             "model": smart_text(obj._meta),
-            "fields": self._current
+            "fields": self._current,
         }
+        if not self.use_natural_primary_keys or not hasattr(obj, 'natural_key'):
+            data["pk"] = smart_text(obj._get_pk_val(), strings_only=True)
+
+        return data
 
     def handle_field(self, obj, field):
         value = field._get_val_from_obj(obj)
@@ -51,7 +54,7 @@ class Serializer(base.Serializer):
             self._current[field.name] = field.value_to_string(obj)
 
     def handle_fk_field(self, obj, field):
-        if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
+        if self.use_natural_foreign_keys and hasattr(field.rel.to, 'natural_key'):
             related = getattr(obj, field.name)
             if related:
                 value = related.natural_key()
@@ -63,7 +66,7 @@ class Serializer(base.Serializer):
 
     def handle_m2m_field(self, obj, field):
         if field.rel.through._meta.auto_created:
-            if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
+            if self.use_natural_foreign_keys and hasattr(field.rel.to, 'natural_key'):
                 m2m_value = lambda value: value.natural_key()
             else:
                 m2m_value = lambda value: smart_text(value._get_pk_val(), strings_only=True)
@@ -88,7 +91,9 @@ def Deserializer(object_list, **options):
     for d in object_list:
         # Look up the model and starting build a dict of data for it.
         Model = _get_model(d["model"])
-        data = {Model._meta.pk.attname: Model._meta.pk.to_python(d.get("pk", None))}
+        data = {}
+        if 'pk' in d:
+            data[Model._meta.pk.attname] = Model._meta.pk.to_python(d.get("pk", None))
         m2m_data = {}
         model_fields = Model._meta.get_all_field_names()
 
@@ -139,7 +144,8 @@ def Deserializer(object_list, **options):
             else:
                 data[field.name] = field.to_python(field_value)
 
-        yield base.DeserializedObject(Model(**data), m2m_data)
+        obj = base.build_instance(Model, data, db)
+        yield base.DeserializedObject(obj, m2m_data)
 
 def _get_model(model_identifier):
     """

django/core/serializers/xml_serializer.py

@@ -46,14 +46,11 @@ class Serializer(base.Serializer):
             raise base.SerializationError("Non-model object (%s) encountered during serialization" % type(obj))
 
         self.indent(1)
-        obj_pk = obj._get_pk_val()
-        if obj_pk is None:
-            attrs = {"model": smart_text(obj._meta),}
-        else:
-            attrs = {
-                "pk": smart_text(obj._get_pk_val()),
-                "model": smart_text(obj._meta),
-            }
+        attrs = {"model": smart_text(obj._meta)}
+        if not self.use_natural_primary_keys or not hasattr(obj, 'natural_key'):
+            obj_pk = obj._get_pk_val()
+            if obj_pk is not None:
+                attrs['pk'] = smart_text(obj_pk)
 
         self.xml.startElement("object", attrs)
 
@@ -91,7 +88,7 @@ class Serializer(base.Serializer):
         self._start_relational_field(field)
         related_att = getattr(obj, field.get_attname())
         if related_att is not None:
-            if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
+            if self.use_natural_foreign_keys and hasattr(field.rel.to, 'natural_key'):
                 related = getattr(obj, field.name)
                 # If related object has a natural key, use it
                 related = related.natural_key()
@@ -114,7 +111,7 @@ class Serializer(base.Serializer):
         """
         if field.rel.through._meta.auto_created:
             self._start_relational_field(field)
-            if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
+            if self.use_natural_foreign_keys and hasattr(field.rel.to, 'natural_key'):
                 # If the objects in the m2m have a natural key, use it
                 def handle_m2m(value):
                     natural = value.natural_key()
@@ -177,13 +174,10 @@ class Deserializer(base.Deserializer):
         Model = self._get_model_from_node(node, "model")
 
         # Start building a data dictionary from the object.
-        # If the node is missing the pk set it to None
-        if node.hasAttribute("pk"):
-            pk = node.getAttribute("pk")
-        else:
-            pk = None
-
-        data = {Model._meta.pk.attname : Model._meta.pk.to_python(pk)}
+        data = {}
+        if node.hasAttribute('pk'):
+            data[Model._meta.pk.attname] = Model._meta.pk.to_python(
+                                                    node.getAttribute('pk'))
 
         # Also start building a dict of m2m data (this is saved as
         # {m2m_accessor_attribute : [list_of_related_objects]})
@@ -217,8 +211,10 @@ class Deserializer(base.Deserializer):
                     value = field.to_python(getInnerText(field_node).strip())
                 data[field.name] = value
 
+        obj = base.build_instance(Model, data, self.db)
+
         # Return a DeserializedObject so that the m2m data has a place to live.
-        return base.DeserializedObject(Model(**data), m2m_data)
+        return base.DeserializedObject(obj, m2m_data)
 
     def _handle_fk_field_node(self, node, field):
         """

docs/internals/deprecation.txt

@@ -461,6 +461,12 @@ these changes.
   ``BaseMemcachedCache._get_memcache_timeout()`` method to
   ``get_backend_timeout()``.
 
+* The ``--natural`` and ``-n`` options for :djadmin:`dumpdata` will be removed.
+  Use :djadminopt:`--natural-foreign` instead.
+
+* The ``use_natural_keys`` argument for ``serializers.serialize()`` will be
+  removed. Use ``use_natural_foreign_keys`` instead.
+
 2.0
 ---
 

docs/ref/django-admin.txt

@@ -220,13 +220,34 @@ also mix application names and model names.
 The :djadminopt:`--database` option can be used to specify the database
 from which data will be dumped.
 
+.. django-admin-option:: --natural-foreign
+
+.. versionadded:: 1.7
+
+When this option is specified, Django will use the ``natural_key()`` model
+method to serialize any foreign key and many-to-many relationship to objects of
+the type that defines the method. If you are dumping ``contrib.auth``
+``Permission`` objects or ``contrib.contenttypes`` ``ContentType`` objects, you
+should probably be using this flag. See the :ref:`natural keys
+<topics-serialization-natural-keys>` documentation for more details on this
+and the next option.
+
+.. django-admin-option:: --natural-primary
+
+.. versionadded:: 1.7
+
+When this option is specified, Django will not provide the primary key in the
+serialized data of this object since it can be calculated during
+deserialization.
+
 .. django-admin-option:: --natural
 
+.. deprecated:: 1.7
+    Equivalent to the :djadminopt:`--natural-foreign` option; use that instead.
+
 Use :ref:`natural keys <topics-serialization-natural-keys>` to represent
 any foreign key and many-to-many relationship with a model that provides
-a natural key definition. If you are dumping ``contrib.auth`` ``Permission``
-objects or ``contrib.contenttypes`` ``ContentType`` objects, you should
-probably be using this flag.
+a natural key definition.
 
 .. versionadded:: 1.6
 

docs/releases/1.7.txt

@@ -294,6 +294,11 @@ Management Commands
 * The :djadminopt:`--no-color` option for ``django-admin.py`` allows you to
   disable the colorization of management command output.
 
+* The new :djadminopt:`--natural-foreign` and :djadminopt:`--natural-primary`
+  options for :djadmin:`dumpdata`, and the new ``use_natural_foreign_keys`` and
+  ``use_natural_primary_keys`` arguments for ``serializers.serialize()``, allow
+  the use of natural primary keys when serializing.
+
 Models
 ^^^^^^
 
@@ -588,3 +593,12 @@ The :class:`django.db.models.IPAddressField` and
 The ``BaseMemcachedCache._get_memcache_timeout()`` method has been renamed to
 ``get_backend_timeout()``. Despite being a private API, it will go through the
 normal deprecation.
+
+Natural key serialization options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``--natural`` and ``-n`` options for :djadmin:`dumpdata` have been
+deprecated. Use :djadminopt:`--natural-foreign` instead.
+
+Similarly, the ``use_natural_keys`` argument for ``serializers.serialize()``
+has been deprecated. Use ``use_natural_foreign_keys`` instead.

docs/topics/serialization.txt

@@ -404,6 +404,12 @@ into the primary key of an actual ``Person`` object.
     fields will be effectively unique, you can still use those fields
     as a natural key.
 
+.. versionadded:: 1.7
+
+Deserialization of objects with no primary key will always check whether the
+model's manager has a ``get_by_natural_key()`` method and if so, use it to
+populate the deserialized object's primary key.
+
 Serialization of natural keys
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -426,17 +432,39 @@ Firstly, you need to add another method -- this time to the model itself::
 
 That method should always return a natural key tuple -- in this
 example, ``(first name, last name)``. Then, when you call
-``serializers.serialize()``, you provide a ``use_natural_keys=True``
-argument::
+``serializers.serialize()``, you provide ``use_natural_foreign_keys=True``
+or ``use_natural_primary_keys=True`` arguments::
+
+    >>> serializers.serialize('json', [book1, book2], indent=2,
+    ...      use_natural_foreign_keys=True, use_natural_primary_keys=True)
+
+When ``use_natural_foreign_keys=True`` is specified, Django will use the
+``natural_key()`` method to serialize any foreign key reference to objects
+of the type that defines the method.
 
-    >>> serializers.serialize('json', [book1, book2], indent=2, use_natural_keys=True)
+When ``use_natural_primary_keys=True`` is specified, Django will not provide the
+primary key in the serialized data of this object since it can be calculated
+during deserialization::
+
+    ...
+    {
+        "model": "store.person",
+        "fields": {
+            "first_name": "Douglas",
+            "last_name": "Adams",
+            "birth_date": "1952-03-11",
+        }
+    }
+    ...
 
-When ``use_natural_keys=True`` is specified, Django will use the
-``natural_key()`` method to serialize any reference to objects of the
-type that defines the method.
+This can be useful when you need to load serialized data into an existing
+database and you cannot guarantee that the serialized primary key value is not
+already in use, and do not need to ensure that deserialized objects retain the
+same primary keys.
 
-If you are using :djadmin:`dumpdata` to generate serialized data, you
-use the :djadminopt:`--natural` command line flag to generate natural keys.
+If you are using :djadmin:`dumpdata` to generate serialized data, use the
+:djadminopt:`--natural-foreign` and :djadminopt:`--natural-primary` command
+line flags to generate natural keys.
 
 .. note::
 
@@ -450,6 +478,19 @@ use the :djadminopt:`--natural` command line flag to generate natural keys.
     natural keys during serialization, but *not* be able to load those
     key values, just don't define the ``get_by_natural_key()`` method.
 
+.. versionchanged:: 1.7
+
+Previously there was only a ``use_natural_keys`` argument for
+``serializers.serialize()`` and the `-n` or `--natural` command line flags.
+These have been deprecated in favor of the ``use_natural_foreign_keys`` and
+``use_natural_primary_keys`` arguments and the corresponding
+:djadminopt:`--natural-foreign` and :djadminopt:`--natural-primary` options
+for :djadmin:`dumpdata`.
+
+The original argument and command line flags remain for backwards
+compatibility and map to the new ``use_natural_foreign_keys`` argument and
+`--natural-foreign` command line flag. They'll be removed in Django 1.9.
+
 Dependencies during serialization
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -459,7 +500,7 @@ a "forward reference" with natural keys -- the data you're referencing
 must exist before you include a natural key reference to that data.
 
 To accommodate this limitation, calls to :djadmin:`dumpdata` that use
-the :djadminopt:`--natural` option will serialize any model with a
+the :djadminopt:`--natural-foreign` option will serialize any model with a
 ``natural_key()`` method before serializing standard primary key objects.
 
 However, this may not always be enough. If your natural key refers to

tests/fixtures_regress/tests.py

@@ -546,12 +546,13 @@ class NaturalKeyFixtureTests(TestCase):
             'fixtures_regress.store',
             verbosity=0,
             format='json',
-            use_natural_keys=True,
+            use_natural_foreign_keys=True,
+            use_natural_primary_keys=True,
             stdout=stdout,
         )
         self.assertJSONEqual(
             stdout.getvalue(),
-            """[{"pk": 2, "model": "fixtures_regress.store", "fields": {"main": null, "name": "Amazon"}}, {"pk": 3, "model": "fixtures_regress.store", "fields": {"main": null, "name": "Borders"}}, {"pk": 4, "model": "fixtures_regress.person", "fields": {"name": "Neal Stephenson"}}, {"pk": 1, "model": "fixtures_regress.book", "fields": {"stores": [["Amazon"], ["Borders"]], "name": "Cryptonomicon", "author": ["Neal Stephenson"]}}]"""
+            """[{"fields": {"main": null, "name": "Amazon"}, "model": "fixtures_regress.store"}, {"fields": {"main": null, "name": "Borders"}, "model": "fixtures_regress.store"}, {"fields": {"name": "Neal Stephenson"}, "model": "fixtures_regress.person"}, {"pk": 1, "model": "fixtures_regress.book", "fields": {"stores": [["Amazon"], ["Borders"]], "name": "Cryptonomicon", "author": ["Neal Stephenson"]}}]"""
         )
 
     def test_dependency_sorting(self):

tests/serializers_regress/models.py

@@ -118,6 +118,7 @@ class NaturalKeyAnchor(models.Model):
     objects = NaturalKeyAnchorManager()
 
     data = models.CharField(max_length=100, unique=True)
+    title = models.CharField(max_length=100, null=True)
 
     def natural_key(self):
         return (self.data,)

tests/serializers_regress/tests.py

@@ -11,6 +11,7 @@ from __future__ import unicode_literals
 import datetime
 import decimal
 from unittest import expectedFailure, skipUnless
+import warnings
 
 try:
     import yaml
@@ -476,9 +477,12 @@ def naturalKeySerializerTest(format, self):
     for klass in instance_count:
         instance_count[klass] = klass.objects.count()
 
-    # Serialize the test database
-    serialized_data = serializers.serialize(format, objects, indent=2,
-        use_natural_keys=True)
+    # use_natural_keys is deprecated and to be removed in Django 1.9
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        # Serialize the test database
+        serialized_data = serializers.serialize(format, objects, indent=2,
+            use_natural_keys=True)
 
     for obj in serializers.deserialize(format, serialized_data):
         obj.save()
@@ -523,6 +527,35 @@ def streamTest(format, self):
         else:
             self.assertEqual(string_data, stream.content.decode('utf-8'))
 
+
+def naturalKeyTest(format, self):
+    book1 = {'data': '978-1590597255', 'title': 'The Definitive Guide to '
+             'Django: Web Development Done Right'}
+    book2 = {'data':'978-1590599969', 'title': 'Practical Django Projects'}
+
+    # Create the books.
+    adrian = NaturalKeyAnchor.objects.create(**book1)
+    james = NaturalKeyAnchor.objects.create(**book2)
+
+    # Serialize the books.
+    string_data = serializers.serialize(format, NaturalKeyAnchor.objects.all(),
+                                        indent=2, use_natural_foreign_keys=True,
+                                        use_natural_primary_keys=True)
+
+    # Delete one book (to prove that the natural key generation will only
+    # restore the primary keys of books found in the database via the
+    # get_natural_key manager method).
+    james.delete()
+
+    # Deserialize and test.
+    books = list(serializers.deserialize(format, string_data))
+    self.assertEqual(len(books), 2)
+    self.assertEqual(books[0].object.title, book1['title'])
+    self.assertEqual(books[0].object.pk, adrian.pk)
+    self.assertEqual(books[1].object.title, book2['title'])
+    self.assertEqual(books[1].object.pk, None)
+
+
 for format in [
             f for f in serializers.get_serializer_formats()
             if not isinstance(serializers.get_serializer(f), serializers.BadSerializer)
@@ -530,6 +563,7 @@ for format in [
     setattr(SerializerTests, 'test_' + format + '_serializer', curry(serializerTest, format))
     setattr(SerializerTests, 'test_' + format + '_natural_key_serializer', curry(naturalKeySerializerTest, format))
     setattr(SerializerTests, 'test_' + format + '_serializer_fields', curry(fieldsTest, format))
+    setattr(SerializerTests, 'test_' + format + '_serializer_natural_keys', curry(naturalKeyTest, format))
     if format != 'python':
         setattr(SerializerTests, 'test_' + format + '_serializer_stream', curry(streamTest, format))