Fastimport python3 support.

.travis.yml

@@ -11,7 +11,7 @@ matrix:
     - python: "3.4"
       env: TEST_REQUIRE=
     - python: "3.5"
-      env: TEST_REQUIRE=
+      env: TEST_REQUIRE="fastimport"
 cache:
   directories:
     - $HOME/.cache/pip

NEWS

@@ -24,6 +24,9 @@
   * Don't rely on working tell() and seek() methods
     on wsgi.input. (Jonas Haag)
 
+  * Support fastexport/fastimport functionality on python3 with newer
+    versions of fastimport (>= 0.9.5).
+
 0.12.0	2015-12-13
 
  IMPROVEMENTS

dulwich/fastexport.py

@@ -20,6 +20,8 @@
 
 """Fast export/import functionality."""
 
+import sys
+
 from dulwich.index import (
     commit_tree,
     )
@@ -28,6 +30,9 @@ from dulwich.objects import (
     Commit,
     Tag,
     )
+from fastimport import __version__ as fastimport_version
+if fastimport_version <= (0, 9, 5) and sys.version_info[0] == 3 and sys.version_info[1] < 5:
+    raise ImportError("Older versions of fastimport don't support python3<3.5")
 from fastimport import (
     commands,
     errors as fastimport_errors,
@@ -39,8 +44,8 @@ import stat
 
 
 def split_email(text):
-    (name, email) = text.rsplit(" <", 1)
-    return (name, email.rstrip(">"))
+    (name, email) = text.rsplit(b" <", 1)
+    return (name, email.rstrip(b">"))
 
 
 class GitFastExporter(object):
@@ -53,11 +58,11 @@ class GitFastExporter(object):
         self._marker_idx = 0
 
     def print_cmd(self, cmd):
-        self.outf.write("%r\n" % cmd)
+        self.outf.write(getattr(cmd, "__bytes__", cmd.__repr__)() + b"\n")
 
     def _allocate_marker(self):
         self._marker_idx+=1
-        return str(self._marker_idx)
+        return b"%d" % (self._marker_idx,)
 
     def _export_blob(self, blob):
         marker = self._allocate_marker()
@@ -148,10 +153,10 @@ class GitImportProcessor(processor.ImportProcessor):
         (author_name, author_email, author_timestamp, author_timezone) = author
         (committer_name, committer_email, commit_timestamp,
             commit_timezone) = cmd.committer
-        commit.author = "%s <%s>" % (author_name, author_email)
+        commit.author = author_name + b" <" + author_email + b">"
         commit.author_timezone = author_timezone
         commit.author_time = int(author_timestamp)
-        commit.committer = "%s <%s>" % (committer_name, committer_email)
+        commit.committer = committer_name + b" <" + committer_email + b">"
         commit.commit_timezone = commit_timezone
         commit.commit_time = int(commit_timestamp)
         commit.message = cmd.message
@@ -159,32 +164,32 @@ class GitImportProcessor(processor.ImportProcessor):
         if cmd.from_:
             self._reset_base(cmd.from_)
         for filecmd in cmd.iter_files():
-            if filecmd.name == "filemodify":
+            if filecmd.name == b"filemodify":
                 if filecmd.data is not None:
                     blob = Blob.from_string(filecmd.data)
                     self.repo.object_store.add(blob)
                     blob_id = blob.id
                 else:
-                    assert filecmd.dataref[0] == ":", \
-                        "non-marker refs not supported yet"
+                    assert filecmd.dataref.startswith(b":"), \
+                        "non-marker refs not supported yet (%r)" % filecmd.dataref
                     blob_id = self.markers[filecmd.dataref[1:]]
                 self._contents[filecmd.path] = (filecmd.mode, blob_id)
-            elif filecmd.name == "filedelete":
+            elif filecmd.name == b"filedelete":
                 del self._contents[filecmd.path]
-            elif filecmd.name == "filecopy":
+            elif filecmd.name == b"filecopy":
                 self._contents[filecmd.dest_path] = self._contents[
                     filecmd.src_path]
-            elif filecmd.name == "filerename":
+            elif filecmd.name == b"filerename":
                 self._contents[filecmd.new_path] = self._contents[
                     filecmd.old_path]
                 del self._contents[filecmd.old_path]
-            elif filecmd.name == "filedeleteall":
+            elif filecmd.name == b"filedeleteall":
                 self._contents = {}
             else:
                 raise Exception("Command %s not supported" % filecmd.name)
         commit.tree = commit_tree(self.repo.object_store,
             ((path, hexsha, mode) for (path, (mode, hexsha)) in
-                self._contents.iteritems()))
+                self._contents.items()))
         if self.last_commit is not None:
             commit.parents.append(self.last_commit)
         commit.parents += cmd.merges

dulwich/tests/test_fastexport.py

@@ -56,25 +56,25 @@ class GitFastExporterTests(TestCase):
 
     def test_emit_blob(self):
         b = Blob()
-        b.data = "fooBAR"
+        b.data = b"fooBAR"
         self.fastexporter.emit_blob(b)
-        self.assertEqual('blob\nmark :1\ndata 6\nfooBAR\n',
+        self.assertEqual(b'blob\nmark :1\ndata 6\nfooBAR\n',
             self.stream.getvalue())
 
     def test_emit_commit(self):
         b = Blob()
-        b.data = "FOO"
+        b.data = b"FOO"
         t = Tree()
-        t.add("foo", stat.S_IFREG | 0o644, b.id)
+        t.add(b"foo", stat.S_IFREG | 0o644, b.id)
         c = Commit()
-        c.committer = c.author = "Jelmer <jelmer@host>"
+        c.committer = c.author = b"Jelmer <jelmer@host>"
         c.author_time = c.commit_time = 1271345553
         c.author_timezone = c.commit_timezone = 0
-        c.message = "msg"
+        c.message = b"msg"
         c.tree = t.id
         self.store.add_objects([(b, None), (t, None), (c, None)])
-        self.fastexporter.emit_commit(c, "refs/heads/master")
-        self.assertEqual("""blob
+        self.fastexporter.emit_commit(c, b"refs/heads/master")
+        self.assertEqual(b"""blob
 mark :1
 data 3
 FOO
@@ -103,30 +103,30 @@ class GitImportProcessorTests(TestCase):
     def test_reset_handler(self):
         from fastimport import commands
         [c1] = build_commit_graph(self.repo.object_store, [[1]])
-        cmd = commands.ResetCommand("refs/heads/foo", c1.id)
+        cmd = commands.ResetCommand(b"refs/heads/foo", c1.id)
         self.processor.reset_handler(cmd)
-        self.assertEqual(c1.id, self.repo.get_refs()["refs/heads/foo"])
+        self.assertEqual(c1.id, self.repo.get_refs()[b"refs/heads/foo"])
 
     def test_commit_handler(self):
         from fastimport import commands
-        cmd = commands.CommitCommand("refs/heads/foo", "mrkr",
-            ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
-            ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
-            "FOO", None, [], [])
+        cmd = commands.CommitCommand(b"refs/heads/foo",  b"mrkr",
+            (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600),
+            (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600),
+            b"FOO", None, [], [])
         self.processor.commit_handler(cmd)
         commit = self.repo[self.processor.last_commit]
-        self.assertEqual("Jelmer <jelmer@samba.org>", commit.author)
-        self.assertEqual("Jelmer <jelmer@samba.org>", commit.committer)
-        self.assertEqual("FOO", commit.message)
+        self.assertEqual(b"Jelmer <jelmer@samba.org>", commit.author)
+        self.assertEqual(b"Jelmer <jelmer@samba.org>", commit.committer)
+        self.assertEqual(b"FOO", commit.message)
         self.assertEqual([], commit.parents)
         self.assertEqual(432432432.0, commit.commit_time)
         self.assertEqual(432432432.0, commit.author_time)
         self.assertEqual(3600, commit.commit_timezone)
         self.assertEqual(3600, commit.author_timezone)
-        self.assertEqual(commit, self.repo["refs/heads/foo"])
+        self.assertEqual(commit, self.repo[b"refs/heads/foo"])
 
     def test_import_stream(self):
-        markers = self.processor.import_stream(BytesIO("""blob
+        markers = self.processor.import_stream(BytesIO(b"""blob
 mark :1
 data 11
 text for a
@@ -140,31 +140,31 @@ M 100644 :1 a
 
 """))
         self.assertEqual(2, len(markers))
-        self.assertTrue(isinstance(self.repo[markers["1"]], Blob))
-        self.assertTrue(isinstance(self.repo[markers["2"]], Commit))
+        self.assertTrue(isinstance(self.repo[markers[b"1"]], Blob))
+        self.assertTrue(isinstance(self.repo[markers[b"2"]], Commit))
 
     def test_file_add(self):
         from fastimport import commands
-        cmd = commands.BlobCommand("23", "data")
+        cmd = commands.BlobCommand(b"23", b"data")
         self.processor.blob_handler(cmd)
-        cmd = commands.CommitCommand("refs/heads/foo", "mrkr",
-            ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
-            ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
-            "FOO", None, [], [commands.FileModifyCommand("path", 0o100644, ":23", None)])
+        cmd = commands.CommitCommand(b"refs/heads/foo", b"mrkr",
+            (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600),
+            (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600),
+            b"FOO", None, [], [commands.FileModifyCommand(b"path", 0o100644, b":23", None)])
         self.processor.commit_handler(cmd)
         commit = self.repo[self.processor.last_commit]
         self.assertEqual([
-            ('path', 0o100644, '6320cd248dd8aeaab759d5871f8781b5c0505172')],
+            (b'path', 0o100644, b'6320cd248dd8aeaab759d5871f8781b5c0505172')],
             self.repo[commit.tree].items())
 
     def simple_commit(self):
         from fastimport import commands
-        cmd = commands.BlobCommand("23", "data")
+        cmd = commands.BlobCommand(b"23", b"data")
         self.processor.blob_handler(cmd)
-        cmd = commands.CommitCommand("refs/heads/foo", "mrkr",
-            ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
-            ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
-            "FOO", None, [], [commands.FileModifyCommand("path", 0o100644, ":23", None)])
+        cmd = commands.CommitCommand(b"refs/heads/foo", b"mrkr",
+            (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600),
+            (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600),
+            b"FOO", None, [], [commands.FileModifyCommand(b"path", 0o100644, b":23", None)])
         self.processor.commit_handler(cmd)
         commit = self.repo[self.processor.last_commit]
         return commit
@@ -176,34 +176,34 @@ M 100644 :1 a
         :return: The created commit object
         """
         from fastimport import commands
-        cmd = commands.CommitCommand("refs/heads/foo", "mrkr",
-            ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
-            ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
-            "FOO", None, [], file_cmds)
+        cmd = commands.CommitCommand(b"refs/heads/foo", b"mrkr",
+            (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600),
+            (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600),
+            b"FOO", None, [], file_cmds)
         self.processor.commit_handler(cmd)
         return self.repo[self.processor.last_commit]
 
     def test_file_copy(self):
         from fastimport import commands
         self.simple_commit()
-        commit = self.make_file_commit([commands.FileCopyCommand("path", "new_path")])
+        commit = self.make_file_commit([commands.FileCopyCommand(b"path", b"new_path")])
         self.assertEqual([
-            ('new_path', 0o100644, '6320cd248dd8aeaab759d5871f8781b5c0505172'),
-            ('path', 0o100644, '6320cd248dd8aeaab759d5871f8781b5c0505172'),
+            (b'new_path', 0o100644, b'6320cd248dd8aeaab759d5871f8781b5c0505172'),
+            (b'path', 0o100644, b'6320cd248dd8aeaab759d5871f8781b5c0505172'),
             ], self.repo[commit.tree].items())
 
     def test_file_move(self):
         from fastimport import commands
         self.simple_commit()
-        commit = self.make_file_commit([commands.FileRenameCommand("path", "new_path")])
+        commit = self.make_file_commit([commands.FileRenameCommand(b"path", b"new_path")])
         self.assertEqual([
-            ('new_path', 0o100644, '6320cd248dd8aeaab759d5871f8781b5c0505172'),
+            (b'new_path', 0o100644, b'6320cd248dd8aeaab759d5871f8781b5c0505172'),
             ], self.repo[commit.tree].items())
 
     def test_file_delete(self):
         from fastimport import commands
         self.simple_commit()
-        commit = self.make_file_commit([commands.FileDeleteCommand("path")])
+        commit = self.make_file_commit([commands.FileDeleteCommand(b"path")])
         self.assertEqual([], self.repo[commit.tree].items())
 
     def test_file_deleteall(self):