Port diff_tree and walk to Python 3

dulwich/diff_tree.py

@@ -17,7 +17,7 @@
 # MA  02110-1301, USA.
 
 """Utilities for diffing files and trees."""
-
+import sys
 from collections import (
     defaultdict,
     namedtuple,
@@ -36,6 +36,12 @@ from dulwich.objects import (
     S_ISGITLINK,
     TreeEntry,
     )
+from dulwich._py3_compat import (
+    items,
+    )
+
+if sys.version_info[0] == 3:
+    xrange = range
 
 # TreeChange type constants.
 CHANGE_ADD = 'add'
@@ -140,7 +146,7 @@ def walk_trees(store, tree1_id, tree2_id, prune_identical=False):
     # case.
     mode1 = tree1_id and stat.S_IFDIR or None
     mode2 = tree2_id and stat.S_IFDIR or None
-    todo = [(TreeEntry('', mode1, tree1_id), TreeEntry('', mode2, tree2_id))]
+    todo = [(TreeEntry(b'', mode1, tree1_id), TreeEntry(b'', mode2, tree2_id))]
     while todo:
         entry1, entry2 = todo.pop()
         is_tree1 = _is_tree(entry1)
@@ -261,7 +267,7 @@ def tree_changes_for_merge(store, parent_tree_ids, tree_id,
     change_type = lambda c: c.type
 
     # Yield only conflicting changes.
-    for _, changes in sorted(changes_by_path.iteritems()):
+    for _, changes in sorted(items(changes_by_path)):
         assert len(changes) == num_parents
         have = [c for c in changes if c is not None]
         if _all_eq(have, change_type, CHANGE_DELETE):
@@ -298,9 +304,11 @@ def _count_blocks(obj):
     block_getvalue = block.getvalue
 
     for c in chain(*obj.as_raw_chunks()):
+        if sys.version_info[0] == 3:
+            c = c.to_bytes(1, 'big')
         block_write(c)
         n += 1
-        if c == '\n' or n == _BLOCK_SIZE:
+        if c == b'\n' or n == _BLOCK_SIZE:
             value = block_getvalue()
             block_counts[hash(value)] += len(value)
             block_seek(0)
@@ -324,7 +332,7 @@ def _common_bytes(blocks1, blocks2):
     if len(blocks1) > len(blocks2):
         blocks1, blocks2 = blocks2, blocks1
     score = 0
-    for block, count1 in blocks1.iteritems():
+    for block, count1 in items(blocks1):
         count2 = blocks2.get(block)
         if count2:
             score += min(count1, count2)
@@ -452,7 +460,7 @@ class RenameDetector(object):
 
         add_paths = set()
         delete_paths = set()
-        for sha, sha_deletes in delete_map.iteritems():
+        for sha, sha_deletes in items(delete_map):
             sha_adds = add_map[sha]
             for (old, is_delete), new in izip(sha_deletes, sha_adds):
                 if stat.S_IFMT(old.mode) != stat.S_IFMT(new.mode):

dulwich/tests/test_walk.py

@@ -162,111 +162,111 @@ class WalkerTest(TestCase):
         self.assertWalkYields([c3], [c3.id], max_entries=1, reverse=True)
 
     def test_changes_one_parent(self):
-        blob_a1 = make_object(Blob, data='a1')
-        blob_a2 = make_object(Blob, data='a2')
-        blob_b2 = make_object(Blob, data='b2')
+        blob_a1 = make_object(Blob, data=b'a1')
+        blob_a2 = make_object(Blob, data=b'a2')
+        blob_b2 = make_object(Blob, data=b'b2')
         c1, c2 = self.make_linear_commits(
-          2, trees={1: [('a', blob_a1)],
-                    2: [('a', blob_a2), ('b', blob_b2)]})
-        e1 = TestWalkEntry(c1, [TreeChange.add(('a', F, blob_a1.id))])
-        e2 = TestWalkEntry(c2, [TreeChange(CHANGE_MODIFY, ('a', F, blob_a1.id),
-                                           ('a', F, blob_a2.id)),
-                                TreeChange.add(('b', F, blob_b2.id))])
+            2, trees={1: [(b'a', blob_a1)],
+                      2: [(b'a', blob_a2), (b'b', blob_b2)]})
+        e1 = TestWalkEntry(c1, [TreeChange.add((b'a', F, blob_a1.id))])
+        e2 = TestWalkEntry(c2, [TreeChange(CHANGE_MODIFY, (b'a', F, blob_a1.id),
+                                           (b'a', F, blob_a2.id)),
+                                TreeChange.add((b'b', F, blob_b2.id))])
         self.assertWalkYields([e2, e1], [c2.id])
 
     def test_changes_multiple_parents(self):
-        blob_a1 = make_object(Blob, data='a1')
-        blob_b2 = make_object(Blob, data='b2')
-        blob_a3 = make_object(Blob, data='a3')
+        blob_a1 = make_object(Blob, data=b'a1')
+        blob_b2 = make_object(Blob, data=b'b2')
+        blob_a3 = make_object(Blob, data=b'a3')
         c1, c2, c3 = self.make_commits(
-          [[1], [2], [3, 1, 2]],
-          trees={1: [('a', blob_a1)], 2: [('b', blob_b2)],
-                 3: [('a', blob_a3), ('b', blob_b2)]})
+            [[1], [2], [3, 1, 2]],
+            trees={1: [(b'a', blob_a1)], 2: [(b'b', blob_b2)],
+                   3: [(b'a', blob_a3), (b'b', blob_b2)]})
         # a is a modify/add conflict and b is not conflicted.
         changes = [[
-          TreeChange(CHANGE_MODIFY, ('a', F, blob_a1.id), ('a', F, blob_a3.id)),
-          TreeChange.add(('a', F, blob_a3.id)),
-          ]]
+            TreeChange(CHANGE_MODIFY, (b'a', F, blob_a1.id), (b'a', F, blob_a3.id)),
+            TreeChange.add((b'a', F, blob_a3.id)),
+        ]]
         self.assertWalkYields([TestWalkEntry(c3, changes)], [c3.id],
                               exclude=[c1.id, c2.id])
 
     def test_path_matches(self):
-        walker = Walker(None, [], paths=['foo', 'bar', 'baz/quux'])
-        self.assertTrue(walker._path_matches('foo'))
-        self.assertTrue(walker._path_matches('foo/a'))
-        self.assertTrue(walker._path_matches('foo/a/b'))
-        self.assertTrue(walker._path_matches('bar'))
-        self.assertTrue(walker._path_matches('baz/quux'))
-        self.assertTrue(walker._path_matches('baz/quux/a'))
+        walker = Walker(None, [], paths=[b'foo', b'bar', b'baz/quux'])
+        self.assertTrue(walker._path_matches(b'foo'))
+        self.assertTrue(walker._path_matches(b'foo/a'))
+        self.assertTrue(walker._path_matches(b'foo/a/b'))
+        self.assertTrue(walker._path_matches(b'bar'))
+        self.assertTrue(walker._path_matches(b'baz/quux'))
+        self.assertTrue(walker._path_matches(b'baz/quux/a'))
 
         self.assertFalse(walker._path_matches(None))
-        self.assertFalse(walker._path_matches('oops'))
-        self.assertFalse(walker._path_matches('fool'))
-        self.assertFalse(walker._path_matches('baz'))
-        self.assertFalse(walker._path_matches('baz/quu'))
+        self.assertFalse(walker._path_matches(b'oops'))
+        self.assertFalse(walker._path_matches(b'fool'))
+        self.assertFalse(walker._path_matches(b'baz'))
+        self.assertFalse(walker._path_matches(b'baz/quu'))
 
     def test_paths(self):
-        blob_a1 = make_object(Blob, data='a1')
-        blob_b2 = make_object(Blob, data='b2')
-        blob_a3 = make_object(Blob, data='a3')
-        blob_b3 = make_object(Blob, data='b3')
+        blob_a1 = make_object(Blob, data=b'a1')
+        blob_b2 = make_object(Blob, data=b'b2')
+        blob_a3 = make_object(Blob, data=b'a3')
+        blob_b3 = make_object(Blob, data=b'b3')
         c1, c2, c3 = self.make_linear_commits(
-          3, trees={1: [('a', blob_a1)],
-                    2: [('a', blob_a1), ('x/b', blob_b2)],
-                    3: [('a', blob_a3), ('x/b', blob_b3)]})
+            3, trees={1: [(b'a', blob_a1)],
+                      2: [(b'a', blob_a1), (b'x/b', blob_b2)],
+                      3: [(b'a', blob_a3), (b'x/b', blob_b3)]})
 
         self.assertWalkYields([c3, c2, c1], [c3.id])
-        self.assertWalkYields([c3, c1], [c3.id], paths=['a'])
-        self.assertWalkYields([c3, c2], [c3.id], paths=['x/b'])
+        self.assertWalkYields([c3, c1], [c3.id], paths=[b'a'])
+        self.assertWalkYields([c3, c2], [c3.id], paths=[b'x/b'])
 
         # All changes are included, not just for requested paths.
         changes = [
-          TreeChange(CHANGE_MODIFY, ('a', F, blob_a1.id),
-                     ('a', F, blob_a3.id)),
-          TreeChange(CHANGE_MODIFY, ('x/b', F, blob_b2.id),
-                     ('x/b', F, blob_b3.id)),
-          ]
+            TreeChange(CHANGE_MODIFY, (b'a', F, blob_a1.id),
+                       (b'a', F, blob_a3.id)),
+            TreeChange(CHANGE_MODIFY, (b'x/b', F, blob_b2.id),
+                       (b'x/b', F, blob_b3.id)),
+        ]
         self.assertWalkYields([TestWalkEntry(c3, changes)], [c3.id],
-                              max_entries=1, paths=['a'])
+                              max_entries=1, paths=[b'a'])
 
     def test_paths_subtree(self):
-        blob_a = make_object(Blob, data='a')
-        blob_b = make_object(Blob, data='b')
+        blob_a = make_object(Blob, data=b'a')
+        blob_b = make_object(Blob, data=b'b')
         c1, c2, c3 = self.make_linear_commits(
-          3, trees={1: [('x/a', blob_a)],
-                    2: [('b', blob_b), ('x/a', blob_a)],
-                    3: [('b', blob_b), ('x/a', blob_a), ('x/b', blob_b)]})
-        self.assertWalkYields([c2], [c3.id], paths=['b'])
-        self.assertWalkYields([c3, c1], [c3.id], paths=['x'])
+            3, trees={1: [(b'x/a', blob_a)],
+                      2: [(b'b', blob_b), (b'x/a', blob_a)],
+                      3: [(b'b', blob_b), (b'x/a', blob_a), (b'x/b', blob_b)]})
+        self.assertWalkYields([c2], [c3.id], paths=[b'b'])
+        self.assertWalkYields([c3, c1], [c3.id], paths=[b'x'])
 
     def test_paths_max_entries(self):
-        blob_a = make_object(Blob, data='a')
-        blob_b = make_object(Blob, data='b')
+        blob_a = make_object(Blob, data=b'a')
+        blob_b = make_object(Blob, data=b'b')
         c1, c2 = self.make_linear_commits(
-          2, trees={1: [('a', blob_a)],
-                    2: [('a', blob_a), ('b', blob_b)]})
-        self.assertWalkYields([c2], [c2.id], paths=['b'], max_entries=1)
-        self.assertWalkYields([c1], [c1.id], paths=['a'], max_entries=1)
+            2, trees={1: [(b'a', blob_a)],
+                      2: [(b'a', blob_a), (b'b', blob_b)]})
+        self.assertWalkYields([c2], [c2.id], paths=[b'b'], max_entries=1)
+        self.assertWalkYields([c1], [c1.id], paths=[b'a'], max_entries=1)
 
     def test_paths_merge(self):
-        blob_a1 = make_object(Blob, data='a1')
-        blob_a2 = make_object(Blob, data='a2')
-        blob_a3 = make_object(Blob, data='a3')
+        blob_a1 = make_object(Blob, data=b'a1')
+        blob_a2 = make_object(Blob, data=b'a2')
+        blob_a3 = make_object(Blob, data=b'a3')
         x1, y2, m3, m4 = self.make_commits(
-          [[1], [2], [3, 1, 2], [4, 1, 2]],
-          trees={1: [('a', blob_a1)],
-                 2: [('a', blob_a2)],
-                 3: [('a', blob_a3)],
-                 4: [('a', blob_a1)]})  # Non-conflicting
-        self.assertWalkYields([m3, y2, x1], [m3.id], paths=['a'])
-        self.assertWalkYields([y2, x1], [m4.id], paths=['a'])
+            [[1], [2], [3, 1, 2], [4, 1, 2]],
+            trees={1: [(b'a', blob_a1)],
+                   2: [(b'a', blob_a2)],
+                   3: [(b'a', blob_a3)],
+                   4: [(b'a', blob_a1)]})  # Non-conflicting
+        self.assertWalkYields([m3, y2, x1], [m3.id], paths=[b'a'])
+        self.assertWalkYields([y2, x1], [m4.id], paths=[b'a'])
 
     def test_changes_with_renames(self):
-        blob = make_object(Blob, data='blob')
+        blob = make_object(Blob, data=b'blob')
         c1, c2 = self.make_linear_commits(
-          2, trees={1: [('a', blob)], 2: [('b', blob)]})
-        entry_a = ('a', F, blob.id)
-        entry_b = ('b', F, blob.id)
+            2, trees={1: [(b'a', blob)], 2: [(b'b', blob)]})
+        entry_a = (b'a', F, blob.id)
+        entry_b = (b'b', F, blob.id)
         changes_without_renames = [TreeChange.delete(entry_a),
                                    TreeChange.add(entry_b)]
         changes_with_renames = [TreeChange(CHANGE_RENAME, entry_a, entry_b)]
@@ -278,37 +278,37 @@ class WalkerTest(TestCase):
           rename_detector=detector)
 
     def test_follow_rename(self):
-        blob = make_object(Blob, data='blob')
-        names = ['a', 'a', 'b', 'b', 'c', 'c']
+        blob = make_object(Blob, data=b'blob')
+        names = [b'a', b'a', b'b', b'b', b'c', b'c']
 
         trees = dict((i + 1, [(n, blob, F)]) for i, n in enumerate(names))
         c1, c2, c3, c4, c5, c6 = self.make_linear_commits(6, trees=trees)
-        self.assertWalkYields([c5], [c6.id], paths=['c'])
+        self.assertWalkYields([c5], [c6.id], paths=[b'c'])
 
         e = lambda n: (n, F, blob.id)
         self.assertWalkYields(
-          [TestWalkEntry(c5, [TreeChange(CHANGE_RENAME, e('b'), e('c'))]),
-           TestWalkEntry(c3, [TreeChange(CHANGE_RENAME, e('a'), e('b'))]),
-           TestWalkEntry(c1, [TreeChange.add(e('a'))])],
-          [c6.id], paths=['c'], follow=True)
+            [TestWalkEntry(c5, [TreeChange(CHANGE_RENAME, e(b'b'), e(b'c'))]),
+             TestWalkEntry(c3, [TreeChange(CHANGE_RENAME, e(b'a'), e(b'b'))]),
+             TestWalkEntry(c1, [TreeChange.add(e(b'a'))])],
+            [c6.id], paths=[b'c'], follow=True)
 
     def test_follow_rename_remove_path(self):
-        blob = make_object(Blob, data='blob')
+        blob = make_object(Blob, data=b'blob')
         _, _, _, c4, c5, c6 = self.make_linear_commits(
-          6, trees={1: [('a', blob), ('c', blob)],
-                    2: [],
-                    3: [],
-                    4: [('b', blob)],
-                    5: [('a', blob)],
-                    6: [('c', blob)]})
+            6, trees={1: [(b'a', blob), (b'c', blob)],
+                      2: [],
+                      3: [],
+                      4: [(b'b', blob)],
+                      5: [(b'a', blob)],
+                      6: [(b'c', blob)]})
 
         e = lambda n: (n, F, blob.id)
         # Once the path changes to b, we aren't interested in a or c anymore.
         self.assertWalkYields(
-          [TestWalkEntry(c6, [TreeChange(CHANGE_RENAME, e('a'), e('c'))]),
-           TestWalkEntry(c5, [TreeChange(CHANGE_RENAME, e('b'), e('a'))]),
-           TestWalkEntry(c4, [TreeChange.add(e('b'))])],
-          [c6.id], paths=['c'], follow=True)
+            [TestWalkEntry(c6, [TreeChange(CHANGE_RENAME, e(b'a'), e(b'c'))]),
+             TestWalkEntry(c5, [TreeChange(CHANGE_RENAME, e(b'b'), e(b'a'))]),
+             TestWalkEntry(c4, [TreeChange.add(e(b'b'))])],
+            [c6.id], paths=[b'c'], follow=True)
 
     def test_since(self):
         c1, c2, c3 = self.make_linear_commits(3)

dulwich/walk.py

@@ -251,7 +251,7 @@ class Walker(object):
             if changed_path == followed_path:
                 return True
             if (changed_path.startswith(followed_path) and
-                changed_path[len(followed_path)] == '/'):
+                    changed_path[len(followed_path)] == b'/'[0]):
                 return True
         return False