Port object_store to Python3.

dulwich/object_store.py

@@ -26,6 +26,7 @@ import errno
 from itertools import chain
 import os
 import stat
+import sys
 import tempfile
 
 from dulwich.diff_tree import (
@@ -65,12 +66,16 @@ from dulwich.pack import (
 INFODIR = 'info'
 PACKDIR = 'pack'
 
+if sys.version_info[0] == 2:
+    iteritems = lambda d: d.iteritems()
+else:
+    iteritems = lambda d: d.items()
 
 class BaseObjectStore(object):
     """Object store interface."""
 
     def determine_wants_all(self, refs):
-        return [sha for (ref, sha) in refs.iteritems()
+        return [sha for (ref, sha) in iteritems(refs)
                 if not sha in self and not ref.endswith("^{}") and
                    not sha == ZERO_SHA]
 
@@ -695,7 +700,10 @@ class MemoryObjectStore(BaseObjectStore):
 
     def __iter__(self):
         """Iterate over the SHAs that are present in this store."""
-        return self._data.iterkeys()
+        if sys.version_info[0] == 2:
+            return self._data.iterkeys()
+        else:
+            return iter(self._data.keys())
 
     @property
     def packs(self):

dulwich/tests/test_object_store.py

@@ -63,21 +63,21 @@ testobject = make_object(Blob, data=b"yummy data")
 class ObjectStoreTests(object):
 
     def test_determine_wants_all(self):
-        self.assertEqual(["1" * 40],
-            self.store.determine_wants_all({"refs/heads/foo": "1" * 40}))
+        self.assertEqual([b"1" * 40],
+            self.store.determine_wants_all({"refs/heads/foo": b"1" * 40}))
 
     def test_determine_wants_all_zero(self):
         self.assertEqual([],
-            self.store.determine_wants_all({"refs/heads/foo": "0" * 40}))
+            self.store.determine_wants_all({"refs/heads/foo": b"0" * 40}))
 
     def test_iter(self):
         self.assertEqual([], list(self.store))
 
     def test_get_nonexistant(self):
-        self.assertRaises(KeyError, lambda: self.store["a" * 40])
+        self.assertRaises(KeyError, lambda: self.store[b"a" * 40])
 
     def test_contains_nonexistant(self):
-        self.assertFalse(("a" * 40) in self.store)
+        self.assertFalse((b"a" * 40) in self.store)
 
     def test_add_objects_empty(self):
         self.store.add_objects([])
@@ -103,66 +103,66 @@ class ObjectStoreTests(object):
         self.assertEqual(r, testobject)
 
     def test_tree_changes(self):
-        blob_a1 = make_object(Blob, data='a1')
-        blob_a2 = make_object(Blob, data='a2')
-        blob_b = make_object(Blob, data='b')
+        blob_a1 = make_object(Blob, data=b'a1')
+        blob_a2 = make_object(Blob, data=b'a2')
+        blob_b = make_object(Blob, data=b'b')
         for blob in [blob_a1, blob_a2, blob_b]:
             self.store.add_object(blob)
 
-        blobs_1 = [('a', blob_a1.id, 0o100644), ('b', blob_b.id, 0o100644)]
+        blobs_1 = [(b'a', blob_a1.id, 0o100644), (b'b', blob_b.id, 0o100644)]
         tree1_id = commit_tree(self.store, blobs_1)
-        blobs_2 = [('a', blob_a2.id, 0o100644), ('b', blob_b.id, 0o100644)]
+        blobs_2 = [(b'a', blob_a2.id, 0o100644), (b'b', blob_b.id, 0o100644)]
         tree2_id = commit_tree(self.store, blobs_2)
-        change_a = (('a', 'a'), (0o100644, 0o100644), (blob_a1.id, blob_a2.id))
+        change_a = ((b'a', b'a'), (0o100644, 0o100644), (blob_a1.id, blob_a2.id))
         self.assertEqual([change_a],
                           list(self.store.tree_changes(tree1_id, tree2_id)))
         self.assertEqual(
-          [change_a, (('b', 'b'), (0o100644, 0o100644), (blob_b.id, blob_b.id))],
-          list(self.store.tree_changes(tree1_id, tree2_id,
-                                       want_unchanged=True)))
+            [change_a, ((b'b', b'b'), (0o100644, 0o100644), (blob_b.id, blob_b.id))],
+            list(self.store.tree_changes(tree1_id, tree2_id,
+                                         want_unchanged=True)))
 
     def test_iter_tree_contents(self):
-        blob_a = make_object(Blob, data='a')
-        blob_b = make_object(Blob, data='b')
-        blob_c = make_object(Blob, data='c')
+        blob_a = make_object(Blob, data=b'a')
+        blob_b = make_object(Blob, data=b'b')
+        blob_c = make_object(Blob, data=b'c')
         for blob in [blob_a, blob_b, blob_c]:
             self.store.add_object(blob)
 
         blobs = [
-          ('a', blob_a.id, 0o100644),
-          ('ad/b', blob_b.id, 0o100644),
-          ('ad/bd/c', blob_c.id, 0o100755),
-          ('ad/c', blob_c.id, 0o100644),
-          ('c', blob_c.id, 0o100644),
-          ]
+            (b'a', blob_a.id, 0o100644),
+            (b'ad/b', blob_b.id, 0o100644),
+            (b'ad/bd/c', blob_c.id, 0o100755),
+            (b'ad/c', blob_c.id, 0o100644),
+            (b'c', blob_c.id, 0o100644),
+        ]
         tree_id = commit_tree(self.store, blobs)
         self.assertEqual([TreeEntry(p, m, h) for (p, h, m) in blobs],
                           list(self.store.iter_tree_contents(tree_id)))
 
     def test_iter_tree_contents_include_trees(self):
-        blob_a = make_object(Blob, data='a')
-        blob_b = make_object(Blob, data='b')
-        blob_c = make_object(Blob, data='c')
+        blob_a = make_object(Blob, data=b'a')
+        blob_b = make_object(Blob, data=b'b')
+        blob_c = make_object(Blob, data=b'c')
         for blob in [blob_a, blob_b, blob_c]:
             self.store.add_object(blob)
 
         blobs = [
-          ('a', blob_a.id, 0o100644),
-          ('ad/b', blob_b.id, 0o100644),
-          ('ad/bd/c', blob_c.id, 0o100755),
+          (b'a', blob_a.id, 0o100644),
+          (b'ad/b', blob_b.id, 0o100644),
+          (b'ad/bd/c', blob_c.id, 0o100755),
           ]
         tree_id = commit_tree(self.store, blobs)
         tree = self.store[tree_id]
-        tree_ad = self.store[tree['ad'][1]]
-        tree_bd = self.store[tree_ad['bd'][1]]
+        tree_ad = self.store[tree[b'ad'][1]]
+        tree_bd = self.store[tree_ad[b'bd'][1]]
 
         expected = [
-          TreeEntry('', 0o040000, tree_id),
-          TreeEntry('a', 0o100644, blob_a.id),
-          TreeEntry('ad', 0o040000, tree_ad.id),
-          TreeEntry('ad/b', 0o100644, blob_b.id),
-          TreeEntry('ad/bd', 0o040000, tree_bd.id),
-          TreeEntry('ad/bd/c', 0o100755, blob_c.id),
+          TreeEntry(b'', 0o040000, tree_id),
+          TreeEntry(b'a', 0o100644, blob_a.id),
+          TreeEntry(b'ad', 0o040000, tree_ad.id),
+          TreeEntry(b'ad/b', 0o100644, blob_b.id),
+          TreeEntry(b'ad/bd', 0o040000, tree_bd.id),
+          TreeEntry(b'ad/bd/c', 0o100755, blob_c.id),
           ]
         actual = self.store.iter_tree_contents(tree_id, include_trees=True)
         self.assertEqual(expected, list(actual))
@@ -174,15 +174,15 @@ class ObjectStoreTests(object):
 
     def test_peel_sha(self):
         self.store.add_object(testobject)
-        tag1 = self.make_tag('1', testobject)
-        tag2 = self.make_tag('2', testobject)
-        tag3 = self.make_tag('3', testobject)
+        tag1 = self.make_tag(b'1', testobject)
+        tag2 = self.make_tag(b'2', testobject)
+        tag3 = self.make_tag(b'3', testobject)
         for obj in [testobject, tag1, tag2, tag3]:
             self.assertEqual(testobject, self.store.peel_sha(obj.id))
 
     def test_get_raw(self):
         self.store.add_object(testobject)
-        self.assertEqual((Blob.type_num, 'yummy data'),
+        self.assertEqual((Blob.type_num, b'yummy data'),
                          self.store.get_raw(testobject.id))
 
     def test_close(self):
@@ -191,7 +191,6 @@ class ObjectStoreTests(object):
         self.store.close()
 
 
-@skipIfPY3
 class MemoryObjectStoreTests(ObjectStoreTests, TestCase):
 
     def setUp(self):
@@ -202,7 +201,7 @@ class MemoryObjectStoreTests(ObjectStoreTests, TestCase):
         o = MemoryObjectStore()
         f, commit, abort = o.add_pack()
         try:
-            b = make_object(Blob, data="more yummy data")
+            b = make_object(Blob, data=b"more yummy data")
             write_pack_objects(f, [(b, None)])
         except:
             abort()
@@ -217,16 +216,16 @@ class MemoryObjectStoreTests(ObjectStoreTests, TestCase):
 
     def test_add_thin_pack(self):
         o = MemoryObjectStore()
-        blob = make_object(Blob, data='yummy data')
+        blob = make_object(Blob, data=b'yummy data')
         o.add_object(blob)
 
         f = BytesIO()
         entries = build_pack(f, [
-          (REF_DELTA, (blob.id, 'more yummy data')),
-          ], store=o)
+            (REF_DELTA, (blob.id, b'more yummy data')),
+            ], store=o)
         o.add_thin_pack(f.read, None)
         packed_blob_sha = sha_to_hex(entries[0][3])
-        self.assertEqual((Blob.type_num, 'more yummy data'),
+        self.assertEqual((Blob.type_num, b'more yummy data'),
                          o.get_raw(packed_blob_sha))
 
 
@@ -250,9 +249,9 @@ class PackBasedObjectStoreTests(ObjectStoreTests):
         self.assertEqual([], self.store.packs)
 
     def test_pack_loose_objects(self):
-        b1 = make_object(Blob, data="yummy data")
+        b1 = make_object(Blob, data=b"yummy data")
         self.store.add_object(b1)
-        b2 = make_object(Blob, data="more yummy data")
+        b2 = make_object(Blob, data=b"more yummy data")
         self.store.add_object(b2)
         self.assertEqual([], self.store.packs)
         self.assertEqual(2, self.store.pack_loose_objects())
@@ -260,7 +259,6 @@ class PackBasedObjectStoreTests(ObjectStoreTests):
         self.assertEqual(0, self.store.pack_loose_objects())
 
 
-@skipIfPY3
 class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
 
     def setUp(self):
@@ -277,7 +275,7 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
         alternate_dir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, alternate_dir)
         alternate_store = DiskObjectStore(alternate_dir)
-        b2 = make_object(Blob, data="yummy data")
+        b2 = make_object(Blob, data=b"yummy data")
         alternate_store.add_object(b2)
         store = DiskObjectStore(self.store_dir)
         self.assertRaises(KeyError, store.__getitem__, b2.id)
@@ -299,7 +297,7 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
         alternate_dir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, alternate_dir)
         alternate_store = DiskObjectStore(alternate_dir)
-        b2 = make_object(Blob, data="yummy data")
+        b2 = make_object(Blob, data=b"yummy data")
         alternate_store.add_object(b2)
         store = DiskObjectStore(self.store_dir)
         self.assertRaises(KeyError, store.__getitem__, b2.id)
@@ -316,7 +314,7 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
         o = DiskObjectStore(self.store_dir)
         f, commit, abort = o.add_pack()
         try:
-            b = make_object(Blob, data="more yummy data")
+            b = make_object(Blob, data=b"more yummy data")
             write_pack_objects(f, [(b, None)])
         except:
             abort()
@@ -327,12 +325,12 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
     def test_add_thin_pack(self):
         o = DiskObjectStore(self.store_dir)
         try:
-            blob = make_object(Blob, data='yummy data')
+            blob = make_object(Blob, data=b'yummy data')
             o.add_object(blob)
 
             f = BytesIO()
             entries = build_pack(f, [
-              (REF_DELTA, (blob.id, 'more yummy data')),
+              (REF_DELTA, (blob.id, b'more yummy data')),
               ], store=o)
 
             with o.add_thin_pack(f.read, None) as pack:
@@ -341,7 +339,7 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
                 self.assertEqual(sorted([blob.id, packed_blob_sha]), list(pack))
                 self.assertTrue(o.contains_packed(packed_blob_sha))
                 self.assertTrue(o.contains_packed(blob.id))
-                self.assertEqual((Blob.type_num, 'more yummy data'),
+                self.assertEqual((Blob.type_num, b'more yummy data'),
                                  o.get_raw(packed_blob_sha))
         finally:
             o.close()
@@ -361,18 +359,18 @@ class TreeLookupPathTests(TestCase):
     def setUp(self):
         TestCase.setUp(self)
         self.store = MemoryObjectStore()
-        blob_a = make_object(Blob, data='a')
-        blob_b = make_object(Blob, data='b')
-        blob_c = make_object(Blob, data='c')
+        blob_a = make_object(Blob, data=b'a')
+        blob_b = make_object(Blob, data=b'b')
+        blob_c = make_object(Blob, data=b'c')
         for blob in [blob_a, blob_b, blob_c]:
             self.store.add_object(blob)
 
         blobs = [
-          ('a', blob_a.id, 0o100644),
-          ('ad/b', blob_b.id, 0o100644),
-          ('ad/bd/c', blob_c.id, 0o100755),
-          ('ad/c', blob_c.id, 0o100644),
-          ('c', blob_c.id, 0o100644),
+          (b'a', blob_a.id, 0o100644),
+          (b'ad/b', blob_b.id, 0o100644),
+          (b'ad/bd/c', blob_c.id, 0o100755),
+          (b'ad/c', blob_c.id, 0o100644),
+          (b'c', blob_c.id, 0o100644),
           ]
         self.tree_id = commit_tree(self.store, blobs)
 
@@ -380,24 +378,23 @@ class TreeLookupPathTests(TestCase):
         return self.store[sha]
 
     def test_lookup_blob(self):
-        o_id = tree_lookup_path(self.get_object, self.tree_id, 'a')[1]
+        o_id = tree_lookup_path(self.get_object, self.tree_id, b'a')[1]
         self.assertTrue(isinstance(self.store[o_id], Blob))
 
     def test_lookup_tree(self):
-        o_id = tree_lookup_path(self.get_object, self.tree_id, 'ad')[1]
+        o_id = tree_lookup_path(self.get_object, self.tree_id, b'ad')[1]
         self.assertTrue(isinstance(self.store[o_id], Tree))
-        o_id = tree_lookup_path(self.get_object, self.tree_id, 'ad/bd')[1]
+        o_id = tree_lookup_path(self.get_object, self.tree_id, b'ad/bd')[1]
         self.assertTrue(isinstance(self.store[o_id], Tree))
-        o_id = tree_lookup_path(self.get_object, self.tree_id, 'ad/bd/')[1]
+        o_id = tree_lookup_path(self.get_object, self.tree_id, b'ad/bd/')[1]
         self.assertTrue(isinstance(self.store[o_id], Tree))
 
     def test_lookup_nonexistent(self):
-        self.assertRaises(KeyError, tree_lookup_path, self.get_object, self.tree_id, 'j')
+        self.assertRaises(KeyError, tree_lookup_path, self.get_object, self.tree_id, b'j')
 
     def test_lookup_not_tree(self):
-        self.assertRaises(NotTreeError, tree_lookup_path, self.get_object, self.tree_id, 'ad/b/j')
+        self.assertRaises(NotTreeError, tree_lookup_path, self.get_object, self.tree_id, b'ad/b/j')
 
-@skipIfPY3
 class ObjectStoreGraphWalkerTests(TestCase):
 
     def get_walker(self, heads, parent_map):
@@ -413,30 +410,30 @@ class ObjectStoreGraphWalkerTests(TestCase):
     def test_empty(self):
         gw = self.get_walker([], {})
         self.assertIs(None, next(gw))
-        gw.ack("a" * 40)
+        gw.ack(b"a" * 40)
         self.assertIs(None, next(gw))
 
     def test_descends(self):
-        gw = self.get_walker(["a"], {"a": ["b"], "b": []})
-        self.assertEqual("a" * 40, next(gw))
-        self.assertEqual("b" * 40, next(gw))
+        gw = self.get_walker([b"a"], {b"a": [b"b"], b"b": []})
+        self.assertEqual(b"a" * 40, next(gw))
+        self.assertEqual(b"b" * 40, next(gw))
 
     def test_present(self):
-        gw = self.get_walker(["a"], {"a": ["b"], "b": []})
-        gw.ack("a" * 40)
+        gw = self.get_walker([b"a"], {b"a": [b"b"], b"b": []})
+        gw.ack(b"a" * 40)
         self.assertIs(None, next(gw))
 
     def test_parent_present(self):
-        gw = self.get_walker(["a"], {"a": ["b"], "b": []})
-        self.assertEqual("a" * 40, next(gw))
-        gw.ack("a" * 40)
+        gw = self.get_walker([b"a"], {b"a": [b"b"], b"b": []})
+        self.assertEqual(b"a" * 40, next(gw))
+        gw.ack(b"a" * 40)
         self.assertIs(None, next(gw))
 
     def test_child_ack_later(self):
-        gw = self.get_walker(["a"], {"a": ["b"], "b": ["c"], "c": []})
-        self.assertEqual("a" * 40, next(gw))
-        self.assertEqual("b" * 40, next(gw))
-        gw.ack("a" * 40)
+        gw = self.get_walker([b"a"], {b"a": [b"b"], b"b": [b"c"], b"c": []})
+        self.assertEqual(b"a" * 40, next(gw))
+        self.assertEqual(b"b" * 40, next(gw))
+        gw.ack(b"a" * 40)
         self.assertIs(None, next(gw))
 
     def test_only_once(self):
@@ -445,12 +442,12 @@ class ObjectStoreGraphWalkerTests(TestCase):
         # c  d
         # \ /
         #  e
-        gw = self.get_walker(["a", "b"], {
-                "a": ["c"],
-                "b": ["d"],
-                "c": ["e"],
-                "d": ["e"],
-                "e": [],
+        gw = self.get_walker([b"a", b"b"], {
+                b"a": [b"c"],
+                b"b": [b"d"],
+                b"c": [b"e"],
+                b"d": [b"e"],
+                b"e": [],
                 })
         walk = []
         acked = False
@@ -458,18 +455,18 @@ class ObjectStoreGraphWalkerTests(TestCase):
         walk.append(next(gw))
         # A branch (a, c) or (b, d) may be done after 2 steps or 3 depending on
         # the order walked: 3-step walks include (a, b, c) and (b, a, d), etc.
-        if walk == ["a" * 40, "c" * 40] or walk == ["b" * 40, "d" * 40]:
+        if walk == [b"a" * 40, b"c" * 40] or walk == [b"b" * 40, b"d" * 40]:
           gw.ack(walk[0])
           acked = True
 
         walk.append(next(gw))
-        if not acked and walk[2] == "c" * 40:
-          gw.ack("a" * 40)
-        elif not acked and walk[2] == "d" * 40:
-          gw.ack("b" * 40)
+        if not acked and walk[2] == b"c" * 40:
+          gw.ack(b"a" * 40)
+        elif not acked and walk[2] == b"d" * 40:
+          gw.ack(b"b" * 40)
         walk.append(next(gw))
         self.assertIs(None, next(gw))
 
-        self.assertEqual(["a" * 40, "b" * 40, "c" * 40, "d" * 40], sorted(walk))
-        self.assertLess(walk.index("a" * 40), walk.index("c" * 40))
-        self.assertLess(walk.index("b" * 40), walk.index("d" * 40))
+        self.assertEqual([b"a" * 40, b"b" * 40, b"c" * 40, b"d" * 40], sorted(walk))
+        self.assertLess(walk.index(b"a" * 40), walk.index(b"c" * 40))
+        self.assertLess(walk.index(b"b" * 40), walk.index(b"d" * 40))

dulwich/tests/utils.py

@@ -145,12 +145,12 @@ def make_tag(target, **attrs):
     target_id = target.id
     target_type = object_class(target.type_name)
     default_time = int(time.mktime(datetime.datetime(2010, 1, 1).timetuple()))
-    all_attrs = {'tagger': 'Test Author <test@nodomain.com>',
+    all_attrs = {'tagger': b'Test Author <test@nodomain.com>',
                  'tag_time': default_time,
                  'tag_timezone': 0,
-                 'message': 'Test message.',
+                 'message': b'Test message.',
                  'object': (target_type, target_id),
-                 'name': 'Test Tag',
+                 'name': b'Test Tag',
                  }
     all_attrs.update(attrs)
     return make_object(Tag, **all_attrs)