Support thin packs in Pack.iterobjects(), Pack.get_raw().

dulwich/pack.py

@@ -1023,10 +1023,16 @@ class PackData(object):
         # TODO: cache these results
         if self.pack is None:
             raise KeyError(sha)
-        offset = self.pack.index.object_index(sha)
-        if not offset:
+        try:
+            offset = self.pack.index.object_index(sha)
+        except KeyError:
+            offset = None
+        if offset:
+            type, obj = self.get_object_at(offset)
+        elif self.pack is not None and self.pack.resolve_ext_ref:
+            type, obj = self.pack.resolve_ext_ref(sha)
+        else:
             raise KeyError(sha)
-        type, obj = self.get_object_at(offset)
         return offset, type, obj
 
     def resolve_object(self, offset, type, obj, get_ref=None):
@@ -1094,7 +1100,11 @@ class PackData(object):
         :return: iterator of tuples with (sha, offset, crc32)
         """
         num_objects = self._num_objects
-        for i, result in enumerate(PackIndexer.for_pack_data(self)):
+        resolve_ext_ref = (
+            self.pack.resolve_ext_ref if self.pack is not None else None)
+        indexer = PackIndexer.for_pack_data(
+            self, resolve_ext_ref=resolve_ext_ref)
+        for i, result in enumerate(indexer):
             if progress is not None:
                 progress(i, num_objects)
             yield result
@@ -1747,6 +1757,7 @@ class Pack(object):
         self._data_path = self._basename + '.pack'
         self._data_load = lambda: PackData(self._data_path)
         self._idx_load = lambda: load_pack_index(self._idx_path)
+        self.resolve_ext_ref = None
 
     @classmethod
     def from_lazy_objects(self, data_fn, idx_fn):
@@ -1851,7 +1862,8 @@ class Pack(object):
 
     def iterobjects(self):
         """Iterate over the objects in this pack."""
-        return iter(PackInflater.for_pack_data(self.data))
+        return iter(PackInflater.for_pack_data(
+            self.data, resolve_ext_ref=self.resolve_ext_ref))
 
     def pack_tuples(self):
         """Provide an iterable for use with write_pack_objects.

dulwich/tests/test_pack.py

@@ -420,6 +420,58 @@ class TestPack(PackTests):
         self.assertTrue(isinstance(objs[commit_sha], Commit))
 
 
+class TestThinPack(PackTests):
+
+    def setUp(self):
+        super(TestThinPack, self).setUp()
+        self.store = MemoryObjectStore()
+        self.blobs = {}
+        for blob in ('foo', 'bar', 'foo1234', 'bar2468'):
+            self.blobs[blob] = make_object(Blob, data=blob)
+        self.store.add_object(self.blobs['foo'])
+        self.store.add_object(self.blobs['bar'])
+
+        # Build a thin pack. 'foo' is as an external reference, 'bar' an
+        # internal reference.
+        self.pack_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.pack_dir)
+        self.pack_prefix = os.path.join(self.pack_dir, 'pack')
+        with open(self.pack_prefix + '.pack', 'wb') as f:
+            build_pack(f, [
+                (REF_DELTA, (self.blobs['foo'].id, 'foo1234')),
+                (Blob.type_num, 'bar'),
+                (REF_DELTA, (self.blobs['bar'].id, 'bar2468'))],
+                store=self.store)
+
+        # Index the new pack.
+        pack = self.make_pack(resolve_ext_ref=True)
+        data = PackData(pack._data_path)
+        data.pack = pack
+        data.create_index(self.pack_prefix + '.idx')
+
+        del self.store[self.blobs['bar'].id]
+
+    def make_pack(self, resolve_ext_ref):
+        pack = Pack(self.pack_prefix)
+        if resolve_ext_ref:
+            pack.resolve_ext_ref = self.store.get_raw
+        return pack
+
+    def test_get_raw(self):
+        self.assertRaises(
+            KeyError, self.make_pack(False).get_raw, self.blobs['foo1234'].id)
+        self.assertEqual(
+            (3, 'foo1234'),
+            self.make_pack(True).get_raw(self.blobs['foo1234'].id))
+
+    def test_iterobjects(self):
+        self.assertRaises(KeyError, list, self.make_pack(False).iterobjects())
+        self.assertEqual(
+            sorted([self.blobs['foo1234'].id, self.blobs['bar'].id,
+                    self.blobs['bar2468'].id]),
+            sorted(o.id for o in self.make_pack(True).iterobjects()))
+
+
 class WritePackTests(TestCase):
 
     def test_write_pack_header(self):