test_object_store.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476
  1. # test_object_store.py -- tests for object_store.py
  2. # Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
  3. #
  4. # This program is free software; you can redistribute it and/or
  5. # modify it under the terms of the GNU General Public License
  6. # as published by the Free Software Foundation; version 2
  7. # or (at your option) any later version of the License.
  8. #
  9. # This program is distributed in the hope that it will be useful,
  10. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. # GNU General Public License for more details.
  13. #
  14. # You should have received a copy of the GNU General Public License
  15. # along with this program; if not, write to the Free Software
  16. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  17. # MA 02110-1301, USA.
  18. """Tests for the object store interface."""
  19. from contextlib import closing
  20. from io import BytesIO
  21. import os
  22. import shutil
  23. import sys
  24. import tempfile
  25. from dulwich.index import (
  26. commit_tree,
  27. )
  28. from dulwich.errors import (
  29. NotTreeError,
  30. )
  31. from dulwich.objects import (
  32. sha_to_hex,
  33. Blob,
  34. Tree,
  35. TreeEntry,
  36. )
  37. from dulwich.object_store import (
  38. DiskObjectStore,
  39. MemoryObjectStore,
  40. ObjectStoreGraphWalker,
  41. tree_lookup_path,
  42. )
  43. from dulwich.pack import (
  44. REF_DELTA,
  45. write_pack_objects,
  46. )
  47. from dulwich.tests import (
  48. TestCase,
  49. )
  50. from dulwich.tests.utils import (
  51. make_object,
  52. make_tag,
  53. build_pack,
  54. )
  55. testobject = make_object(Blob, data=b"yummy data")
  56. class ObjectStoreTests(object):
  57. def test_determine_wants_all(self):
  58. self.assertEqual([b"1" * 40],
  59. self.store.determine_wants_all({b"refs/heads/foo": b"1" * 40}))
  60. def test_determine_wants_all_zero(self):
  61. self.assertEqual([],
  62. self.store.determine_wants_all({b"refs/heads/foo": b"0" * 40}))
  63. def test_iter(self):
  64. self.assertEqual([], list(self.store))
  65. def test_get_nonexistant(self):
  66. self.assertRaises(KeyError, lambda: self.store[b"a" * 40])
  67. def test_contains_nonexistant(self):
  68. self.assertFalse((b"a" * 40) in self.store)
  69. def test_add_objects_empty(self):
  70. self.store.add_objects([])
  71. def test_add_commit(self):
  72. # TODO: Argh, no way to construct Git commit objects without
  73. # access to a serialized form.
  74. self.store.add_objects([])
  75. def test_add_object(self):
  76. self.store.add_object(testobject)
  77. self.assertEqual(set([testobject.id]), set(self.store))
  78. self.assertTrue(testobject.id in self.store)
  79. r = self.store[testobject.id]
  80. self.assertEqual(r, testobject)
  81. def test_add_objects(self):
  82. data = [(testobject, "mypath")]
  83. self.store.add_objects(data)
  84. self.assertEqual(set([testobject.id]), set(self.store))
  85. self.assertTrue(testobject.id in self.store)
  86. r = self.store[testobject.id]
  87. self.assertEqual(r, testobject)
  88. def test_tree_changes(self):
  89. blob_a1 = make_object(Blob, data=b'a1')
  90. blob_a2 = make_object(Blob, data=b'a2')
  91. blob_b = make_object(Blob, data=b'b')
  92. for blob in [blob_a1, blob_a2, blob_b]:
  93. self.store.add_object(blob)
  94. blobs_1 = [(b'a', blob_a1.id, 0o100644), (b'b', blob_b.id, 0o100644)]
  95. tree1_id = commit_tree(self.store, blobs_1)
  96. blobs_2 = [(b'a', blob_a2.id, 0o100644), (b'b', blob_b.id, 0o100644)]
  97. tree2_id = commit_tree(self.store, blobs_2)
  98. change_a = ((b'a', b'a'), (0o100644, 0o100644), (blob_a1.id, blob_a2.id))
  99. self.assertEqual([change_a],
  100. list(self.store.tree_changes(tree1_id, tree2_id)))
  101. self.assertEqual(
  102. [change_a, ((b'b', b'b'), (0o100644, 0o100644), (blob_b.id, blob_b.id))],
  103. list(self.store.tree_changes(tree1_id, tree2_id,
  104. want_unchanged=True)))
  105. def test_iter_tree_contents(self):
  106. blob_a = make_object(Blob, data=b'a')
  107. blob_b = make_object(Blob, data=b'b')
  108. blob_c = make_object(Blob, data=b'c')
  109. for blob in [blob_a, blob_b, blob_c]:
  110. self.store.add_object(blob)
  111. blobs = [
  112. (b'a', blob_a.id, 0o100644),
  113. (b'ad/b', blob_b.id, 0o100644),
  114. (b'ad/bd/c', blob_c.id, 0o100755),
  115. (b'ad/c', blob_c.id, 0o100644),
  116. (b'c', blob_c.id, 0o100644),
  117. ]
  118. tree_id = commit_tree(self.store, blobs)
  119. self.assertEqual([TreeEntry(p, m, h) for (p, h, m) in blobs],
  120. list(self.store.iter_tree_contents(tree_id)))
  121. def test_iter_tree_contents_include_trees(self):
  122. blob_a = make_object(Blob, data=b'a')
  123. blob_b = make_object(Blob, data=b'b')
  124. blob_c = make_object(Blob, data=b'c')
  125. for blob in [blob_a, blob_b, blob_c]:
  126. self.store.add_object(blob)
  127. blobs = [
  128. (b'a', blob_a.id, 0o100644),
  129. (b'ad/b', blob_b.id, 0o100644),
  130. (b'ad/bd/c', blob_c.id, 0o100755),
  131. ]
  132. tree_id = commit_tree(self.store, blobs)
  133. tree = self.store[tree_id]
  134. tree_ad = self.store[tree[b'ad'][1]]
  135. tree_bd = self.store[tree_ad[b'bd'][1]]
  136. expected = [
  137. TreeEntry(b'', 0o040000, tree_id),
  138. TreeEntry(b'a', 0o100644, blob_a.id),
  139. TreeEntry(b'ad', 0o040000, tree_ad.id),
  140. TreeEntry(b'ad/b', 0o100644, blob_b.id),
  141. TreeEntry(b'ad/bd', 0o040000, tree_bd.id),
  142. TreeEntry(b'ad/bd/c', 0o100755, blob_c.id),
  143. ]
  144. actual = self.store.iter_tree_contents(tree_id, include_trees=True)
  145. self.assertEqual(expected, list(actual))
  146. def make_tag(self, name, obj):
  147. tag = make_tag(obj, name=name)
  148. self.store.add_object(tag)
  149. return tag
  150. def test_peel_sha(self):
  151. self.store.add_object(testobject)
  152. tag1 = self.make_tag(b'1', testobject)
  153. tag2 = self.make_tag(b'2', testobject)
  154. tag3 = self.make_tag(b'3', testobject)
  155. for obj in [testobject, tag1, tag2, tag3]:
  156. self.assertEqual(testobject, self.store.peel_sha(obj.id))
  157. def test_get_raw(self):
  158. self.store.add_object(testobject)
  159. self.assertEqual((Blob.type_num, b'yummy data'),
  160. self.store.get_raw(testobject.id))
  161. def test_close(self):
  162. # For now, just check that close doesn't barf.
  163. self.store.add_object(testobject)
  164. self.store.close()
  165. class MemoryObjectStoreTests(ObjectStoreTests, TestCase):
  166. def setUp(self):
  167. TestCase.setUp(self)
  168. self.store = MemoryObjectStore()
  169. def test_add_pack(self):
  170. o = MemoryObjectStore()
  171. f, commit, abort = o.add_pack()
  172. try:
  173. b = make_object(Blob, data=b"more yummy data")
  174. write_pack_objects(f, [(b, None)])
  175. except:
  176. abort()
  177. raise
  178. else:
  179. commit()
  180. def test_add_pack_emtpy(self):
  181. o = MemoryObjectStore()
  182. f, commit, abort = o.add_pack()
  183. commit()
  184. def test_add_thin_pack(self):
  185. o = MemoryObjectStore()
  186. blob = make_object(Blob, data=b'yummy data')
  187. o.add_object(blob)
  188. f = BytesIO()
  189. entries = build_pack(f, [
  190. (REF_DELTA, (blob.id, b'more yummy data')),
  191. ], store=o)
  192. o.add_thin_pack(f.read, None)
  193. packed_blob_sha = sha_to_hex(entries[0][3])
  194. self.assertEqual((Blob.type_num, b'more yummy data'),
  195. o.get_raw(packed_blob_sha))
  196. def test_add_thin_pack_empty(self):
  197. o = MemoryObjectStore()
  198. f = BytesIO()
  199. entries = build_pack(f, [], store=o)
  200. self.assertEqual([], entries)
  201. o.add_thin_pack(f.read, None)
  202. class PackBasedObjectStoreTests(ObjectStoreTests):
  203. def tearDown(self):
  204. for pack in self.store.packs:
  205. pack.close()
  206. def test_empty_packs(self):
  207. self.assertEqual([], list(self.store.packs))
  208. def test_pack_loose_objects(self):
  209. b1 = make_object(Blob, data=b"yummy data")
  210. self.store.add_object(b1)
  211. b2 = make_object(Blob, data=b"more yummy data")
  212. self.store.add_object(b2)
  213. self.assertEqual([], list(self.store.packs))
  214. self.assertEqual(2, self.store.pack_loose_objects())
  215. self.assertNotEqual([], list(self.store.packs))
  216. self.assertEqual(0, self.store.pack_loose_objects())
  217. class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
  218. def setUp(self):
  219. TestCase.setUp(self)
  220. self.store_dir = tempfile.mkdtemp()
  221. if not isinstance(self.store_dir, bytes):
  222. self.store_dir = self.store_dir.encode(sys.getfilesystemencoding())
  223. self.addCleanup(shutil.rmtree, self.store_dir)
  224. self.store = DiskObjectStore.init(self.store_dir)
  225. def tearDown(self):
  226. TestCase.tearDown(self)
  227. PackBasedObjectStoreTests.tearDown(self)
  228. def test_alternates(self):
  229. alternate_dir = tempfile.mkdtemp()
  230. if not isinstance(alternate_dir, bytes):
  231. alternate_dir = alternate_dir.encode(sys.getfilesystemencoding())
  232. self.addCleanup(shutil.rmtree, alternate_dir)
  233. alternate_store = DiskObjectStore(alternate_dir)
  234. b2 = make_object(Blob, data=b"yummy data")
  235. alternate_store.add_object(b2)
  236. store = DiskObjectStore(self.store_dir)
  237. self.assertRaises(KeyError, store.__getitem__, b2.id)
  238. store.add_alternate_path(alternate_dir)
  239. self.assertIn(b2.id, store)
  240. self.assertEqual(b2, store[b2.id])
  241. def test_add_alternate_path(self):
  242. store = DiskObjectStore(self.store_dir)
  243. self.assertEqual([], list(store._read_alternate_paths()))
  244. store.add_alternate_path(b'/foo/path')
  245. self.assertEqual([b'/foo/path'], list(store._read_alternate_paths()))
  246. store.add_alternate_path(b'/bar/path')
  247. self.assertEqual(
  248. [b'/foo/path', b'/bar/path'],
  249. list(store._read_alternate_paths()))
  250. def test_rel_alternative_path(self):
  251. alternate_dir = tempfile.mkdtemp()
  252. if not isinstance(alternate_dir, bytes):
  253. alternate_dir = alternate_dir.encode(sys.getfilesystemencoding())
  254. self.addCleanup(shutil.rmtree, alternate_dir)
  255. alternate_store = DiskObjectStore(alternate_dir)
  256. b2 = make_object(Blob, data=b"yummy data")
  257. alternate_store.add_object(b2)
  258. store = DiskObjectStore(self.store_dir)
  259. self.assertRaises(KeyError, store.__getitem__, b2.id)
  260. store.add_alternate_path(os.path.relpath(alternate_dir, self.store_dir))
  261. self.assertEqual(list(alternate_store), list(store.alternates[0]))
  262. self.assertIn(b2.id, store)
  263. self.assertEqual(b2, store[b2.id])
  264. def test_pack_dir(self):
  265. o = DiskObjectStore(self.store_dir)
  266. self.assertEqual(os.path.join(self.store_dir, b'pack'), o.pack_dir)
  267. def test_add_pack(self):
  268. o = DiskObjectStore(self.store_dir)
  269. f, commit, abort = o.add_pack()
  270. try:
  271. b = make_object(Blob, data=b"more yummy data")
  272. write_pack_objects(f, [(b, None)])
  273. except:
  274. abort()
  275. raise
  276. else:
  277. commit()
  278. def test_add_thin_pack(self):
  279. o = DiskObjectStore(self.store_dir)
  280. try:
  281. blob = make_object(Blob, data=b'yummy data')
  282. o.add_object(blob)
  283. f = BytesIO()
  284. entries = build_pack(f, [
  285. (REF_DELTA, (blob.id, b'more yummy data')),
  286. ], store=o)
  287. with o.add_thin_pack(f.read, None) as pack:
  288. packed_blob_sha = sha_to_hex(entries[0][3])
  289. pack.check_length_and_checksum()
  290. self.assertEqual(sorted([blob.id, packed_blob_sha]), list(pack))
  291. self.assertTrue(o.contains_packed(packed_blob_sha))
  292. self.assertTrue(o.contains_packed(blob.id))
  293. self.assertEqual((Blob.type_num, b'more yummy data'),
  294. o.get_raw(packed_blob_sha))
  295. finally:
  296. o.close()
  297. def test_add_thin_pack_empty(self):
  298. with closing(DiskObjectStore(self.store_dir)) as o:
  299. f = BytesIO()
  300. entries = build_pack(f, [], store=o)
  301. self.assertEqual([], entries)
  302. o.add_thin_pack(f.read, None)
  303. class TreeLookupPathTests(TestCase):
  304. def setUp(self):
  305. TestCase.setUp(self)
  306. self.store = MemoryObjectStore()
  307. blob_a = make_object(Blob, data=b'a')
  308. blob_b = make_object(Blob, data=b'b')
  309. blob_c = make_object(Blob, data=b'c')
  310. for blob in [blob_a, blob_b, blob_c]:
  311. self.store.add_object(blob)
  312. blobs = [
  313. (b'a', blob_a.id, 0o100644),
  314. (b'ad/b', blob_b.id, 0o100644),
  315. (b'ad/bd/c', blob_c.id, 0o100755),
  316. (b'ad/c', blob_c.id, 0o100644),
  317. (b'c', blob_c.id, 0o100644),
  318. ]
  319. self.tree_id = commit_tree(self.store, blobs)
  320. def get_object(self, sha):
  321. return self.store[sha]
  322. def test_lookup_blob(self):
  323. o_id = tree_lookup_path(self.get_object, self.tree_id, b'a')[1]
  324. self.assertTrue(isinstance(self.store[o_id], Blob))
  325. def test_lookup_tree(self):
  326. o_id = tree_lookup_path(self.get_object, self.tree_id, b'ad')[1]
  327. self.assertTrue(isinstance(self.store[o_id], Tree))
  328. o_id = tree_lookup_path(self.get_object, self.tree_id, b'ad/bd')[1]
  329. self.assertTrue(isinstance(self.store[o_id], Tree))
  330. o_id = tree_lookup_path(self.get_object, self.tree_id, b'ad/bd/')[1]
  331. self.assertTrue(isinstance(self.store[o_id], Tree))
  332. def test_lookup_nonexistent(self):
  333. self.assertRaises(KeyError, tree_lookup_path, self.get_object, self.tree_id, b'j')
  334. def test_lookup_not_tree(self):
  335. self.assertRaises(NotTreeError, tree_lookup_path, self.get_object, self.tree_id, b'ad/b/j')
  336. class ObjectStoreGraphWalkerTests(TestCase):
  337. def get_walker(self, heads, parent_map):
  338. new_parent_map = dict([
  339. (k * 40, [(p * 40) for p in ps]) for (k, ps) in parent_map.items()])
  340. return ObjectStoreGraphWalker([x * 40 for x in heads],
  341. new_parent_map.__getitem__)
  342. def test_ack_invalid_value(self):
  343. gw = self.get_walker([], {})
  344. self.assertRaises(ValueError, gw.ack, "tooshort")
  345. def test_empty(self):
  346. gw = self.get_walker([], {})
  347. self.assertIs(None, next(gw))
  348. gw.ack(b"a" * 40)
  349. self.assertIs(None, next(gw))
  350. def test_descends(self):
  351. gw = self.get_walker([b"a"], {b"a": [b"b"], b"b": []})
  352. self.assertEqual(b"a" * 40, next(gw))
  353. self.assertEqual(b"b" * 40, next(gw))
  354. def test_present(self):
  355. gw = self.get_walker([b"a"], {b"a": [b"b"], b"b": []})
  356. gw.ack(b"a" * 40)
  357. self.assertIs(None, next(gw))
  358. def test_parent_present(self):
  359. gw = self.get_walker([b"a"], {b"a": [b"b"], b"b": []})
  360. self.assertEqual(b"a" * 40, next(gw))
  361. gw.ack(b"a" * 40)
  362. self.assertIs(None, next(gw))
  363. def test_child_ack_later(self):
  364. gw = self.get_walker([b"a"], {b"a": [b"b"], b"b": [b"c"], b"c": []})
  365. self.assertEqual(b"a" * 40, next(gw))
  366. self.assertEqual(b"b" * 40, next(gw))
  367. gw.ack(b"a" * 40)
  368. self.assertIs(None, next(gw))
  369. def test_only_once(self):
  370. # a b
  371. # | |
  372. # c d
  373. # \ /
  374. # e
  375. gw = self.get_walker([b"a", b"b"], {
  376. b"a": [b"c"],
  377. b"b": [b"d"],
  378. b"c": [b"e"],
  379. b"d": [b"e"],
  380. b"e": [],
  381. })
  382. walk = []
  383. acked = False
  384. walk.append(next(gw))
  385. walk.append(next(gw))
  386. # A branch (a, c) or (b, d) may be done after 2 steps or 3 depending on
  387. # the order walked: 3-step walks include (a, b, c) and (b, a, d), etc.
  388. if walk == [b"a" * 40, b"c" * 40] or walk == [b"b" * 40, b"d" * 40]:
  389. gw.ack(walk[0])
  390. acked = True
  391. walk.append(next(gw))
  392. if not acked and walk[2] == b"c" * 40:
  393. gw.ack(b"a" * 40)
  394. elif not acked and walk[2] == b"d" * 40:
  395. gw.ack(b"b" * 40)
  396. walk.append(next(gw))
  397. self.assertIs(None, next(gw))
  398. self.assertEqual([b"a" * 40, b"b" * 40, b"c" * 40, b"d" * 40], sorted(walk))
  399. self.assertLess(walk.index(b"a" * 40), walk.index(b"c" * 40))
  400. self.assertLess(walk.index(b"b" * 40), walk.index(b"d" * 40))