Add test verifying git can read Dulwich-written bitmap files

tests/compat/test_bitmap.py

@@ -0,0 +1,275 @@
+# test_bitmap.py -- Compatibility tests for git pack bitmaps.
+# Copyright (C) 2025 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+# General Public License as published by the Free Software Foundation; version 2.0
+# or (at your option) any later version. You can redistribute it and/or
+# modify it under the terms of either of these two licenses.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# You should have received a copy of the licenses; if not, see
+# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+# License, Version 2.0.
+#
+
+"""Compatibility tests for git pack bitmaps."""
+
+import os
+import shutil
+import tempfile
+
+from dulwich.pack import Pack
+from dulwich.repo import Repo
+
+from .. import SkipTest, TestCase
+from .utils import require_git_version, run_git_or_fail
+
+
+class BitmapCompatTests(TestCase):
+    """Compatibility tests for reading git-generated bitmaps."""
+
+    def setUp(self):
+        super().setUp()
+        # Git bitmap support was added in 2.0.0
+        require_git_version((2, 0, 0))
+        self._tempdir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self._tempdir)
+
+    def _init_repo_with_bitmap(self):
+        """Create a repo and generate a bitmap using git."""
+        repo_path = os.path.join(self._tempdir, "test-repo")
+        os.mkdir(repo_path)
+
+        # Initialize repo
+        run_git_or_fail(["init"], cwd=repo_path)
+
+        # Create some commits
+        test_file = os.path.join(repo_path, "test.txt")
+        for i in range(5):
+            with open(test_file, "w") as f:
+                f.write(f"Content {i}\n")
+            run_git_or_fail(["add", "test.txt"], cwd=repo_path)
+            run_git_or_fail(
+                ["commit", "-m", f"Commit {i}"],
+                cwd=repo_path,
+                env={"GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.com"},
+            )
+
+        # Enable bitmap writing and repack
+        run_git_or_fail(
+            ["config", "pack.writeBitmaps", "true"],
+            cwd=repo_path,
+        )
+        run_git_or_fail(["repack", "-a", "-d", "-b"], cwd=repo_path)
+
+        return repo_path
+
+    def test_read_git_generated_bitmap(self):
+        """Test that Dulwich can read a bitmap generated by git."""
+        repo_path = self._init_repo_with_bitmap()
+
+        # Find the pack file with bitmap
+        pack_dir = os.path.join(repo_path, ".git", "objects", "pack")
+        bitmap_files = [f for f in os.listdir(pack_dir) if f.endswith(".bitmap")]
+
+        if not bitmap_files:
+            raise SkipTest("Git did not generate a bitmap file")
+
+        # Get the pack file (basename without extension)
+        bitmap_name = bitmap_files[0]
+        pack_basename = bitmap_name.replace(".bitmap", "")
+        pack_path = os.path.join(pack_dir, pack_basename)
+
+        # Verify bitmap file exists at expected location
+        bitmap_path = pack_path + ".bitmap"
+        self.assertTrue(
+            os.path.exists(bitmap_path), f"Bitmap file not found at {bitmap_path}"
+        )
+
+        # Try to load the bitmap using Dulwich
+        pack = Pack(pack_path)
+        bitmap = pack.bitmap
+
+        # Basic checks
+        self.assertIsNotNone(bitmap, f"Failed to load bitmap from {pack_path}")
+        self.assertIsNotNone(bitmap.pack_checksum, "Bitmap missing pack checksum")
+
+        # Check that we have some type bitmaps
+        # At minimum, we should have some commits
+        self.assertGreater(
+            len(bitmap.commit_bitmap.bits),
+            0,
+            "Commit bitmap should not be empty",
+        )
+
+    def test_git_can_use_dulwich_repo_with_bitmap(self):
+        """Test that git can work with a repo that has Dulwich-created objects."""
+        repo_path = os.path.join(self._tempdir, "dulwich-repo")
+
+        # Create a repo with Dulwich and add commits to ensure git creates bitmaps
+        repo = Repo.init(repo_path, mkdir=True)
+        self.addCleanup(repo.close)
+
+        # Create actual commits, not just loose objects - git needs commits for bitmaps
+        test_file = os.path.join(repo_path, "test.txt")
+        for i in range(5):
+            with open(test_file, "w") as f:
+                f.write(f"Content {i}\n")
+            run_git_or_fail(["add", "test.txt"], cwd=repo_path)
+            run_git_or_fail(
+                ["commit", "-m", f"Commit {i}"],
+                cwd=repo_path,
+                env={"GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.com"},
+            )
+
+        # Configure git to write bitmaps
+        run_git_or_fail(
+            ["config", "pack.writeBitmaps", "true"],
+            cwd=repo_path,
+        )
+
+        # Git should be able to repack with bitmaps
+        run_git_or_fail(["repack", "-a", "-d", "-b"], cwd=repo_path)
+
+        # Verify git created a bitmap
+        pack_dir = os.path.join(repo_path, ".git", "objects", "pack")
+        self.assertTrue(os.path.exists(pack_dir), "Pack directory should exist")
+
+        bitmap_files = [f for f in os.listdir(pack_dir) if f.endswith(".bitmap")]
+        self.assertGreater(
+            len(bitmap_files), 0, "Git should have created a bitmap file after repack"
+        )
+
+    def test_git_can_read_dulwich_bitmap(self):
+        """Test that git can read a bitmap file written by Dulwich."""
+        repo_path = os.path.join(self._tempdir, "dulwich-bitmap-repo")
+
+        # Create a repo with git and add commits
+        run_git_or_fail(["init"], cwd=None, env={"GIT_DIR": repo_path})
+
+        test_file = os.path.join(repo_path, "..", "test.txt")
+        os.makedirs(os.path.dirname(test_file), exist_ok=True)
+
+        for i in range(5):
+            with open(test_file, "w") as f:
+                f.write(f"Content {i}\n")
+            run_git_or_fail(
+                ["add", test_file],
+                cwd=os.path.dirname(repo_path),
+                env={
+                    "GIT_DIR": repo_path,
+                    "GIT_WORK_TREE": os.path.dirname(repo_path),
+                },
+            )
+            run_git_or_fail(
+                ["commit", "-m", f"Commit {i}"],
+                cwd=os.path.dirname(repo_path),
+                env={
+                    "GIT_DIR": repo_path,
+                    "GIT_WORK_TREE": os.path.dirname(repo_path),
+                    "GIT_AUTHOR_NAME": "Test",
+                    "GIT_AUTHOR_EMAIL": "test@example.com",
+                },
+            )
+
+        # Create a pack with git first
+        run_git_or_fail(["repack", "-a", "-d"], cwd=None, env={"GIT_DIR": repo_path})
+
+        # Now use Dulwich to write a bitmap for the pack
+        from dulwich.bitmap import (
+            BITMAP_OPT_FULL_DAG,
+            BITMAP_OPT_HASH_CACHE,
+            BITMAP_OPT_LOOKUP_TABLE,
+            BitmapEntry,
+            EWAHBitmap,
+            PackBitmap,
+            write_bitmap,
+        )
+
+        pack_dir = os.path.join(repo_path, "objects", "pack")
+        pack_files = [f for f in os.listdir(pack_dir) if f.endswith(".pack")]
+        self.assertGreater(len(pack_files), 0, "Should have at least one pack file")
+
+        pack_basename = pack_files[0].replace(".pack", "")
+        pack_path = os.path.join(pack_dir, pack_basename)
+
+        # Load the pack
+        pack = Pack(pack_path)
+        self.addCleanup(pack.close)
+
+        # Create a simple bitmap for testing
+        # Git requires BITMAP_OPT_FULL_DAG flag
+        bitmap = PackBitmap(
+            flags=BITMAP_OPT_FULL_DAG | BITMAP_OPT_HASH_CACHE | BITMAP_OPT_LOOKUP_TABLE
+        )
+        bitmap.pack_checksum = pack.get_stored_checksum()
+
+        # Add bitmap entries for the first few commits in the pack
+        for i, (sha, offset, crc) in enumerate(pack.index.iterentries()):
+            if i >= 3:  # Just add 3 entries
+                break
+
+            ewah = EWAHBitmap()
+            # Mark this object and a couple others as reachable
+            for j in range(i + 1):
+                ewah.add(j)
+
+            entry = BitmapEntry(object_pos=i, xor_offset=0, flags=0, bitmap=ewah)
+            bitmap.entries[sha] = entry
+            bitmap.entries_list.append((sha, entry))
+
+        # Add name hash cache
+        bitmap.name_hash_cache = [0x12345678, 0xABCDEF00, 0xFEDCBA98]
+
+        # Write the bitmap
+        bitmap_path = pack_path + ".bitmap"
+        write_bitmap(bitmap_path, bitmap)
+
+        # Verify git can use the repository with our bitmap
+        # This should succeed if git can read our bitmap
+        run_git_or_fail(
+            ["rev-list", "--count", "--use-bitmap-index", "HEAD"],
+            cwd=None,
+            env={"GIT_DIR": repo_path},
+        )
+
+        # Verify git count-objects works with our bitmap
+        run_git_or_fail(["count-objects", "-v"], cwd=None, env={"GIT_DIR": repo_path})
+
+    def test_bitmap_file_format_structure(self):
+        """Test that git-generated bitmap has expected structure."""
+        repo_path = self._init_repo_with_bitmap()
+
+        # Find bitmap
+        pack_dir = os.path.join(repo_path, ".git", "objects", "pack")
+        bitmap_files = [f for f in os.listdir(pack_dir) if f.endswith(".bitmap")]
+
+        if not bitmap_files:
+            raise SkipTest("Git did not generate a bitmap file")
+
+        bitmap_path = os.path.join(pack_dir, bitmap_files[0])
+
+        # Read the raw file to verify header
+        with open(bitmap_path, "rb") as f:
+            signature = f.read(4)
+            self.assertEqual(b"BITM", signature, "Invalid bitmap signature")
+
+            version = int.from_bytes(f.read(2), byteorder="big")
+            self.assertGreaterEqual(version, 1, "Bitmap version should be >= 1")
+
+        # Load with Dulwich and verify structure
+        bitmap_name = bitmap_files[0]
+        pack_basename = bitmap_name.replace(".bitmap", "")
+        pack_path = os.path.join(pack_dir, pack_basename)
+        pack = Pack(pack_path)
+        bitmap = pack.bitmap
+
+        self.assertIsNotNone(bitmap)
+        self.assertEqual(bitmap.version, version)