dulwich/tests/compat/test_bitmap.py

# test_bitmap.py -- Compatibility tests for git pack bitmaps.
# Copyright (C) 2025 Jelmer Vernooij <jelmer@jelmer.uk>
#
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
# General Public License as published by the Free Software Foundation; version 2.0
# or (at your option) any later version. You can redistribute it and/or
# modify it under the terms of either of these two licenses.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# You should have received a copy of the licenses; if not, see
# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
# License, Version 2.0.
#

"""Compatibility tests for git pack bitmaps."""

import os
import tempfile

from dulwich.bitmap import (
    BITMAP_OPT_FULL_DAG,
    BITMAP_OPT_HASH_CACHE,
    BITMAP_OPT_LOOKUP_TABLE,
    BitmapEntry,
    EWAHBitmap,
    PackBitmap,
    write_bitmap,
)
from dulwich.object_format import DEFAULT_OBJECT_FORMAT
from dulwich.pack import Pack
from dulwich.repo import Repo

from .. import TestCase
from .utils import remove_ro, require_git_version, rmtree_ro, run_git_or_fail


class BitmapCompatTests(TestCase):
    """Compatibility tests for reading git-generated bitmaps."""

    def setUp(self):
        super().setUp()
        # Git bitmap support was added in 2.0.0
        require_git_version((2, 0, 0))
        self._tempdir = tempfile.mkdtemp()
        self.addCleanup(rmtree_ro, self._tempdir)

    def _init_repo_with_bitmap(self):
        """Create a repo and generate a bitmap using git."""
        repo_path = os.path.join(self._tempdir, "test-repo")
        os.mkdir(repo_path)

        # Initialize repo
        run_git_or_fail(["init"], cwd=repo_path)

        # Create some commits
        test_file = os.path.join(repo_path, "test.txt")
        for i in range(5):
            with open(test_file, "w") as f:
                f.write(f"Content {i}\n")
            run_git_or_fail(["add", "test.txt"], cwd=repo_path)
            run_git_or_fail(
                ["commit", "-m", f"Commit {i}"],
                cwd=repo_path,
                env={"GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.com"},
            )

        # Enable bitmap writing and repack
        run_git_or_fail(
            ["config", "pack.writeBitmaps", "true"],
            cwd=repo_path,
        )
        run_git_or_fail(["repack", "-a", "-d", "-b"], cwd=repo_path)

        return repo_path

    def test_read_git_generated_bitmap(self):
        """Test that Dulwich can read a bitmap generated by git."""
        repo_path = self._init_repo_with_bitmap()

        # Find the pack file with bitmap
        pack_dir = os.path.join(repo_path, ".git", "objects", "pack")
        bitmap_files = [f for f in os.listdir(pack_dir) if f.endswith(".bitmap")]

        # Get the pack file (basename without extension)
        bitmap_name = bitmap_files[0]
        pack_basename = bitmap_name.replace(".bitmap", "")
        pack_path = os.path.join(pack_dir, pack_basename)

        # Verify bitmap file exists at expected location
        bitmap_path = pack_path + ".bitmap"
        self.assertTrue(
            os.path.exists(bitmap_path), f"Bitmap file not found at {bitmap_path}"
        )

        # Try to load the bitmap using Dulwich
        with Pack(pack_path, object_format=DEFAULT_OBJECT_FORMAT) as pack:
            bitmap = pack.bitmap

            # Basic checks
            self.assertIsNotNone(bitmap, f"Failed to load bitmap from {pack_path}")
            self.assertIsNotNone(bitmap.pack_checksum, "Bitmap missing pack checksum")

            # Check that we have some type bitmaps
            # At minimum, we should have some commits
            self.assertGreater(
                len(bitmap.commit_bitmap.bits),
                0,
                "Commit bitmap should not be empty",
            )

    def test_git_can_use_dulwich_repo_with_bitmap(self):
        """Test that git can work with a repo that has Dulwich-created objects."""
        repo_path = os.path.join(self._tempdir, "dulwich-repo")

        # Create a repo with Dulwich and add commits to ensure git creates bitmaps
        repo = Repo.init(repo_path, mkdir=True)
        self.addCleanup(repo.close)

        # Create actual commits, not just loose objects - git needs commits for bitmaps
        test_file = os.path.join(repo_path, "test.txt")
        for i in range(5):
            with open(test_file, "w") as f:
                f.write(f"Content {i}\n")
            run_git_or_fail(["add", "test.txt"], cwd=repo_path)
            run_git_or_fail(
                ["commit", "-m", f"Commit {i}"],
                cwd=repo_path,
                env={"GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.com"},
            )

        # Configure git to write bitmaps
        run_git_or_fail(
            ["config", "pack.writeBitmaps", "true"],
            cwd=repo_path,
        )

        # Git should be able to repack with bitmaps
        run_git_or_fail(["repack", "-a", "-d", "-b"], cwd=repo_path)

        # Verify git created a bitmap
        pack_dir = os.path.join(repo_path, ".git", "objects", "pack")
        self.assertTrue(os.path.exists(pack_dir), "Pack directory should exist")

        bitmap_files = [f for f in os.listdir(pack_dir) if f.endswith(".bitmap")]
        self.assertGreater(
            len(bitmap_files), 0, "Git should have created a bitmap file after repack"
        )

    def test_git_can_read_dulwich_bitmap(self):
        """Test that git can read a bitmap file written by Dulwich."""
        repo_path = os.path.join(self._tempdir, "dulwich-bitmap-repo")

        # Create a repo with git and add commits
        run_git_or_fail(["init"], cwd=None, env={"GIT_DIR": repo_path})

        test_file = os.path.join(repo_path, "..", "test.txt")
        os.makedirs(os.path.dirname(test_file), exist_ok=True)

        for i in range(5):
            with open(test_file, "w") as f:
                f.write(f"Content {i}\n")
            run_git_or_fail(
                ["add", test_file],
                cwd=os.path.dirname(repo_path),
                env={
                    "GIT_DIR": repo_path,
                    "GIT_WORK_TREE": os.path.dirname(repo_path),
                },
            )
            run_git_or_fail(
                ["commit", "-m", f"Commit {i}"],
                cwd=os.path.dirname(repo_path),
                env={
                    "GIT_DIR": repo_path,
                    "GIT_WORK_TREE": os.path.dirname(repo_path),
                    "GIT_AUTHOR_NAME": "Test",
                    "GIT_AUTHOR_EMAIL": "test@example.com",
                },
            )

        # Create a pack with git first
        run_git_or_fail(["repack", "-a", "-d"], cwd=None, env={"GIT_DIR": repo_path})

        # Now use Dulwich to write a bitmap for the pack
        pack_dir = os.path.join(repo_path, "objects", "pack")
        pack_files = [f for f in os.listdir(pack_dir) if f.endswith(".pack")]
        self.assertGreater(len(pack_files), 0, "Should have at least one pack file")

        pack_basename = pack_files[0].replace(".pack", "")
        pack_path = os.path.join(pack_dir, pack_basename)

        # Load the pack and create bitmap data, then close before writing
        with Pack(pack_path, object_format=DEFAULT_OBJECT_FORMAT) as pack:
            # Create a simple bitmap for testing
            # Git requires BITMAP_OPT_FULL_DAG flag
            bitmap = PackBitmap(
                flags=BITMAP_OPT_FULL_DAG
                | BITMAP_OPT_HASH_CACHE
                | BITMAP_OPT_LOOKUP_TABLE
            )
            bitmap.pack_checksum = pack.get_stored_checksum()

            # Add bitmap entries for the first few commits in the pack
            for i, (sha, offset, crc) in enumerate(pack.index.iterentries()):
                if i >= 3:  # Just add 3 entries
                    break

                ewah = EWAHBitmap()
                # Mark this object and a couple others as reachable
                for j in range(i + 1):
                    ewah.add(j)

                entry = BitmapEntry(object_pos=i, xor_offset=0, flags=0, bitmap=ewah)
                bitmap.entries[sha] = entry
                bitmap.entries_list.append((sha, entry))

            # Add name hash cache
            bitmap.name_hash_cache = [0x12345678, 0xABCDEF00, 0xFEDCBA98]

        # Write the bitmap after pack is closed to avoid file locking on Windows
        bitmap_path = pack_path + ".bitmap"
        remove_ro(bitmap_path)
        write_bitmap(bitmap_path, bitmap)

        # Verify git can use the repository with our bitmap
        # This should succeed if git can read our bitmap
        run_git_or_fail(
            ["rev-list", "--count", "--use-bitmap-index", "HEAD"],
            cwd=None,
            env={"GIT_DIR": repo_path},
        )

        # Verify git count-objects works with our bitmap
        run_git_or_fail(["count-objects", "-v"], cwd=None, env={"GIT_DIR": repo_path})

    def test_bitmap_file_format_structure(self):
        """Test that git-generated bitmap has expected structure."""
        repo_path = self._init_repo_with_bitmap()

        # Find bitmap
        pack_dir = os.path.join(repo_path, ".git", "objects", "pack")
        bitmap_files = [f for f in os.listdir(pack_dir) if f.endswith(".bitmap")]

        bitmap_path = os.path.join(pack_dir, bitmap_files[0])

        # Read the raw file to verify header
        with open(bitmap_path, "rb") as f:
            signature = f.read(4)
            self.assertEqual(b"BITM", signature, "Invalid bitmap signature")

            version = int.from_bytes(f.read(2), byteorder="big")
            self.assertGreaterEqual(version, 1, "Bitmap version should be >= 1")

        # Load with Dulwich and verify structure
        bitmap_name = bitmap_files[0]
        pack_basename = bitmap_name.replace(".bitmap", "")
        pack_path = os.path.join(pack_dir, pack_basename)
        with Pack(pack_path, object_format=DEFAULT_OBJECT_FORMAT) as pack:
            bitmap = pack.bitmap

            self.assertIsNotNone(bitmap)
            self.assertEqual(bitmap.version, version)