|
@@ -822,7 +822,7 @@ class MemoryPackIndex(PackIndex):
|
|
|
def for_pack(cls, pack_data: "PackData") -> "MemoryPackIndex":
|
|
def for_pack(cls, pack_data: "PackData") -> "MemoryPackIndex":
|
|
|
"""Create a MemoryPackIndex from a PackData object."""
|
|
"""Create a MemoryPackIndex from a PackData object."""
|
|
|
return MemoryPackIndex(
|
|
return MemoryPackIndex(
|
|
|
- list(pack_data.sorted_entries()), pack_data.get_stored_checksum()
|
|
|
|
|
|
|
+ list(pack_data.sorted_entries()), pack_checksum=pack_data.get_stored_checksum(), object_format=pack.object_format,
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
@classmethod
|
|
@classmethod
|
|
@@ -1645,6 +1645,7 @@ class PackData:
|
|
|
depth: int | None = None,
|
|
depth: int | None = None,
|
|
|
threads: int | None = None,
|
|
threads: int | None = None,
|
|
|
big_file_threshold: int | None = None,
|
|
big_file_threshold: int | None = None,
|
|
|
|
|
+ object_format=None,
|
|
|
) -> None:
|
|
) -> None:
|
|
|
"""Create a PackData object representing the pack in the given filename.
|
|
"""Create a PackData object representing the pack in the given filename.
|
|
|
|
|
|
|
@@ -1654,6 +1655,16 @@ class PackData:
|
|
|
Currently there is a restriction on the size of the pack as the python
|
|
Currently there is a restriction on the size of the pack as the python
|
|
|
mmap implementation is flawed.
|
|
mmap implementation is flawed.
|
|
|
"""
|
|
"""
|
|
|
|
|
+ from .object_format import DEFAULT_OBJECT_FORMAT
|
|
|
|
|
+
|
|
|
|
|
+ if object_format is None:
|
|
|
|
|
+ warnings.warn(
|
|
|
|
|
+ "PackData() should be called with object_format parameter",
|
|
|
|
|
+ DeprecationWarning,
|
|
|
|
|
+ stacklevel=2,
|
|
|
|
|
+ )
|
|
|
|
|
+ object_format = DEFAULT_OBJECT_FORMAT
|
|
|
|
|
+ self.object_format = object_format
|
|
|
self._filename = filename
|
|
self._filename = filename
|
|
|
self._size = size
|
|
self._size = size
|
|
|
self._header_size = 12
|
|
self._header_size = 12
|
|
@@ -1696,29 +1707,31 @@ class PackData:
|
|
|
return self._filename
|
|
return self._filename
|
|
|
|
|
|
|
|
@classmethod
|
|
@classmethod
|
|
|
- def from_file(cls, file: IO[bytes], size: int | None = None) -> "PackData":
|
|
|
|
|
|
|
+ def from_file(cls, file: IO[bytes], size: int | None = None, object_format: ObjectFormat | None = None) -> "PackData":
|
|
|
"""Create a PackData object from an open file.
|
|
"""Create a PackData object from an open file.
|
|
|
|
|
|
|
|
Args:
|
|
Args:
|
|
|
file: Open file object
|
|
file: Open file object
|
|
|
size: Optional file size
|
|
size: Optional file size
|
|
|
|
|
+ object_format: Object format used by the repository
|
|
|
|
|
|
|
|
Returns:
|
|
Returns:
|
|
|
PackData instance
|
|
PackData instance
|
|
|
"""
|
|
"""
|
|
|
- return cls(str(file), file=file, size=size)
|
|
|
|
|
|
|
+ return cls(str(file), file=file, size=size, object_format=object_format)
|
|
|
|
|
|
|
|
@classmethod
|
|
@classmethod
|
|
|
- def from_path(cls, path: str | os.PathLike[str]) -> "PackData":
|
|
|
|
|
|
|
+ def from_path(cls, path: str | os.PathLike[str], object_format: ObjectFormat | None = None) -> "PackData":
|
|
|
"""Create a PackData object from a file path.
|
|
"""Create a PackData object from a file path.
|
|
|
|
|
|
|
|
Args:
|
|
Args:
|
|
|
path: Path to the pack file
|
|
path: Path to the pack file
|
|
|
|
|
+ object_format: Object format used by the repository
|
|
|
|
|
|
|
|
Returns:
|
|
Returns:
|
|
|
PackData instance
|
|
PackData instance
|
|
|
"""
|
|
"""
|
|
|
- return cls(filename=path)
|
|
|
|
|
|
|
+ return cls(filename=path, object_format=object_format)
|
|
|
|
|
|
|
|
def close(self) -> None:
|
|
def close(self) -> None:
|
|
|
"""Close the underlying pack file."""
|
|
"""Close the underlying pack file."""
|
|
@@ -2575,6 +2588,7 @@ def pack_object_header(
|
|
|
Returns: A header for a packed object.
|
|
Returns: A header for a packed object.
|
|
|
"""
|
|
"""
|
|
|
from .object_format import DEFAULT_OBJECT_FORMAT
|
|
from .object_format import DEFAULT_OBJECT_FORMAT
|
|
|
|
|
+
|
|
|
if object_format is None:
|
|
if object_format is None:
|
|
|
warnings.warn(
|
|
warnings.warn(
|
|
|
"pack_object_header() should be called with object_format parameter",
|
|
"pack_object_header() should be called with object_format parameter",
|
|
@@ -2610,6 +2624,7 @@ def pack_object_chunks(
|
|
|
type: int,
|
|
type: int,
|
|
|
object: list[bytes] | tuple[bytes | int, list[bytes]],
|
|
object: list[bytes] | tuple[bytes | int, list[bytes]],
|
|
|
compression_level: int = -1,
|
|
compression_level: int = -1,
|
|
|
|
|
+ object_format: "ObjectFormat" | None = None,
|
|
|
) -> Iterator[bytes]:
|
|
) -> Iterator[bytes]:
|
|
|
"""Generate chunks for a pack object.
|
|
"""Generate chunks for a pack object.
|
|
|
|
|
|
|
@@ -2617,8 +2632,18 @@ def pack_object_chunks(
|
|
|
type: Numeric type of the object
|
|
type: Numeric type of the object
|
|
|
object: Object to write
|
|
object: Object to write
|
|
|
compression_level: the zlib compression level
|
|
compression_level: the zlib compression level
|
|
|
|
|
+ object_format: Object format (hash algorithm) to use
|
|
|
Returns: Chunks
|
|
Returns: Chunks
|
|
|
"""
|
|
"""
|
|
|
|
|
+ from .object_format import DEFAULT_OBJECT_FORMAT
|
|
|
|
|
+
|
|
|
|
|
+ if object_format is None:
|
|
|
|
|
+ warnings.warn(
|
|
|
|
|
+ "pack_object_chunks() should be called with object_format parameter",
|
|
|
|
|
+ DeprecationWarning,
|
|
|
|
|
+ stacklevel=2,
|
|
|
|
|
+ )
|
|
|
|
|
+ object_format = DEFAULT_OBJECT_FORMAT
|
|
|
if type in DELTA_TYPES:
|
|
if type in DELTA_TYPES:
|
|
|
if isinstance(object, tuple):
|
|
if isinstance(object, tuple):
|
|
|
delta_base, object = object
|
|
delta_base, object = object
|
|
@@ -2638,7 +2663,7 @@ def pack_object_chunks(
|
|
|
# Shouldn't reach here with proper typing
|
|
# Shouldn't reach here with proper typing
|
|
|
raise TypeError(f"Unexpected object type: {object.__class__.__name__}")
|
|
raise TypeError(f"Unexpected object type: {object.__class__.__name__}")
|
|
|
|
|
|
|
|
- yield bytes(pack_object_header(type, delta_base, sum(map(len, chunks))))
|
|
|
|
|
|
|
+ yield bytes(pack_object_header(type, delta_base, sum(map(len, chunks)), object_format=object_format))
|
|
|
compressor = zlib.compressobj(level=compression_level)
|
|
compressor = zlib.compressobj(level=compression_level)
|
|
|
for data in chunks:
|
|
for data in chunks:
|
|
|
yield compressor.compress(data)
|
|
yield compressor.compress(data)
|
|
@@ -2651,6 +2676,7 @@ def write_pack_object(
|
|
|
object: list[bytes] | tuple[bytes | int, list[bytes]],
|
|
object: list[bytes] | tuple[bytes | int, list[bytes]],
|
|
|
sha: "HashObject | None" = None,
|
|
sha: "HashObject | None" = None,
|
|
|
compression_level: int = -1,
|
|
compression_level: int = -1,
|
|
|
|
|
+ object_format: "ObjectFormat" | None = None,
|
|
|
) -> int:
|
|
) -> int:
|
|
|
"""Write pack object to a file.
|
|
"""Write pack object to a file.
|
|
|
|
|
|
|
@@ -2660,10 +2686,22 @@ def write_pack_object(
|
|
|
object: Object to write
|
|
object: Object to write
|
|
|
sha: Optional SHA-1 hasher to update
|
|
sha: Optional SHA-1 hasher to update
|
|
|
compression_level: the zlib compression level
|
|
compression_level: the zlib compression level
|
|
|
|
|
+ object_format: Object format (hash algorithm) to use
|
|
|
Returns: CRC32 checksum of the written object
|
|
Returns: CRC32 checksum of the written object
|
|
|
"""
|
|
"""
|
|
|
|
|
+ from .object_format import DEFAULT_OBJECT_FORMAT
|
|
|
|
|
+
|
|
|
|
|
+ if object_format is None:
|
|
|
|
|
+ warnings.warn(
|
|
|
|
|
+ "write_pack_object() should be called with object_format parameter",
|
|
|
|
|
+ DeprecationWarning,
|
|
|
|
|
+ stacklevel=2,
|
|
|
|
|
+ )
|
|
|
|
|
+ object_format = DEFAULT_OBJECT_FORMAT
|
|
|
crc32 = 0
|
|
crc32 = 0
|
|
|
- for chunk in pack_object_chunks(type, object, compression_level=compression_level):
|
|
|
|
|
|
|
+ for chunk in pack_object_chunks(
|
|
|
|
|
+ type, object, compression_level=compression_level, object_format=object_format
|
|
|
|
|
+ ):
|
|
|
write(chunk)
|
|
write(chunk)
|
|
|
if sha is not None:
|
|
if sha is not None:
|
|
|
sha.update(chunk)
|
|
sha.update(chunk)
|
|
@@ -2678,6 +2716,7 @@ def write_pack(
|
|
|
deltify: bool | None = None,
|
|
deltify: bool | None = None,
|
|
|
delta_window_size: int | None = None,
|
|
delta_window_size: int | None = None,
|
|
|
compression_level: int = -1,
|
|
compression_level: int = -1,
|
|
|
|
|
+ object_format: "ObjectFormat" | None = None
|
|
|
) -> tuple[bytes, bytes]:
|
|
) -> tuple[bytes, bytes]:
|
|
|
"""Write a new pack data file.
|
|
"""Write a new pack data file.
|
|
|
|
|
|
|
@@ -2687,8 +2726,18 @@ def write_pack(
|
|
|
delta_window_size: Delta window size
|
|
delta_window_size: Delta window size
|
|
|
deltify: Whether to deltify pack objects
|
|
deltify: Whether to deltify pack objects
|
|
|
compression_level: the zlib compression level
|
|
compression_level: the zlib compression level
|
|
|
|
|
+ object_format: Object format
|
|
|
Returns: Tuple with checksum of pack file and index file
|
|
Returns: Tuple with checksum of pack file and index file
|
|
|
"""
|
|
"""
|
|
|
|
|
+ from .object_format import DEFAULT_OBJECT_FORMAT
|
|
|
|
|
+
|
|
|
|
|
+ if object_format is None:
|
|
|
|
|
+ warnings.warn(
|
|
|
|
|
+ "write_pack() should be called with object_format parameter",
|
|
|
|
|
+ DeprecationWarning,
|
|
|
|
|
+ stacklevel=2,
|
|
|
|
|
+ )
|
|
|
|
|
+ object_format = DEFAULT_OBJECT_FORMAT
|
|
|
with GitFile(filename + ".pack", "wb") as f:
|
|
with GitFile(filename + ".pack", "wb") as f:
|
|
|
entries, data_sum = write_pack_objects(
|
|
entries, data_sum = write_pack_objects(
|
|
|
f,
|
|
f,
|
|
@@ -2696,6 +2745,7 @@ def write_pack(
|
|
|
delta_window_size=delta_window_size,
|
|
delta_window_size=delta_window_size,
|
|
|
deltify=deltify,
|
|
deltify=deltify,
|
|
|
compression_level=compression_level,
|
|
compression_level=compression_level,
|
|
|
|
|
+ object_format=object_format,
|
|
|
)
|
|
)
|
|
|
entries_list = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
|
|
entries_list = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
|
|
|
with GitFile(filename + ".idx", "wb") as f:
|
|
with GitFile(filename + ".idx", "wb") as f:
|
|
@@ -2995,7 +3045,8 @@ def write_pack_from_container(
|
|
|
deltify: bool | None = None,
|
|
deltify: bool | None = None,
|
|
|
reuse_deltas: bool = True,
|
|
reuse_deltas: bool = True,
|
|
|
compression_level: int = -1,
|
|
compression_level: int = -1,
|
|
|
- other_haves: set[ObjectID] | None = None,
|
|
|
|
|
|
|
+ other_haves: set[bytes] | None = None,
|
|
|
|
|
+ object_format: "ObjectFormat" | None = None,
|
|
|
) -> tuple[dict[bytes, tuple[int, int]], bytes]:
|
|
) -> tuple[dict[bytes, tuple[int, int]], bytes]:
|
|
|
"""Write a new pack data file.
|
|
"""Write a new pack data file.
|
|
|
|
|
|
|
@@ -3009,6 +3060,7 @@ def write_pack_from_container(
|
|
|
reuse_deltas: Whether to reuse existing deltas
|
|
reuse_deltas: Whether to reuse existing deltas
|
|
|
compression_level: the zlib compression level to use
|
|
compression_level: the zlib compression level to use
|
|
|
other_haves: Set of additional object IDs the receiver has
|
|
other_haves: Set of additional object IDs the receiver has
|
|
|
|
|
+ object_format: Object format (hash algorithm) to use
|
|
|
Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
|
|
Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
|
|
|
"""
|
|
"""
|
|
|
pack_contents_count = len(object_ids)
|
|
pack_contents_count = len(object_ids)
|
|
@@ -3026,6 +3078,7 @@ def write_pack_from_container(
|
|
|
pack_contents,
|
|
pack_contents,
|
|
|
num_records=pack_contents_count,
|
|
num_records=pack_contents_count,
|
|
|
compression_level=compression_level,
|
|
compression_level=compression_level,
|
|
|
|
|
+ object_format=object_format,
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
@@ -3036,6 +3089,7 @@ def write_pack_objects(
|
|
|
delta_window_size: int | None = None,
|
|
delta_window_size: int | None = None,
|
|
|
deltify: bool | None = None,
|
|
deltify: bool | None = None,
|
|
|
compression_level: int = -1,
|
|
compression_level: int = -1,
|
|
|
|
|
+ object_format: "ObjectFormat" | None = None
|
|
|
) -> tuple[dict[bytes, tuple[int, int]], bytes]:
|
|
) -> tuple[dict[bytes, tuple[int, int]], bytes]:
|
|
|
"""Write a new pack data file.
|
|
"""Write a new pack data file.
|
|
|
|
|
|
|
@@ -3046,6 +3100,7 @@ def write_pack_objects(
|
|
|
Set to None for default window size.
|
|
Set to None for default window size.
|
|
|
deltify: Whether to deltify objects
|
|
deltify: Whether to deltify objects
|
|
|
compression_level: the zlib compression level to use
|
|
compression_level: the zlib compression level to use
|
|
|
|
|
+ object_format: Object format (hash algorithm) to use
|
|
|
Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
|
|
Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
|
|
|
"""
|
|
"""
|
|
|
pack_contents_count, pack_contents = pack_objects_to_data(objects, deltify=deltify)
|
|
pack_contents_count, pack_contents = pack_objects_to_data(objects, deltify=deltify)
|
|
@@ -3055,6 +3110,7 @@ def write_pack_objects(
|
|
|
pack_contents,
|
|
pack_contents,
|
|
|
num_records=pack_contents_count,
|
|
num_records=pack_contents_count,
|
|
|
compression_level=compression_level,
|
|
compression_level=compression_level,
|
|
|
|
|
+ object_format=object_format,
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
@@ -3068,6 +3124,7 @@ class PackChunkGenerator:
|
|
|
progress: Callable[..., None] | None = None,
|
|
progress: Callable[..., None] | None = None,
|
|
|
compression_level: int = -1,
|
|
compression_level: int = -1,
|
|
|
reuse_compressed: bool = True,
|
|
reuse_compressed: bool = True,
|
|
|
|
|
+ object_format: "ObjectFormat" | None = None,
|
|
|
) -> None:
|
|
) -> None:
|
|
|
"""Initialize PackChunkGenerator.
|
|
"""Initialize PackChunkGenerator.
|
|
|
|
|
|
|
@@ -3077,7 +3134,18 @@ class PackChunkGenerator:
|
|
|
progress: Optional progress callback
|
|
progress: Optional progress callback
|
|
|
compression_level: Compression level (-1 for default)
|
|
compression_level: Compression level (-1 for default)
|
|
|
reuse_compressed: Whether to reuse compressed chunks
|
|
reuse_compressed: Whether to reuse compressed chunks
|
|
|
|
|
+ object_format: Object format (hash algorithm) to use
|
|
|
"""
|
|
"""
|
|
|
|
|
+ from .object_format import DEFAULT_OBJECT_FORMAT
|
|
|
|
|
+
|
|
|
|
|
+ if object_format is None:
|
|
|
|
|
+ warnings.warn(
|
|
|
|
|
+ "PackChunkGenerator() should be called with object_format parameter",
|
|
|
|
|
+ DeprecationWarning,
|
|
|
|
|
+ stacklevel=2,
|
|
|
|
|
+ )
|
|
|
|
|
+ object_format = DEFAULT_OBJECT_FORMAT
|
|
|
|
|
+ self.object_format = object_format
|
|
|
self.cs = sha1(b"")
|
|
self.cs = sha1(b"")
|
|
|
self.entries: dict[bytes, tuple[int, int]] = {}
|
|
self.entries: dict[bytes, tuple[int, int]] = {}
|
|
|
if records is None:
|
|
if records is None:
|
|
@@ -3151,7 +3219,10 @@ class PackChunkGenerator:
|
|
|
chunks = unpacked.comp_chunks
|
|
chunks = unpacked.comp_chunks
|
|
|
else:
|
|
else:
|
|
|
chunks = pack_object_chunks(
|
|
chunks = pack_object_chunks(
|
|
|
- type_num, raw, compression_level=compression_level
|
|
|
|
|
|
|
+ type_num,
|
|
|
|
|
+ raw,
|
|
|
|
|
+ compression_level=compression_level,
|
|
|
|
|
+ object_format=self.object_format,
|
|
|
)
|
|
)
|
|
|
crc32 = 0
|
|
crc32 = 0
|
|
|
object_size = 0
|
|
object_size = 0
|
|
@@ -3180,6 +3251,7 @@ def write_pack_data(
|
|
|
num_records: int | None = None,
|
|
num_records: int | None = None,
|
|
|
progress: Callable[..., None] | None = None,
|
|
progress: Callable[..., None] | None = None,
|
|
|
compression_level: int = -1,
|
|
compression_level: int = -1,
|
|
|
|
|
+ object_format: "ObjectFormat" | None = None,
|
|
|
) -> tuple[dict[bytes, tuple[int, int]], bytes]:
|
|
) -> tuple[dict[bytes, tuple[int, int]], bytes]:
|
|
|
"""Write a new pack data file.
|
|
"""Write a new pack data file.
|
|
|
|
|
|
|
@@ -3189,6 +3261,7 @@ def write_pack_data(
|
|
|
records: Iterator over type_num, object_id, delta_base, raw
|
|
records: Iterator over type_num, object_id, delta_base, raw
|
|
|
progress: Function to report progress to
|
|
progress: Function to report progress to
|
|
|
compression_level: the zlib compression level
|
|
compression_level: the zlib compression level
|
|
|
|
|
+ object_format: Object format (hash algorithm) to use
|
|
|
Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
|
|
Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
|
|
|
"""
|
|
"""
|
|
|
chunk_generator = PackChunkGenerator(
|
|
chunk_generator = PackChunkGenerator(
|
|
@@ -3196,6 +3269,7 @@ def write_pack_data(
|
|
|
records=records,
|
|
records=records,
|
|
|
progress=progress,
|
|
progress=progress,
|
|
|
compression_level=compression_level,
|
|
compression_level=compression_level,
|
|
|
|
|
+ object_format=object_format,
|
|
|
)
|
|
)
|
|
|
for chunk in chunk_generator:
|
|
for chunk in chunk_generator:
|
|
|
if callable(write):
|
|
if callable(write):
|
|
@@ -3617,6 +3691,7 @@ class Pack:
|
|
|
depth=depth,
|
|
depth=depth,
|
|
|
threads=threads,
|
|
threads=threads,
|
|
|
big_file_threshold=big_file_threshold,
|
|
big_file_threshold=big_file_threshold,
|
|
|
|
|
+ object_format=self.object_format,
|
|
|
)
|
|
)
|
|
|
self._idx_load = lambda: load_pack_index(
|
|
self._idx_load = lambda: load_pack_index(
|
|
|
self._idx_path, object_format=object_format
|
|
self._idx_path, object_format=object_format
|
|
@@ -3976,7 +4051,10 @@ class Pack:
|
|
|
assert isinstance(base_type, int)
|
|
assert isinstance(base_type, int)
|
|
|
elif base_type == REF_DELTA:
|
|
elif base_type == REF_DELTA:
|
|
|
(basename, delta) = base_obj
|
|
(basename, delta) = base_obj
|
|
|
- assert isinstance(basename, bytes) and len(basename) == self.object_format.oid_length
|
|
|
|
|
|
|
+ assert (
|
|
|
|
|
+ isinstance(basename, bytes)
|
|
|
|
|
+ and len(basename) == self.object_format.oid_length
|
|
|
|
|
+ )
|
|
|
base_offset, base_type, base_obj = get_ref(basename)
|
|
base_offset, base_type, base_obj = get_ref(basename)
|
|
|
assert isinstance(base_type, int)
|
|
assert isinstance(base_type, int)
|
|
|
if base_offset == prev_offset: # object is based on itself
|
|
if base_offset == prev_offset: # object is based on itself
|
|
@@ -4075,6 +4153,7 @@ def extend_pack(
|
|
|
that are already in the pack
|
|
that are already in the pack
|
|
|
"""
|
|
"""
|
|
|
from .object_format import DEFAULT_OBJECT_FORMAT
|
|
from .object_format import DEFAULT_OBJECT_FORMAT
|
|
|
|
|
+
|
|
|
if object_format is None:
|
|
if object_format is None:
|
|
|
warnings.warn(
|
|
warnings.warn(
|
|
|
"extend_pack() should be called with object_format parameter",
|
|
"extend_pack() should be called with object_format parameter",
|
|
@@ -4116,6 +4195,7 @@ def extend_pack(
|
|
|
[data], # Convert bytes to list[bytes]
|
|
[data], # Convert bytes to list[bytes]
|
|
|
sha=new_sha,
|
|
sha=new_sha,
|
|
|
compression_level=compression_level,
|
|
compression_level=compression_level,
|
|
|
|
|
+ object_format=object_format,
|
|
|
)
|
|
)
|
|
|
extra_entries.append((object_id, offset, crc32))
|
|
extra_entries.append((object_id, offset, crc32))
|
|
|
pack_sha = new_sha.digest()
|
|
pack_sha = new_sha.digest()
|