Enhance fetch with missing options (#1944)

Fixes #1843

dulwich/cli.py

@@ -983,19 +983,88 @@ class cmd_fetch(Command):
             args: Command line arguments
         """
         parser = argparse.ArgumentParser()
-        parser.add_argument("location", help="Remote location to fetch from")
+
+        # Mutually exclusive group for location vs --all
+        location_group = parser.add_mutually_exclusive_group(required=True)
+        location_group.add_argument(
+            "location", nargs="?", default=None, help="Remote location to fetch from"
+        )
+        location_group.add_argument(
+            "--all", action="store_true", help="Fetch all remotes"
+        )
+
+        # Mutually exclusive group for tag handling
+        tag_group = parser.add_mutually_exclusive_group()
+        tag_group.add_argument(
+            "--tags", action="store_true", help="Fetch all tags from remote"
+        )
+        tag_group.add_argument(
+            "--no-tags", action="store_true", help="Don't fetch any tags from remote"
+        )
+
+        parser.add_argument(
+            "--depth",
+            type=int,
+            help="Create a shallow clone with a history truncated to the specified number of commits",
+        )
+        parser.add_argument(
+            "--shallow-since",
+            type=str,
+            help="Deepen or shorten the history of a shallow repository to include all reachable commits after <date>",
+        )
+        parser.add_argument(
+            "--shallow-exclude",
+            type=str,
+            action="append",
+            help="Deepen or shorten the history of a shallow repository to exclude commits reachable from a specified remote branch or tag",
+        )
         parsed_args = parser.parse_args(args)
-        client, path = get_transport_and_path(parsed_args.location)
+
         r = Repo(".")
 
         def progress(msg: bytes) -> None:
             sys.stdout.buffer.write(msg)
 
-        result = client.fetch(path.encode("utf-8"), r, progress=progress)
-        logger.info("Remote refs:")
-        for ref, sha in result.refs.items():
-            if sha is not None:
-                logger.info("%s → %s", ref.decode(), sha.decode())
+        # Determine include_tags setting
+        include_tags = False
+        if parsed_args.tags:
+            include_tags = True
+        elif not parsed_args.no_tags:
+            # Default behavior - don't force tag inclusion
+            include_tags = False
+
+        if parsed_args.all:
+            # Fetch from all remotes
+            config = r.get_config()
+            remotes = set()
+            for section in config.sections():
+                if len(section) == 2 and section[0] == b"remote":
+                    remotes.add(section[1].decode())
+
+            if not remotes:
+                logger.warning("No remotes configured")
+                return
+
+            for remote_name in sorted(remotes):
+                logger.info("Fetching %s", remote_name)
+                porcelain.fetch(
+                    r,
+                    remote_location=remote_name,
+                    depth=parsed_args.depth,
+                    include_tags=include_tags,
+                    shallow_since=parsed_args.shallow_since,
+                    shallow_exclude=parsed_args.shallow_exclude,
+                )
+        else:
+            # Fetch from specific location
+            porcelain.fetch(
+                r,
+                remote_location=parsed_args.location,
+                depth=parsed_args.depth,
+                include_tags=include_tags,
+                shallow_since=parsed_args.shallow_since,
+                shallow_exclude=parsed_args.shallow_exclude,
+            )
 
 
 class cmd_for_each_ref(Command):

dulwich/client.py

@@ -134,6 +134,8 @@ from .protocol import (
     CAPABILITY_SYMREF,
     CAPABILITY_THIN_PACK,
     COMMAND_DEEPEN,
+    COMMAND_DEEPEN_NOT,
+    COMMAND_DEEPEN_SINCE,
     COMMAND_DONE,
     COMMAND_HAVE,
     COMMAND_SHALLOW,
@@ -715,6 +717,8 @@ def _handle_upload_pack_head(
     can_read: Optional[Callable[[], bool]],
     depth: Optional[int],
     protocol_version: Optional[int],
+    shallow_since: Optional[str] = None,
+    shallow_exclude: Optional[list[str]] = None,
 ) -> tuple[Optional[set[bytes]], Optional[set[bytes]]]:
     """Handle the head of a 'git-upload-pack' request.
 
@@ -727,6 +731,8 @@ def _handle_upload_pack_head(
     whether there is extra graph data to read on proto
       depth: Depth for request
       protocol_version: Neogiated Git protocol version.
+      shallow_since: Deepen the history to include commits after this date
+      shallow_exclude: Deepen the history to exclude commits reachable from these refs
     """
     new_shallow: Optional[set[bytes]]
     new_unshallow: Optional[set[bytes]]
@@ -740,8 +746,11 @@ def _handle_upload_pack_head(
     proto.write_pkt_line(wantcmd)
     for want in wants[1:]:
         proto.write_pkt_line(COMMAND_WANT + b" " + want + b"\n")
-    if depth not in (0, None) or (
-        hasattr(graph_walker, "shallow") and graph_walker.shallow
+    if (
+        depth not in (0, None)
+        or shallow_since is not None
+        or shallow_exclude
+        or (hasattr(graph_walker, "shallow") and graph_walker.shallow)
     ):
         if protocol_version == 2:
             if not find_capability(capabilities, CAPABILITY_FETCH, CAPABILITY_SHALLOW):
@@ -759,6 +768,15 @@ def _handle_upload_pack_head(
             proto.write_pkt_line(
                 COMMAND_DEEPEN + b" " + str(depth).encode("ascii") + b"\n"
             )
+        if shallow_since is not None:
+            proto.write_pkt_line(
+                COMMAND_DEEPEN_SINCE + b" " + shallow_since.encode("ascii") + b"\n"
+            )
+        if shallow_exclude:
+            for ref in shallow_exclude:
+                proto.write_pkt_line(
+                    COMMAND_DEEPEN_NOT + b" " + ref.encode("ascii") + b"\n"
+                )
     if protocol_version != 2:
         proto.write_pkt_line(None)
 
@@ -784,7 +802,7 @@ def _handle_upload_pack_head(
     if protocol_version == 2:
         proto.write_pkt_line(None)
 
-    if depth not in (0, None):
+    if depth not in (0, None) or shallow_since is not None or shallow_exclude:
         if can_read is not None:
             (new_shallow, new_unshallow) = _read_shallow_updates(proto.read_pkt_seq())
         else:
@@ -1096,6 +1114,8 @@ class GitClient:
         ref_prefix: Optional[Sequence[Ref]] = None,
         filter_spec: Optional[bytes] = None,
         protocol_version: Optional[int] = None,
+        shallow_since: Optional[str] = None,
+        shallow_exclude: Optional[list[str]] = None,
     ) -> FetchPackResult:
         """Fetch into a target repository.
 
@@ -1115,6 +1135,8 @@ class GitClient:
             feature, and ignored otherwise.
           protocol_version: Desired Git protocol version. By default the highest
             mutually supported protocol version will be used.
+          shallow_since: Deepen the history to include commits after this date
+          shallow_exclude: Deepen the history to exclude commits reachable from these refs
 
         Returns:
           Dictionary with all remote refs (not just those fetched)
@@ -1153,6 +1175,8 @@ class GitClient:
                 ref_prefix=ref_prefix,
                 filter_spec=filter_spec,
                 protocol_version=protocol_version,
+                shallow_since=shallow_since,
+                shallow_exclude=shallow_exclude,
             )
         except BaseException:
             abort()
@@ -1174,6 +1198,8 @@ class GitClient:
         ref_prefix: Optional[Sequence[Ref]] = None,
         filter_spec: Optional[bytes] = None,
         protocol_version: Optional[int] = None,
+        shallow_since: Optional[str] = None,
+        shallow_exclude: Optional[list[str]] = None,
     ) -> FetchPackResult:
         """Retrieve a pack from a git smart server.
 
@@ -1194,6 +1220,8 @@ class GitClient:
             feature, and ignored otherwise.
           protocol_version: Desired Git protocol version. By default the highest
             mutually supported protocol version will be used.
+          shallow_since: Deepen the history to include commits after this date
+          shallow_exclude: Deepen the history to exclude commits reachable from these refs
 
         Returns:
           FetchPackResult object
@@ -1530,6 +1558,8 @@ class TraditionalGitClient(GitClient):
         ref_prefix: Optional[Sequence[Ref]] = None,
         filter_spec: Optional[bytes] = None,
         protocol_version: Optional[int] = None,
+        shallow_since: Optional[str] = None,
+        shallow_exclude: Optional[list[str]] = None,
     ) -> FetchPackResult:
         """Retrieve a pack from a git smart server.
 
@@ -1550,6 +1580,8 @@ class TraditionalGitClient(GitClient):
             feature, and ignored otherwise.
           protocol_version: Desired Git protocol version. By default the highest
             mutually supported protocol version will be used.
+          shallow_since: Deepen the history to include commits after this date
+          shallow_exclude: Deepen the history to exclude commits reachable from these refs
 
         Returns:
           FetchPackResult object
@@ -1660,6 +1692,8 @@ class TraditionalGitClient(GitClient):
                 can_read,
                 depth=depth,
                 protocol_version=self.protocol_version,
+                shallow_since=shallow_since,
+                shallow_exclude=shallow_exclude,
             )
             _handle_upload_pack_tail(
                 proto,
@@ -2269,6 +2303,8 @@ class LocalGitClient(GitClient):
         ref_prefix: Optional[Sequence[bytes]] = None,
         filter_spec: Optional[bytes] = None,
         protocol_version: Optional[int] = None,
+        shallow_since: Optional[str] = None,
+        shallow_exclude: Optional[list[str]] = None,
     ) -> FetchPackResult:
         """Fetch into a target repository.
 
@@ -2287,6 +2323,8 @@ class LocalGitClient(GitClient):
             Only used if the server supports the Git protocol-v2 'filter'
             feature, and ignored otherwise.
           protocol_version: Optional Git protocol version
+          shallow_since: Deepen the history to include commits after this date
+          shallow_exclude: Deepen the history to exclude commits reachable from these refs
 
         Returns:
           FetchPackResult object
@@ -2314,6 +2352,8 @@ class LocalGitClient(GitClient):
         ref_prefix: Optional[Sequence[Ref]] = None,
         filter_spec: Optional[bytes] = None,
         protocol_version: Optional[int] = None,
+        shallow_since: Optional[str] = None,
+        shallow_exclude: Optional[list[str]] = None,
     ) -> FetchPackResult:
         """Retrieve a pack from a local on-disk repository.
 
@@ -2333,6 +2373,8 @@ class LocalGitClient(GitClient):
             Only used if the server supports the Git protocol-v2 'filter'
             feature, and ignored otherwise.
           protocol_version: Optional Git protocol version
+          shallow_since: Deepen the history to include commits after this date
+          shallow_exclude: Deepen the history to exclude commits reachable from these refs
 
         Returns:
           FetchPackResult object
@@ -2589,6 +2631,8 @@ class BundleClient(GitClient):
         ref_prefix: Optional[Sequence[Ref]] = None,
         filter_spec: Optional[bytes] = None,
         protocol_version: Optional[int] = None,
+        shallow_since: Optional[str] = None,
+        shallow_exclude: Optional[list[str]] = None,
     ) -> FetchPackResult:
         """Fetch into a target repository from a bundle file."""
         bundle = self._open_bundle(path)
@@ -2641,6 +2685,8 @@ class BundleClient(GitClient):
         ref_prefix: Optional[Sequence[Ref]] = None,
         filter_spec: Optional[bytes] = None,
         protocol_version: Optional[int] = None,
+        shallow_since: Optional[str] = None,
+        shallow_exclude: Optional[list[str]] = None,
     ) -> FetchPackResult:
         """Retrieve a pack from a bundle file."""
         bundle = self._open_bundle(path)
@@ -3641,6 +3687,8 @@ class AbstractHttpGitClient(GitClient):
         ref_prefix: Optional[Sequence[Ref]] = None,
         filter_spec: Optional[bytes] = None,
         protocol_version: Optional[int] = None,
+        shallow_since: Optional[str] = None,
+        shallow_exclude: Optional[list[str]] = None,
     ) -> FetchPackResult:
         """Retrieve a pack from a git smart server.
 
@@ -3659,6 +3707,8 @@ class AbstractHttpGitClient(GitClient):
             feature, and ignored otherwise.
           protocol_version: Desired Git protocol version. By default the highest
             mutually supported protocol version will be used.
+          shallow_since: Deepen the history to include commits after this date
+          shallow_exclude: Deepen the history to exclude commits reachable from these refs
 
         Returns:
           FetchPackResult object
@@ -3736,6 +3786,8 @@ class AbstractHttpGitClient(GitClient):
             can_read=None,
             depth=depth,
             protocol_version=self.protocol_version,
+            shallow_since=shallow_since,
+            shallow_exclude=shallow_exclude,
         )
         if self.protocol_version == 2:
             data = pkt_line(b"command=fetch\n") + b"0001"

dulwich/porcelain.py

@@ -3909,6 +3909,8 @@ def fetch(
     password: Optional[str] = None,
     key_filename: Optional[str] = None,
     ssh_command: Optional[str] = None,
+    shallow_since: Optional[str] = None,
+    shallow_exclude: Optional[list[str]] = None,
 ) -> FetchPackResult:
     """Fetch objects from a remote server.
 
@@ -3931,6 +3933,8 @@ def fetch(
       password: Password for authentication
       key_filename: SSH key filename
       ssh_command: SSH command to use
+      shallow_since: Deepen or shorten the history to include commits after this date
+      shallow_exclude: Deepen or shorten the history to exclude commits reachable from these refs
     Returns:
       Dictionary with refs on the remote
     """
@@ -3955,7 +3959,14 @@ def fetch(
         def progress(data: bytes) -> None:
             errstream.write(data)
 
-        fetch_result = client.fetch(path.encode(), r, progress=progress, depth=depth)
+        fetch_result = client.fetch(
+            path.encode(),
+            r,
+            progress=progress,
+            depth=depth,
+            shallow_since=shallow_since,
+            shallow_exclude=shallow_exclude,
+        )
         if remote_name is not None:
             _import_remote_refs(
                 r.refs,

dulwich/protocol.py

@@ -201,6 +201,8 @@ def symref_capabilities(symrefs: Iterable[tuple[bytes, bytes]]) -> list[bytes]:
 
 
 COMMAND_DEEPEN = b"deepen"
+COMMAND_DEEPEN_SINCE = b"deepen-since"
+COMMAND_DEEPEN_NOT = b"deepen-not"
 COMMAND_SHALLOW = b"shallow"
 COMMAND_UNSHALLOW = b"unshallow"
 COMMAND_DONE = b"done"

tests/compat/test_client.py

@@ -336,6 +336,57 @@ class DulwichClientTestBase:
                 },
             )
 
+    def test_fetch_pack_deepen_since(self) -> None:
+        c = self._client()
+        with repo.Repo(os.path.join(self.gitroot, "dest")) as dest:
+            # Fetch commits since a specific date
+            # Using Unix timestamp - the test repo has commits around 1265755064 (Feb 2010)
+            # So we use a timestamp between first and last commit
+            result = c.fetch(
+                self._build_path("/server_new.export"),
+                dest,
+                shallow_since="1265755100",
+            )
+            for r in result.refs.items():
+                dest.refs.set_if_equals(r[0], None, r[1])
+            # Verify that shallow commits were created
+            shallow = dest.get_shallow()
+            self.assertIsNotNone(shallow)
+            self.assertGreater(len(shallow), 0)
+
+    def test_fetch_pack_deepen_not(self) -> None:
+        c = self._client()
+        with repo.Repo(os.path.join(self.gitroot, "dest")) as dest:
+            # Fetch excluding commits reachable from a specific ref
+            result = c.fetch(
+                self._build_path("/server_new.export"),
+                dest,
+                shallow_exclude=["refs/heads/branch"],
+            )
+            for r in result.refs.items():
+                dest.refs.set_if_equals(r[0], None, r[1])
+            # Verify that shallow commits were created
+            shallow = dest.get_shallow()
+            self.assertIsNotNone(shallow)
+            self.assertGreater(len(shallow), 0)
+
+    def test_fetch_pack_deepen_since_and_not(self) -> None:
+        c = self._client()
+        with repo.Repo(os.path.join(self.gitroot, "dest")) as dest:
+            # Fetch combining deepen-since and deepen-not
+            result = c.fetch(
+                self._build_path("/server_new.export"),
+                dest,
+                shallow_since="1265755100",
+                shallow_exclude=["refs/heads/branch"],
+            )
+            for r in result.refs.items():
+                dest.refs.set_if_equals(r[0], None, r[1])
+            # Verify that shallow commits were created
+            shallow = dest.get_shallow()
+            self.assertIsNotNone(shallow)
+            self.assertGreater(len(shallow), 0)
+
     def test_repeat(self) -> None:
         c = self._client()
         with repo.Repo(os.path.join(self.gitroot, "dest")) as dest:

tests/test_client.py

@@ -186,6 +186,145 @@ class GitClientTests(TestCase):
         self.assertEqual({}, ret.symrefs)
         self.assertEqual(self.rout.getvalue(), b"0000")
 
+    def test_handle_upload_pack_head_deepen_since(self) -> None:
+        # Test that deepen-since command is properly sent
+        from dulwich.client import _handle_upload_pack_head
+
+        self.rin.write(b"0008NAK\n0000")
+        self.rin.seek(0)
+
+        class DummyGraphWalker:
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                return None
+
+        proto = Protocol(self.rin.read, self.rout.write)
+        capabilities = [b"shallow", b"deepen-since"]
+        wants = [b"55dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7"]
+        graph_walker = DummyGraphWalker()
+
+        _handle_upload_pack_head(
+            proto=proto,
+            capabilities=capabilities,
+            graph_walker=graph_walker,
+            wants=wants,
+            can_read=None,
+            depth=None,
+            protocol_version=0,
+            shallow_since="2023-01-01T00:00:00Z",
+        )
+
+        # Verify the deepen-since command was sent
+        output = self.rout.getvalue()
+        self.assertIn(b"deepen-since 2023-01-01T00:00:00Z\n", output)
+
+    def test_handle_upload_pack_head_deepen_not(self) -> None:
+        # Test that deepen-not command is properly sent
+        from dulwich.client import _handle_upload_pack_head
+
+        self.rin.write(b"0008NAK\n0000")
+        self.rin.seek(0)
+
+        class DummyGraphWalker:
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                return None
+
+        proto = Protocol(self.rin.read, self.rout.write)
+        capabilities = [b"shallow", b"deepen-not"]
+        wants = [b"55dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7"]
+        graph_walker = DummyGraphWalker()
+
+        _handle_upload_pack_head(
+            proto=proto,
+            capabilities=capabilities,
+            graph_walker=graph_walker,
+            wants=wants,
+            can_read=None,
+            depth=None,
+            protocol_version=0,
+            shallow_exclude=["refs/heads/excluded"],
+        )
+
+        # Verify the deepen-not command was sent
+        output = self.rout.getvalue()
+        self.assertIn(b"deepen-not refs/heads/excluded\n", output)
+
+    def test_handle_upload_pack_head_deepen_not_multiple(self) -> None:
+        # Test that multiple deepen-not commands are properly sent
+        from dulwich.client import _handle_upload_pack_head
+
+        self.rin.write(b"0008NAK\n0000")
+        self.rin.seek(0)
+
+        class DummyGraphWalker:
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                return None
+
+        proto = Protocol(self.rin.read, self.rout.write)
+        capabilities = [b"shallow", b"deepen-not"]
+        wants = [b"55dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7"]
+        graph_walker = DummyGraphWalker()
+
+        _handle_upload_pack_head(
+            proto=proto,
+            capabilities=capabilities,
+            graph_walker=graph_walker,
+            wants=wants,
+            can_read=None,
+            depth=None,
+            protocol_version=0,
+            shallow_exclude=["refs/heads/excluded1", "refs/heads/excluded2"],
+        )
+
+        # Verify both deepen-not commands were sent
+        output = self.rout.getvalue()
+        self.assertIn(b"deepen-not refs/heads/excluded1\n", output)
+        self.assertIn(b"deepen-not refs/heads/excluded2\n", output)
+
+    def test_handle_upload_pack_head_deepen_since_and_not(self) -> None:
+        # Test that deepen-since and deepen-not can be used together
+        from dulwich.client import _handle_upload_pack_head
+
+        self.rin.write(b"0008NAK\n0000")
+        self.rin.seek(0)
+
+        class DummyGraphWalker:
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                return None
+
+        proto = Protocol(self.rin.read, self.rout.write)
+        capabilities = [b"shallow", b"deepen-since", b"deepen-not"]
+        wants = [b"55dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7"]
+        graph_walker = DummyGraphWalker()
+
+        _handle_upload_pack_head(
+            proto=proto,
+            capabilities=capabilities,
+            graph_walker=graph_walker,
+            wants=wants,
+            can_read=None,
+            depth=None,
+            protocol_version=0,
+            shallow_since="2023-01-01T00:00:00Z",
+            shallow_exclude=["refs/heads/excluded"],
+        )
+
+        # Verify both deepen-since and deepen-not commands were sent
+        output = self.rout.getvalue()
+        self.assertIn(b"deepen-since 2023-01-01T00:00:00Z\n", output)
+        self.assertIn(b"deepen-not refs/heads/excluded\n", output)
+
     def test_send_pack_no_sideband64k_with_update_ref_error(self) -> None:
         # No side-bank-64k reported by server shouldn't try to parse
         # side band data