Add porcelain submodule commands (#1647)

- Implement submodule_update() in porcelain to clone/fetch and checkout
submodules
- Add CLI commands: submodule add, submodule update
- Add --recurse-submodules option to clone command

Fixes #506

NEWS

@@ -10,6 +10,10 @@
    functionality works as expected.
    (Jelmer Vernooij, #780)
 
+ * Add porcelain submodule commands: ``submodule_update``, ``submodule_add`` 
+   CLI command, and ``submodule_update`` CLI command. Add ``--recurse-submodules``
+   option to ``clone`` command. (#506, Jelmer Vernooij)
+
 0.23.1	2025-06-30
 
  * Support ``untracked_files="normal"`` argument to ``porcelain.status``,

dulwich/cli.py

@@ -393,6 +393,11 @@ class cmd_clone(Command):
             type=int,
             help="Git protocol version to use",
         )
+        parser.add_argument(
+            "--recurse-submodules",
+            action="store_true",
+            help="Initialize and clone submodules",
+        )
         parser.add_argument("source", help="Repository to clone from")
         parser.add_argument("target", nargs="?", help="Directory to clone into")
         args = parser.parse_args(args)
@@ -407,6 +412,7 @@ class cmd_clone(Command):
                 refspec=args.refspec,
                 filter_spec=args.filter_spec,
                 protocol_version=args.protocol,
+                recurse_submodules=args.recurse_submodules,
             )
         except GitProtocolError as e:
             print(f"{e}")
@@ -921,10 +927,41 @@ class cmd_submodule_init(Command):
         porcelain.submodule_init(".")
 
 
+class cmd_submodule_add(Command):
+    def run(self, argv) -> None:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("url", help="URL of repository to add as submodule")
+        parser.add_argument("path", nargs="?", help="Path where submodule should live")
+        parser.add_argument("--name", help="Name for the submodule")
+        args = parser.parse_args(argv)
+        porcelain.submodule_add(".", args.url, args.path, args.name)
+
+
+class cmd_submodule_update(Command):
+    def run(self, argv) -> None:
+        parser = argparse.ArgumentParser()
+        parser.add_argument(
+            "--init", action="store_true", help="Initialize submodules first"
+        )
+        parser.add_argument(
+            "--force",
+            action="store_true",
+            help="Force update even if local changes exist",
+        )
+        parser.add_argument(
+            "paths", nargs="*", help="Specific submodule paths to update"
+        )
+        args = parser.parse_args(argv)
+        paths = args.paths if args.paths else None
+        porcelain.submodule_update(".", paths=paths, init=args.init, force=args.force)
+
+
 class cmd_submodule(SuperCommand):
     subcommands: ClassVar[dict[str, type[Command]]] = {
+        "add": cmd_submodule_add,
         "init": cmd_submodule_init,
         "list": cmd_submodule_list,
+        "update": cmd_submodule_update,
     }
 
     default_command = cmd_submodule_list

dulwich/porcelain.py

@@ -530,6 +530,7 @@ def clone(
     config: Optional[Config] = None,
     filter_spec=None,
     protocol_version: Optional[int] = None,
+    recurse_submodules: bool = False,
     **kwargs,
 ):
     """Clone a local or remote git repository.
@@ -551,6 +552,7 @@ def clone(
         feature, and ignored otherwise.
       protocol_version: desired Git protocol version. By default the highest
         mutually supported protocol version will be used.
+      recurse_submodules: Whether to initialize and clone submodules
 
     Keyword Args:
       refspecs: refspecs to fetch. Can be a bytestring, a string, or a list of
@@ -589,7 +591,7 @@ def clone(
     if filter_spec:
         filter_spec = filter_spec.encode("ascii")
 
-    return client.clone(
+    repo = client.clone(
         path,
         target,
         mkdir=mkdir,
@@ -603,6 +605,28 @@ def clone(
         protocol_version=protocol_version,
     )
 
+    # Initialize and update submodules if requested
+    if recurse_submodules and not bare:
+        try:
+            submodule_init(repo)
+            submodule_update(repo, init=True)
+        except FileNotFoundError as e:
+            # .gitmodules file doesn't exist - no submodules to process
+            import logging
+
+            logging.debug("No .gitmodules file found: %s", e)
+        except KeyError as e:
+            # Submodule configuration missing
+            import logging
+
+            logging.warning("Submodule configuration error: %s", e)
+            if errstream:
+                errstream.write(
+                    f"Warning: Submodule configuration error: {e}\n".encode()
+                )
+
+    return repo
+
 
 def add(repo: Union[str, os.PathLike, BaseRepo] = ".", paths=None):
     """Add files to the staging area.
@@ -1209,6 +1233,7 @@ def submodule_add(repo, url, path=None, name=None) -> None:
       repo: Path to repository
       url: URL of repository to add as submodule
       path: Path where submodule should live
+      name: Name for the submodule
     """
     with open_repo_closing(repo) as r:
         if path is None:
@@ -1256,6 +1281,130 @@ def submodule_list(repo):
             yield path, sha.decode(DEFAULT_ENCODING)
 
 
+def submodule_update(repo, paths=None, init=False, force=False, errstream=None) -> None:
+    """Update submodules.
+
+    Args:
+      repo: Path to repository
+      paths: Optional list of specific submodule paths to update. If None, updates all.
+      init: If True, initialize submodules first
+      force: Force update even if local changes exist
+    """
+    from .client import get_transport_and_path
+    from .index import build_index_from_tree
+    from .submodule import iter_cached_submodules
+
+    with open_repo_closing(repo) as r:
+        if init:
+            submodule_init(r)
+
+        config = r.get_config()
+        gitmodules_path = os.path.join(r.path, ".gitmodules")
+
+        # Get list of submodules to update
+        submodules_to_update = []
+        for path, sha in iter_cached_submodules(r.object_store, r[r.head()].tree):
+            path_str = (
+                path.decode(DEFAULT_ENCODING) if isinstance(path, bytes) else path
+            )
+            if paths is None or path_str in paths:
+                submodules_to_update.append((path, sha))
+
+        # Read submodule configuration
+        for path, target_sha in submodules_to_update:
+            path_str = (
+                path.decode(DEFAULT_ENCODING) if isinstance(path, bytes) else path
+            )
+
+            # Find the submodule name from .gitmodules
+            submodule_name = None
+            for sm_path, sm_url, sm_name in read_submodules(gitmodules_path):
+                if sm_path == path:
+                    submodule_name = sm_name
+                    break
+
+            if not submodule_name:
+                continue
+
+            # Get the URL from config
+            section = (
+                b"submodule",
+                submodule_name
+                if isinstance(submodule_name, bytes)
+                else submodule_name.encode(),
+            )
+            try:
+                url = config.get(section, b"url")
+                if isinstance(url, bytes):
+                    url = url.decode(DEFAULT_ENCODING)
+            except KeyError:
+                # URL not in config, skip this submodule
+                continue
+
+            # Get or create the submodule repository paths
+            submodule_path = os.path.join(r.path, path_str)
+            submodule_git_dir = os.path.join(r.path, ".git", "modules", path_str)
+
+            # Clone or fetch the submodule
+            if not os.path.exists(submodule_git_dir):
+                # Clone the submodule as bare repository
+                os.makedirs(os.path.dirname(submodule_git_dir), exist_ok=True)
+
+                # Clone to the git directory
+                sub_repo = clone(url, submodule_git_dir, bare=True, checkout=False)
+                sub_repo.close()
+
+                # Create the submodule directory if it doesn't exist
+                if not os.path.exists(submodule_path):
+                    os.makedirs(submodule_path)
+
+                # Create .git file in the submodule directory
+                depth = path_str.count("/") + 1
+                relative_git_dir = "../" * depth + ".git/modules/" + path_str
+                git_file_path = os.path.join(submodule_path, ".git")
+                with open(git_file_path, "w") as f:
+                    f.write(f"gitdir: {relative_git_dir}\n")
+
+                # Set up working directory configuration
+                with open_repo_closing(submodule_git_dir) as sub_repo:
+                    sub_config = sub_repo.get_config()
+                    sub_config.set(
+                        (b"core",),
+                        b"worktree",
+                        os.path.abspath(submodule_path).encode(),
+                    )
+                    sub_config.write_to_path()
+
+                    # Checkout the target commit
+                    sub_repo.refs[b"HEAD"] = target_sha
+
+                    # Build the index and checkout files
+                    tree = sub_repo[target_sha]
+                    if hasattr(tree, "tree"):  # If it's a commit, get the tree
+                        tree_id = tree.tree
+                    else:
+                        tree_id = target_sha
+
+                    build_index_from_tree(
+                        submodule_path,
+                        sub_repo.index_path(),
+                        sub_repo.object_store,
+                        tree_id,
+                    )
+            else:
+                # Fetch and checkout in existing submodule
+                with open_repo_closing(submodule_git_dir) as sub_repo:
+                    # Fetch from remote
+                    client, path_segments = get_transport_and_path(url)
+                    client.fetch(path_segments, sub_repo)
+
+                    # Update to the target commit
+                    sub_repo.refs[b"HEAD"] = target_sha
+
+                    # Reset the working directory
+                    reset(sub_repo, "hard", target_sha)
+
+
 def tag_create(
     repo,
     tag: Union[str, bytes],

tests/test_porcelain.py

@@ -908,6 +908,85 @@ class CloneTests(PorcelainTestCase):
         self.assertEqual(r.path, str(target_path))
         self.assertTrue(os.path.exists(str(target_path)))
 
+    def test_clone_with_recurse_submodules(self) -> None:
+        # Create a submodule repository
+        sub_repo_path = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, sub_repo_path)
+        sub_repo = Repo.init(sub_repo_path)
+        self.addCleanup(sub_repo.close)
+
+        # Add a file to the submodule repo
+        sub_file = os.path.join(sub_repo_path, "subfile.txt")
+        with open(sub_file, "w") as f:
+            f.write("submodule content")
+
+        porcelain.add(sub_repo, paths=[sub_file])
+        sub_commit = porcelain.commit(
+            sub_repo,
+            message=b"Initial submodule commit",
+            author=b"Test Author <test@example.com>",
+            committer=b"Test Committer <test@example.com>",
+        )
+
+        # Create main repository with submodule
+        main_file = os.path.join(self.repo.path, "main.txt")
+        with open(main_file, "w") as f:
+            f.write("main content")
+
+        porcelain.add(self.repo, paths=[main_file])
+        porcelain.submodule_add(self.repo, sub_repo_path, "sub")
+
+        # Manually add the submodule to the index since submodule_add doesn't do it
+        # when the repository is local (to maintain backward compatibility)
+        from dulwich.index import IndexEntry
+        from dulwich.objects import S_IFGITLINK
+
+        index = self.repo.open_index()
+        index[b"sub"] = IndexEntry(
+            ctime=0,
+            mtime=0,
+            dev=0,
+            ino=0,
+            mode=S_IFGITLINK,
+            uid=0,
+            gid=0,
+            size=0,
+            sha=sub_commit,
+            flags=0,
+        )
+        index.write()
+
+        porcelain.add(self.repo, paths=[".gitmodules"])
+        porcelain.commit(
+            self.repo,
+            message=b"Add submodule",
+            author=b"Test Author <test@example.com>",
+            committer=b"Test Committer <test@example.com>",
+        )
+
+        # Clone with recurse_submodules
+        target_path = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, target_path)
+
+        cloned = porcelain.clone(
+            self.repo.path,
+            target_path,
+            recurse_submodules=True,
+        )
+        self.addCleanup(cloned.close)
+
+        # Check main file exists
+        cloned_main = os.path.join(target_path, "main.txt")
+        self.assertTrue(os.path.exists(cloned_main))
+        with open(cloned_main) as f:
+            self.assertEqual(f.read(), "main content")
+
+        # Check submodule file exists
+        cloned_sub_file = os.path.join(target_path, "sub", "subfile.txt")
+        self.assertTrue(os.path.exists(cloned_sub_file))
+        with open(cloned_sub_file) as f:
+            self.assertEqual(f.read(), "submodule content")
+
 
 class InitTests(TestCase):
     def test_non_bare(self) -> None:
@@ -3531,6 +3610,70 @@ class SubmoduleTests(PorcelainTestCase):
         porcelain.submodule_add(self.repo, "../bar.git", "bar")
         porcelain.submodule_init(self.repo)
 
+    def test_update(self) -> None:
+        # Create a submodule repository
+        sub_repo_path = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, sub_repo_path)
+        sub_repo = Repo.init(sub_repo_path)
+        self.addCleanup(sub_repo.close)
+
+        # Add a file to the submodule repo
+        sub_file = os.path.join(sub_repo_path, "test.txt")
+        with open(sub_file, "w") as f:
+            f.write("submodule content")
+
+        porcelain.add(sub_repo, paths=[sub_file])
+        sub_commit = porcelain.commit(
+            sub_repo,
+            message=b"Initial submodule commit",
+            author=b"Test Author <test@example.com>",
+            committer=b"Test Committer <test@example.com>",
+        )
+
+        # Add the submodule to the main repository
+        porcelain.submodule_add(self.repo, sub_repo_path, "test_submodule")
+
+        # Manually add the submodule to the index
+        from dulwich.index import IndexEntry
+        from dulwich.objects import S_IFGITLINK
+
+        index = self.repo.open_index()
+        index[b"test_submodule"] = IndexEntry(
+            ctime=0,
+            mtime=0,
+            dev=0,
+            ino=0,
+            mode=S_IFGITLINK,
+            uid=0,
+            gid=0,
+            size=0,
+            sha=sub_commit,
+            flags=0,
+        )
+        index.write()
+
+        porcelain.add(self.repo, paths=[".gitmodules"])
+        porcelain.commit(
+            self.repo,
+            message=b"Add submodule",
+            author=b"Test Author <test@example.com>",
+            committer=b"Test Committer <test@example.com>",
+        )
+
+        # Initialize and update the submodule
+        porcelain.submodule_init(self.repo)
+        porcelain.submodule_update(self.repo)
+
+        # Check that the submodule directory exists
+        submodule_path = os.path.join(self.repo.path, "test_submodule")
+        self.assertTrue(os.path.exists(submodule_path))
+
+        # Check that the submodule file exists
+        submodule_file = os.path.join(submodule_path, "test.txt")
+        self.assertTrue(os.path.exists(submodule_file))
+        with open(submodule_file) as f:
+            self.assertEqual(f.read(), "submodule content")
+
 
 class PushTests(PorcelainTestCase):
     def test_simple(self) -> None: