Merge branch 'status-ignored-dir' of git://github.com/pmrowla/dulwich

NEWS

@@ -6,6 +6,11 @@
  * Fix filename: MERGE_HEADS => MERGE_HEAD.
    (Jelmer Vernooij, #861)
 
+ * For ignored directories, porcelain.add and porcelain.status now only return
+   the path to directory itself in the list of ignored paths. Previously, paths
+   for all files within the directory would also be included in the list.
+   (Peter Rowlands, #853)
+
 0.20.21	2021-03-20
 
  * Add basic support for a GcsObjectStore that stores

dulwich/porcelain.py

@@ -501,6 +501,10 @@ def add(repo=".", paths=None):
       repo: Repository for the files
       paths: Paths to add.  No value passed stages all modified files.
     Returns: Tuple with set of added files and ignored files
+
+    If the repository contains ignored directories, the returned set will
+    contain the path to an ignored directory (with trailing slash). Individual
+    files within ignored directories will not be returned.
     """
     ignored = set()
     with open_repo_closing(repo) as r:
@@ -519,7 +523,9 @@ def add(repo=".", paths=None):
             paths = [paths]
         for p in paths:
             relpath = str(Path(p).resolve().relative_to(repo_path))
-            # FIXME: Support patterns, directories.
+            # FIXME: Support patterns
+            if os.path.isdir(p):
+                relpath = os.path.join(relpath, "")
             if ignore_manager.is_ignored(relpath):
                 ignored.add(relpath)
                 continue
@@ -1217,12 +1223,14 @@ def status(repo=".", ignored=False):
         return GitStatus(tracked_changes, unstaged_changes, untracked_changes)
 
 
-def _walk_working_dir_paths(frompath, basepath):
+def _walk_working_dir_paths(frompath, basepath, prune_dirnames=None):
     """Get path, is_dir for files in working dir from frompath
 
     Args:
       frompath: Path to begin walk
       basepath: Path to compare to
+      prune_dirnames: Optional callback to prune dirnames during os.walk
+        dirnames will be set to result of prune_dirnames(dirpath, dirnames)
     """
     for dirpath, dirnames, filenames in os.walk(frompath):
         # Skip .git and below.
@@ -1230,6 +1238,7 @@ def _walk_working_dir_paths(frompath, basepath):
             dirnames.remove(".git")
             if dirpath != basepath:
                 continue
+
         if ".git" in filenames:
             filenames.remove(".git")
             if dirpath != basepath:
@@ -1242,6 +1251,9 @@ def _walk_working_dir_paths(frompath, basepath):
             filepath = os.path.join(dirpath, filename)
             yield filepath, False
 
+        if prune_dirnames:
+            dirnames[:] = prune_dirnames(dirpath, dirnames)
+
 
 def get_untracked_paths(frompath, basepath, index, exclude_ignored=False):
     """Get untracked paths.
@@ -1251,22 +1263,43 @@ def get_untracked_paths(frompath, basepath, index, exclude_ignored=False):
       basepath: Path to compare to
       index: Index to check against
       exclude_ignored: Whether to exclude ignored paths
+
+    Note: ignored directories will never be walked for performance reasons.
+      If exclude_ignored is False, only the path to an ignored directory will
+      be yielded, no files inside the directory will be returned
     """
-    if exclude_ignored:
-        with open_repo_closing(frompath) as r:
-            ignore_manager = IgnoreFilterManager.from_repo(r)
-    else:
-        ignore_manager = None
+    with open_repo_closing(basepath) as r:
+        ignore_manager = IgnoreFilterManager.from_repo(r)
 
-    for ap, is_dir in _walk_working_dir_paths(frompath, basepath):
-        if ignore_manager is not None and ignore_manager.is_ignored(
-            os.path.relpath(ap, frompath)
-        ):
-            continue
+    ignored_dirs = []
+
+    def prune_dirnames(dirpath, dirnames):
+        for i in range(len(dirnames) - 1, -1, -1):
+            path = os.path.join(dirpath, dirnames[i])
+            ip = os.path.join(os.path.relpath(path, basepath), "")
+            if ignore_manager.is_ignored(ip):
+                if not exclude_ignored:
+                    ignored_dirs.append(
+                        os.path.join(os.path.relpath(path, frompath), "")
+                    )
+                del dirnames[i]
+        return dirnames
+
+    for ap, is_dir in _walk_working_dir_paths(
+        frompath, basepath, prune_dirnames=prune_dirnames
+    ):
         if not is_dir:
             ip = path_to_tree_path(basepath, ap)
             if ip not in index:
-                yield os.path.relpath(ap, frompath)
+                if (
+                    not exclude_ignored
+                    or not ignore_manager.is_ignored(
+                        os.path.relpath(ap, basepath)
+                    )
+                ):
+                    yield os.path.relpath(ap, frompath)
+
+    yield from ignored_dirs
 
 
 def get_tree_changes(repo):

dulwich/tests/test_porcelain.py

@@ -524,21 +524,25 @@ class AddTests(PorcelainTestCase):
 
     def test_add_ignored(self):
         with open(os.path.join(self.repo.path, ".gitignore"), "w") as f:
-            f.write("foo")
+            f.write("foo\nsubdir/")
         with open(os.path.join(self.repo.path, "foo"), "w") as f:
             f.write("BAR")
         with open(os.path.join(self.repo.path, "bar"), "w") as f:
             f.write("BAR")
+        os.mkdir(os.path.join(self.repo.path, "subdir"))
+        with open(os.path.join(self.repo.path, "subdir", "baz"), "w") as f:
+            f.write("BAZ")
         (added, ignored) = porcelain.add(
             self.repo.path,
             paths=[
                 os.path.join(self.repo.path, "foo"),
                 os.path.join(self.repo.path, "bar"),
+                os.path.join(self.repo.path, "subdir"),
             ],
         )
         self.assertIn(b"bar", self.repo.open_index())
         self.assertEqual(set(["bar"]), set(added))
-        self.assertEqual(set(["foo"]), ignored)
+        self.assertEqual(set(["foo", os.path.join("subdir", "")]), ignored)
 
     def test_add_file_absolute_path(self):
         # Absolute paths are (not yet) supported
@@ -1674,7 +1678,7 @@ class StatusTests(PorcelainTestCase):
             set(porcelain.status(self.repo, ignored=True).untracked),
         )
 
-    def test_get_untracked_paths_nested(self):
+    def test_get_untracked_paths_subrepo(self):
         with open(os.path.join(self.repo.path, ".gitignore"), "w") as f:
             f.write("nested/\n")
         with open(os.path.join(self.repo.path, "notignored"), "w") as f:
@@ -1689,13 +1693,24 @@ class StatusTests(PorcelainTestCase):
             f.write("blop\n")
 
         self.assertEqual(
-            set([".gitignore", "notignored"]),
+            set([".gitignore", "notignored", os.path.join("nested", "")]),
             set(
                 porcelain.get_untracked_paths(
                     self.repo.path, self.repo.path, self.repo.open_index()
                 )
             ),
         )
+        self.assertEqual(
+            set([".gitignore", "notignored"]),
+            set(
+                porcelain.get_untracked_paths(
+                    self.repo.path,
+                    self.repo.path,
+                    self.repo.open_index(),
+                    exclude_ignored=True,
+                )
+            ),
+        )
         self.assertEqual(
             set(["ignored", "with", "manager"]),
             set(
@@ -1705,32 +1720,66 @@ class StatusTests(PorcelainTestCase):
             ),
         )
         self.assertEqual(
+            set(),
             set(
-                [
-                    os.path.join("nested", "ignored"),
-                    os.path.join("nested", "with"),
-                    os.path.join("nested", "manager"),
-                ]
+                porcelain.get_untracked_paths(
+                    subrepo.path,
+                    self.repo.path,
+                    self.repo.open_index(),
+                )
             ),
+        )
+        self.assertEqual(
+            set([os.path.join('nested', 'ignored'),
+                os.path.join('nested', 'with'),
+                os.path.join('nested', 'manager')]),
             set(
                 porcelain.get_untracked_paths(
                     self.repo.path,
                     subrepo.path,
                     self.repo.open_index(),
-                    exclude_ignored=False,
                 )
             ),
         )
+
+    def test_get_untracked_paths_subdir(self):
+        with open(os.path.join(self.repo.path, ".gitignore"), "w") as f:
+            f.write("subdir/\nignored")
+        with open(os.path.join(self.repo.path, "notignored"), "w") as f:
+            f.write("blah\n")
+        os.mkdir(os.path.join(self.repo.path, "subdir"))
+        with open(os.path.join(self.repo.path, "ignored"), "w") as f:
+            f.write("foo")
+        with open(os.path.join(self.repo.path, "subdir", "ignored"), "w") as f:
+            f.write("foo")
+
+        self.assertEqual(
+            set(
+                [
+                    ".gitignore",
+                    "notignored",
+                    "ignored",
+                    os.path.join("subdir", ""),
+                ]
+            ),
+            set(
+                porcelain.get_untracked_paths(
+                    self.repo.path,
+                    self.repo.path,
+                    self.repo.open_index(),
+                )
+            )
+        )
         self.assertEqual(
-            set([]),
+            set([".gitignore", "notignored"]),
             set(
                 porcelain.get_untracked_paths(
                     self.repo.path,
-                    subrepo.path,
+                    self.repo.path,
                     self.repo.open_index(),
                     exclude_ignored=True,
                 )
-            ),
+            )
         )