Merge branch 'ignore'

NEWS

@@ -14,6 +14,12 @@
    add from current working directory rather than repository root.
    (Jelmer Vernooij, #521)
 
+ IMPROVEMENTS
+
+  * Add basic support for reading ignore files in ``dulwich.ignore``.
+    Note that this is not yet hooked into the other parts of Dulwich.
+    (Jelmer Vernooij, #524)
+
  DOCUMENTATION
 
   * Clarified docstrings for Client.{send_pack,fetch_pack} implementations.

dulwich/ignore.py

@@ -0,0 +1,175 @@
+# Copyright (C) 2017 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+# General Public License as public by the Free Software Foundation; version 2.0
+# or (at your option) any later version. You can redistribute it and/or
+# modify it under the terms of either of these two licenses.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# You should have received a copy of the licenses; if not, see
+# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+# License, Version 2.0.
+#
+
+"""Parsing of gitignore files.
+
+For details for the matching rules, see https://git-scm.com/docs/gitignore
+"""
+
+import re
+
+
+def translate(pat):
+    """Translate a shell PATTERN to a regular expression.
+
+    There is no way to quote meta-characters.
+
+    Originally copied from fnmatch in Python 2.7, but modified for Dulwich
+    to cope with features in Git ignore patterns.
+    """
+
+    res = ''
+
+    if '/' not in pat:
+        # If there's no slash, this is a filename-based match
+        res = '(.*\/)?'
+
+    if pat.startswith('**/'):
+        # Leading **/
+        pat = pat[2:]
+        res = '(.*\/)?'
+
+    if pat.startswith(b'/'):
+        pat = pat[1:]
+
+    i, n = 0, len(pat)
+
+    while i < n:
+        if pat[i:i+3] == b'/**':
+            res = res + '(\\/.*)?'
+            i = i+3
+            continue
+        c = pat[i]
+        i = i+1
+        if c == '*':
+            res = res + '[^\/]+'
+        elif c == '?':
+            res = res + '.'
+        elif c == '[':
+            j = i
+            if j < n and pat[j] == '!':
+                j = j+1
+            if j < n and pat[j] == ']':
+                j = j+1
+            while j < n and pat[j] != ']':
+                j = j+1
+            if j >= n:
+                res = res + '\\['
+            else:
+                stuff = pat[i:j].replace('\\','\\\\')
+                i = j+1
+                if stuff[0] == '!':
+                    stuff = '^' + stuff[1:]
+                elif stuff[0] == '^':
+                    stuff = '\\' + stuff
+                res = '%s[%s]' % (res, stuff)
+        else:
+            res = res + re.escape(c)
+    return res + '\Z(?ms)'
+
+
+def read_ignore_patterns(f):
+    """Read a git ignore file.
+
+    :param f: File-like object to read from
+    :return: List of patterns
+    """
+
+    for l in f:
+        l = l.rstrip(b"\n")
+
+        # Ignore blank lines, they're used for readability.
+        if not l:
+            continue
+
+        if l[0:1] == b'#':
+            # Comment
+            continue
+
+        # Trailing spaces are ignored unless they are quoted with a backslash.
+        while l.endswith(b' ') and not l.endswith(b'\\ '):
+            l = l[:-1]
+        l = l.replace(b'\\ ', b' ')
+
+        yield l
+
+
+def match_pattern(path, pattern):
+    """Match a gitignore-style pattern against a path.
+
+    :param path: Path to match
+    :param pattern: Pattern to match
+    :return: bool indicating whether the pattern matched
+    """
+    re_pattern = translate(pattern)
+    return re.match(re_pattern, path)
+
+
+class IgnoreFilter(object):
+
+    def __init__(self, patterns):
+        self._patterns = []
+        for pattern in patterns:
+            self.append_pattern(pattern)
+
+    def append_pattern(self, pattern):
+        """Add a pattern to the set."""
+        self._patterns.append(pattern)
+
+    def is_ignored(self, path):
+        """Check whether a path is ignored.
+
+        For directories, include a trailing slash.
+
+        :return: None if file is not mentioned, True if it is included, False
+            if it is explicitly excluded.
+        """
+        status = None
+        for pattern in self._patterns:
+            if pattern[0:1] == b'!':
+                if match_pattern(pattern[1:], path):
+                    # Explicitly excluded.
+                    return False
+            else:
+                if pattern[0:1] == b'\\':
+                    pattern = pattern[1:]
+                if match_pattern(pattern, path):
+                    status = True
+        return status
+
+
+class IgnoreFilterStack(object):
+    """Check for ignore status in multiple filters."""
+
+    def __init__(self, filters):
+        self._filters = filters
+
+    def is_ignored(self, path):
+        """Check whether a path is explicitly included or excluded in ignores.
+
+        :param path: Path to check
+        :return: None if the file is not mentioned, True if it is included,
+            False if it is explicitly excluded.
+        """
+        status = None
+        for filter in self._filters:
+            status = filter.is_ignored(path)
+            if status is not None:
+                return status
+        return status

dulwich/tests/__init__.py

@@ -101,6 +101,7 @@ def self_test_suite():
         'grafts',
         'greenthreads',
         'hooks',
+        'ignore',
         'index',
         'lru_cache',
         'objects',

dulwich/tests/test_ignore.py

@@ -0,0 +1,142 @@
+# test_ignore.py -- Tests for ignore files.
+# Copyright (C) 2017 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+# General Public License as public by the Free Software Foundation; version 2.0
+# or (at your option) any later version. You can redistribute it and/or
+# modify it under the terms of either of these two licenses.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# You should have received a copy of the licenses; if not, see
+# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+# License, Version 2.0.
+#
+
+"""Tests for ignore files."""
+
+from io import BytesIO
+import unittest
+
+from dulwich.ignore import (
+    IgnoreFilter,
+    IgnoreFilterStack,
+    match_pattern,
+    read_ignore_patterns,
+    translate,
+    )
+
+
+POSITIVE_MATCH_TESTS = [
+    ("foo.c", "*.c"),
+    ("foo/foo.c", "*.c"),
+    ("foo/foo.c", "foo.c"),
+    ("foo.c", "/*.c"),
+    ("foo.c", "/foo.c"),
+    ("foo.c", "foo.c"),
+    ("foo.c", "foo.[ch]"),
+    ("foo/bar/bla.c", "foo/**"),
+    ("foo/bar/bla/blie.c", "foo/**/blie.c"),
+    ("foo/bar/bla.c", "**/bla.c"),
+    ("bla.c", "**/bla.c"),
+    ("foo/bar", "foo/**/bar"),
+    ("foo/bla/bar", "foo/**/bar"),
+]
+
+NEGATIVE_MATCH_TESTS = [
+    ("foo.c", "foo.[dh]"),
+    ("foo/foo.c", "/foo.c"),
+    ("foo/foo.c", "/*.c"),
+]
+
+
+TRANSLATE_TESTS = [
+    ("*.c", '(.*\\/)?[^\\/]+\\.c\\Z(?ms)'),
+    ("foo.c", '(.*\\/)?foo\\.c\\Z(?ms)'),
+    ("/*.c", '[^\\/]+\\.c\\Z(?ms)'),
+    ("/foo.c", 'foo\\.c\\Z(?ms)'),
+    ("foo.c", '(.*\\/)?foo\\.c\\Z(?ms)'),
+    ("foo.[ch]", '(.*\\/)?foo\\.[ch]\\Z(?ms)'),
+    ("foo/**", 'foo(\\/.*)?\\Z(?ms)'),
+    ("foo/**/blie.c", 'foo(\\/.*)?\\/blie\\.c\\Z(?ms)'),
+    ("**/bla.c", '(.*\\/)?bla\\.c\\Z(?ms)'),
+    ("foo/**/bar", 'foo(\\/.*)?\\/bar\\Z(?ms)'),
+]
+
+
+class TranslateTests(unittest.TestCase):
+
+    def test_translate(self):
+        for (pattern, regex) in TRANSLATE_TESTS:
+            self.assertEqual(
+                regex, translate(pattern),
+                "orig pattern: %r, regex: %r, expected: %r" %
+                (pattern, translate(pattern), regex))
+
+
+class ReadIgnorePatterns(unittest.TestCase):
+
+    def test_read_file(self):
+        f = BytesIO(b"""
+# a comment
+
+# and an empty line:
+
+\#not a comment
+!negative
+with trailing whitespace 
+with escaped trailing whitespace\ 
+""")
+        self.assertEqual(list(read_ignore_patterns(f)), [
+            '\\#not a comment',
+            '!negative',
+            'with trailing whitespace',
+            'with escaped trailing whitespace '
+        ])
+
+
+class MatchPatternTests(unittest.TestCase):
+
+    def test_matches(self):
+        for (path, pattern) in POSITIVE_MATCH_TESTS:
+            self.assertTrue(
+                match_pattern(path, pattern),
+                "path: %r, pattern: %r" % (path, pattern))
+
+    def test_no_matches(self):
+        for (path, pattern) in NEGATIVE_MATCH_TESTS:
+            self.assertFalse(
+                match_pattern(path, pattern),
+                "path: %r, pattern: %r" % (path, pattern))
+
+
+class IgnoreFilterTests(unittest.TestCase):
+
+    def test_included(self):
+        filter = IgnoreFilter(['a.c', 'b.c'])
+        self.assertTrue(filter.is_ignored('a.c'))
+        self.assertIs(None, filter.is_ignored('c.c'))
+
+    def test_excluded(self):
+        filter = IgnoreFilter(['a.c', 'b.c', '!c.c'])
+        self.assertFalse(filter.is_ignored('c.c'))
+        self.assertIs(None, filter.is_ignored('d.c'))
+
+
+class IgnoreFilterStackTests(unittest.TestCase):
+
+    def test_stack_first(self):
+        filter1 = IgnoreFilter(['a.c', 'b.c', '!d.c'])
+        filter2 = IgnoreFilter(['a.c', '!b,c', 'c.c', 'd.c'])
+        stack = IgnoreFilterStack([filter1, filter2])
+        self.assertIs(True, stack.is_ignored('a.c'))
+        self.assertIs(True, stack.is_ignored('b.c'))
+        self.assertIs(True, stack.is_ignored('c.c'))
+        self.assertIs(False, stack.is_ignored('d.c'))
+        self.assertIs(None, stack.is_ignored('e.c'))
+