Import upstream version 0.19.12, md5 4efcf6fd1903e9a5334b095333a77aa4

.travis.yml

@@ -8,12 +8,12 @@ python:
   - 3.4
   - 3.5
   - 3.6
-  - 3.6-dev
   - pypy3.5
 
 env:
   - PYTHONHASHSEED=random
     TEST_REQUIRE="gevent greenlet geventhttpclient fastimport"
+    PURE=false
 
 matrix:
   include:
@@ -23,6 +23,10 @@ matrix:
       env: TEST_REQUIRE=fastimport
       dist: xenial
       sudo: true
+    - python: 3.6
+      env: PURE=true
+    - python: 2.7
+      env: PURE=true
     # flakes checker fails on python 3.8-dev:
     #- python: 3.8-dev
     #  env: TEST_REQUIRE=fastimport
@@ -33,16 +37,30 @@ install:
   - travis_retry pip install -U pip coverage codecov flake8 $TEST_REQUIRE
 
 script:
-  # Test without c extensions
-  - python -m coverage run -p -m unittest dulwich.tests.test_suite
-
-  # Test with c extensions
-  - python setup.py build_ext -i
+  - if [ $PURE = false ]; then python setup.py build_ext -i; fi
   - python -m coverage run -p -m unittest dulwich.tests.test_suite
 
   # Style
   - make style
 
+  - if [ $PURE = true ]; then SETUP_ARGS=--pure; fi
+  - python setup.py $SETUP_ARGS bdist_wheel
+
 after_success:
   - python -m coverage combine
   - codecov
+
+deploy:
+  provider: pypi
+  user: dulwich-bot
+  password:
+    secure: Q8DDDojBugQWzXvmmEQiU90UkVPk+OYoFZwv1H9LYpQ4u5CfwQNWpf8qXYhlGMdr/gzWaSWsqLvgWLpzfkvqS4Vyk2bO9mr+dSskfD8uwc82LiiL9CNd/NY03CjH9RaFgVMD/+exMjY/yCtlyH1jL4kjgOyNnC+x4B37CliZHcE=
+  skip_cleanup: true
+  skip_existing: true
+  file_glob: true
+  file:
+    - dist/dulwich*.whl
+    - dist/dulwich*.tar.gz
+  on:
+    tags: true
+    repo: dulwich/dulwich

AUTHORS

@@ -147,5 +147,6 @@ Boris Feld <lothiraldan@gmail.com>
 KS Chan <mrkschan@gmail.com>
 egor <egor@sourced.tech>
 Antoine Lambert <anlambert@softwareheritage.org>
+Lane Barlow <lane.barlow@gmail.com>
 
 If you contributed but are missing from this list, please send me an e-mail.

NEWS

@@ -1,3 +1,26 @@
+0.19.12	2019-08-13
+
+ BUG FIXES
+
+ * Update directory detection for `get_unstaged_changes` for Python 3.
+   (Boris Feld, #684)
+
+ * Add a basic ``porcelain.clean``. (Lane Barlow, #398)
+
+ * Fix output format of ``porcelain.diff`` to match that of
+   C Git. (Boris Feld)
+
+ * Return a 404 not found error when repository is not found.
+
+ * Mark ``.git`` directories as hidden on Windows.
+   (Martin Packman, #585)
+
+ * Implement ``RefsContainer.__iter__``
+   (Jelmer Vernooij, #717)
+
+ * Don't trust modes if they can't be modified after a file has been created.
+   (Jelmer Vernooij, #719)
+
 0.19.11	2019-02-07
 
  IMPROVEMENTS

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.19.11
+Version: 0.19.12
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij
@@ -82,11 +82,8 @@ Description: .. image:: https://travis-ci.org/dulwich/dulwich.png?branch=master
         Further documentation
         ---------------------
         
-        The dulwich documentation can be found in docs/ and
-        `on the web <https://www.dulwich.io/docs/>`_.
-        
-        The API reference can be generated using pydoctor, by running "make pydoctor",
-        or `on the web <https://www.dulwich.io/apidocs>`_.
+        The dulwich documentation can be found in docs/ and built by running ``make
+        doc``. It can also be found `on the web <https://www.dulwich.io/docs/>`_.
         
         Help
         ----
@@ -107,7 +104,8 @@ Description: .. image:: https://travis-ci.org/dulwich/dulwich.png?branch=master
         Supported versions of Python
         ----------------------------
         
-        At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6 and Pypy.
+        At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6,
+        3.7 and Pypy.
         
 Keywords: git vcs
 Platform: UNKNOWN

README.rst

@@ -71,11 +71,8 @@ And to print it using porcelain::
 Further documentation
 ---------------------
 
-The dulwich documentation can be found in docs/ and
-`on the web <https://www.dulwich.io/docs/>`_.
-
-The API reference can be generated using pydoctor, by running "make pydoctor",
-or `on the web <https://www.dulwich.io/apidocs>`_.
+The dulwich documentation can be found in docs/ and built by running ``make
+doc``. It can also be found `on the web <https://www.dulwich.io/docs/>`_.
 
 Help
 ----
@@ -96,4 +93,5 @@ file and `list of open issues <https://github.com/dulwich/dulwich/issues>`_.
 Supported versions of Python
 ----------------------------
 
-At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6 and Pypy.
+At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6,
+3.7 and Pypy.

appveyor.yml

@@ -1,5 +1,10 @@
 environment:
 
+  TWINE_USERNAME: "dulwich-bot"
+  TWINE_PASSWORD:
+    # See https://www.appveyor.com/docs/build-configuration/#secure-variables
+    secure: e7DTu4CwOCARfN/mwA8lXQ==
+
   matrix:
 
     - PYTHON: "C:\\Python27"
@@ -10,15 +15,6 @@ environment:
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "64"
 
-    - PYTHON: "C:\\Python34"
-      PYTHON_VERSION: "3.4.x"
-      PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python34-x64"
-      PYTHON_VERSION: "3.4.x"
-      PYTHON_ARCH: "64"
-      DISTUTILS_USE_SDK: "1"
-
     - PYTHON: "C:\\Python35"
       PYTHON_VERSION: "3.5.x"
       PYTHON_ARCH: "32"
@@ -84,5 +80,9 @@ after_test:
   - "build.cmd %PYTHON%\\python.exe setup.py bdist_msi"
   - ps: "ls dist"
 
+deploy_script:
+  - if "%APPVEYOR_REPO_TAG%"=="true" pip install twine
+  - if "%APPVEYOR_REPO_TAG%"=="true" twine upload dist\dulwich-*.whl
+
 artifacts:
   - path: dist\*

dulwich.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.19.11
+Version: 0.19.12
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij
@@ -82,11 +82,8 @@ Description: .. image:: https://travis-ci.org/dulwich/dulwich.png?branch=master
         Further documentation
         ---------------------
         
-        The dulwich documentation can be found in docs/ and
-        `on the web <https://www.dulwich.io/docs/>`_.
-        
-        The API reference can be generated using pydoctor, by running "make pydoctor",
-        or `on the web <https://www.dulwich.io/apidocs>`_.
+        The dulwich documentation can be found in docs/ and built by running ``make
+        doc``. It can also be found `on the web <https://www.dulwich.io/docs/>`_.
         
         Help
         ----
@@ -107,7 +104,8 @@ Description: .. image:: https://travis-ci.org/dulwich/dulwich.png?branch=master
         Supported versions of Python
         ----------------------------
         
-        At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6 and Pypy.
+        At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6,
+        3.7 and Pypy.
         
 Keywords: git vcs
 Platform: UNKNOWN

dulwich.egg-info/SOURCES.txt

@@ -128,6 +128,7 @@ dulwich/tests/compat/__init__.py
 dulwich/tests/compat/server_utils.py
 dulwich/tests/compat/test_client.py
 dulwich/tests/compat/test_pack.py
+dulwich/tests/compat/test_patch.py
 dulwich/tests/compat/test_repository.py
 dulwich/tests/compat/test_server.py
 dulwich/tests/compat/test_utils.py

dulwich.egg-info/requires.txt

@@ -1,11 +1,11 @@
 certifi
-urllib3>=1.23
+urllib3>=1.24.1
 
 [fastimport]
 fastimport
 
 [https]
-urllib3[secure]>=1.23
+urllib3[secure]>=1.24.1
 
 [pgp]
 gpg

dulwich/__init__.py

@@ -22,4 +22,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 19, 11)
+__version__ = (0, 19, 12)

dulwich/client.py

@@ -1374,7 +1374,7 @@ def default_urllib3_manager(config, **override_kwargs):
             ssl_verify = True
 
         try:
-            ca_certs = config.get_boolean(b"http", b"sslCAInfo")
+            ca_certs = config.get(b"http", b"sslCAInfo")
         except KeyError:
             ca_certs = None
 

dulwich/contrib/swift.py

@@ -199,11 +199,8 @@ def swift_load_pack_index(scon, filename):
     :param filename: Path to the index file objectise
     :return: a `PackIndexer` instance
     """
-    f = scon.get_object(filename)
-    try:
+    with scon.get_object(filename) as f:
         return load_pack_index_file(filename, f)
-    finally:
-        f.close()
 
 
 def pack_info_create(pack_data, pack_index):
@@ -935,10 +932,9 @@ class SwiftRepo(BaseRepo):
         :param filename: the path to the object to put on Swift
         :param contents: the content as bytestring
         """
-        f = BytesIO()
-        f.write(contents)
-        self.scon.put_object(filename, f)
-        f.close()
+        with BytesIO() as f:
+            f.write(contents)
+            self.scon.put_object(filename, f)
 
     @classmethod
     def init_bare(cls, scon, conf):
@@ -992,7 +988,6 @@ def cmd_daemon(args):
         sys.exit(1)
     import gevent.monkey
     gevent.monkey.patch_socket()
-    from dulwich.contrib.swift import load_conf
     from dulwich import log_utils
     logger = log_utils.getLogger(__name__)
     conf = load_conf(options.swift_config)

dulwich/errors.py

@@ -121,24 +121,21 @@ class GitProtocolError(Exception):
 class SendPackError(GitProtocolError):
     """An error occurred during send_pack."""
 
-    def __init__(self, *args, **kwargs):
-        Exception.__init__(self, *args, **kwargs)
-
 
 class UpdateRefsError(GitProtocolError):
     """The server reported errors updating refs."""
 
     def __init__(self, *args, **kwargs):
         self.ref_status = kwargs.pop('ref_status')
-        Exception.__init__(self, *args, **kwargs)
+        super(UpdateRefsError, self).__init__(*args, **kwargs)
 
 
 class HangupException(GitProtocolError):
     """Hangup exception."""
 
     def __init__(self):
-        Exception.__init__(
-            self, "The remote server unexpectedly closed the connection.")
+        super(HangupException, self).__init__(
+            "The remote server unexpectedly closed the connection.")
 
 
 class UnexpectedCommandError(GitProtocolError):
@@ -149,7 +146,8 @@ class UnexpectedCommandError(GitProtocolError):
             command = 'flush-pkt'
         else:
             command = 'command %s' % command
-        GitProtocolError.__init__(self, 'Protocol got unexpected %s' % command)
+        super(UnexpectedCommandError, self).__init__(
+            'Protocol got unexpected %s' % command)
 
 
 class FileFormatException(Exception):
@@ -165,10 +163,7 @@ class ObjectFormatException(FileFormatException):
 
 
 class EmptyFileException(FileFormatException):
-    """Indicates an empty file instead of the object's disk
-       representation.
-
-    """
+    """An unexpectedly empty file was encountered."""
 
 
 class NoIndexPresent(Exception):

dulwich/ignore.py

@@ -38,7 +38,7 @@ def _translate_segment(segment):
         if c == b'*':
             res += b'[^/]*'
         elif c == b'?':
-            res += b'.'
+            res += b'[^/]'
         elif c == b'[':
             j = i
             if j < n and segment[j:j+1] == b'!':

dulwich/index.py

@@ -604,6 +604,31 @@ def read_submodule_head(path):
         return None
 
 
+def _has_directory_changed(tree_path, entry):
+    """Check if a directory has changed after getting an error.
+
+    When handling an error trying to create a blob from a path, call this
+    function. It will check if the path is a directory. If it's a directory
+    and a submodule, check the submodule head to see if it's has changed. If
+    not, consider the file as changed as Git tracked a file and not a
+    directory.
+
+    Return true if the given path should be considered as changed and False
+    otherwise or if the path is not a directory.
+    """
+    # This is actually a directory
+    if os.path.exists(os.path.join(tree_path, b'.git')):
+        # Submodule
+        head = read_submodule_head(tree_path)
+        if entry.sha != head:
+            return True
+    else:
+        # The file was changed to a directory, so consider it removed.
+        return True
+
+    return False
+
+
 def get_unstaged_changes(index, root_path, filter_blob_callback=None):
     """Walk through an index and check for differences against working tree.
 
@@ -618,30 +643,23 @@ def get_unstaged_changes(index, root_path, filter_blob_callback=None):
     for tree_path, entry in index.iteritems():
         full_path = _tree_to_fs_path(root_path, tree_path)
         try:
-            blob = blob_from_path_and_stat(
-                full_path, os.lstat(full_path)
-            )
+            st = os.lstat(full_path)
+            if stat.S_ISDIR(st.st_mode):
+                if _has_directory_changed(tree_path, entry):
+                    yield tree_path
+                continue
+
+            blob = blob_from_path_and_stat(full_path, st)
 
             if filter_blob_callback is not None:
                 blob = filter_blob_callback(blob, tree_path)
-        except OSError as e:
-            if e.errno != errno.ENOENT:
-                raise
-            # The file was removed, so we assume that counts as
-            # different from whatever file used to exist.
-            yield tree_path
-        except IOError as e:
-            if e.errno != errno.EISDIR:
-                raise
-            # This is actually a directory
-            if os.path.exists(os.path.join(tree_path, '.git')):
-                # Submodule
-                head = read_submodule_head(tree_path)
-                if entry.sha != head:
-                    yield tree_path
-            else:
-                # The file was changed to a directory, so consider it removed.
+        except EnvironmentError as e:
+            if e.errno == errno.ENOENT:
+                # The file was removed, so we assume that counts as
+                # different from whatever file used to exist.
                 yield tree_path
+            else:
+                raise
         else:
             if blob.id != entry.sha:
                 yield tree_path
@@ -697,28 +715,23 @@ def index_entry_from_path(path, object_store=None):
     :param path: Path to create an index entry for
     :param object_store: Optional object store to
         save new blobs in
-    :return: An index entry
+    :return: An index entry; None for directories
     """
     assert isinstance(path, bytes)
-    try:
-        st = os.lstat(path)
-        blob = blob_from_path_and_stat(path, st)
-    except EnvironmentError as e:
-        if e.errno == errno.EISDIR:
-            if os.path.exists(os.path.join(path, b'.git')):
-                head = read_submodule_head(path)
-                if head is None:
-                    return None
-                return index_entry_from_stat(
-                    st, head, 0, mode=S_IFGITLINK)
-            else:
-                raise
-        else:
-            raise
-    else:
-        if object_store is not None:
-            object_store.add_object(blob)
-        return index_entry_from_stat(st, blob.id, 0)
+    st = os.lstat(path)
+    if stat.S_ISDIR(st.st_mode):
+        if os.path.exists(os.path.join(path, b'.git')):
+            head = read_submodule_head(path)
+            if head is None:
+                return None
+            return index_entry_from_stat(
+                st, head, 0, mode=S_IFGITLINK)
+        return None
+
+    blob = blob_from_path_and_stat(path, st)
+    if object_store is not None:
+        object_store.add_object(blob)
+    return index_entry_from_stat(st, blob.id, 0)
 
 
 def iter_fresh_entries(paths, root_path, object_store=None):

dulwich/object_store.py

@@ -615,12 +615,13 @@ class DiskObjectStore(PackBasedObjectStore):
         os.remove(self._get_shafile_path(sha))
 
     def _remove_pack(self, pack):
-        os.remove(pack.data.path)
-        os.remove(pack.index.path)
         try:
             del self._pack_cache[os.path.basename(pack._basename)]
         except KeyError:
             pass
+        pack.close()
+        os.remove(pack.data.path)
+        os.remove(pack.index.path)
 
     def _get_pack_basepath(self, entries):
         suffix = iter_sha1(entry[0] for entry in entries)
@@ -726,8 +727,10 @@ class DiskObjectStore(PackBasedObjectStore):
         with PackData(path) as p:
             entries = p.sorted_entries()
             basename = self._get_pack_basepath(entries)
-            with GitFile(basename+".idx", "wb") as f:
-                write_pack_index_v2(f, entries, p.get_stored_checksum())
+            index_name = basename + ".idx"
+            if not os.path.exists(index_name):
+                with GitFile(index_name, "wb") as f:
+                    write_pack_index_v2(f, entries, p.get_stored_checksum())
         for pack in self.packs:
             if pack._basename == basename:
                 return pack
@@ -996,9 +999,14 @@ class ObjectStoreIterator(ObjectIterator):
         return len(list(self.itershas()))
 
     def empty(self):
-        iter = self.itershas()
+        import warnings
+        warnings.warn('Use bool() instead.', DeprecationWarning)
+        return self._empty()
+
+    def _empty(self):
+        it = self.itershas()
         try:
-            iter()
+            next(it)
         except StopIteration:
             return True
         else:
@@ -1006,7 +1014,7 @@ class ObjectStoreIterator(ObjectIterator):
 
     def __bool__(self):
         """Indicate whether this object has contents."""
-        return not self.empty()
+        return not self._empty()
 
 
 def tree_lookup_path(lookup_obj, root_sha, path):
@@ -1318,22 +1326,19 @@ class OverlayObjectStore(BaseObjectStore):
                 return b.get_raw(sha_id)
             except KeyError:
                 pass
-        else:
-            raise KeyError(sha_id)
+        raise KeyError(sha_id)
 
     def contains_packed(self, sha):
         for b in self.bases:
             if b.contains_packed(sha):
                 return True
-        else:
-            return False
+        return False
 
     def contains_loose(self, sha):
         for b in self.bases:
             if b.contains_loose(sha):
                 return True
-        else:
-            return False
+        return False
 
 
 def read_packs_file(f):

dulwich/objects.py

@@ -259,7 +259,10 @@ class ShaFile(object):
             start = len(header)
         header = header[:end]
         type_name, size = header.split(b' ', 1)
-        size = int(size)  # sanity check
+        try:
+            int(size)  # sanity check
+        except ValueError as e:
+            raise ObjectFormatException("Object size not an integer: %s" % e)
         obj_class = object_class(type_name)
         if not obj_class:
             raise ObjectFormatException("Not a known type: %s" % type_name)

dulwich/objectspec.py

@@ -74,8 +74,7 @@ def parse_ref(container, refspec):
     for ref in possible_refs:
         if ref in container:
             return ref
-    else:
-        raise KeyError(refspec)
+    raise KeyError(refspec)
 
 
 def parse_reftuple(lh_container, rh_container, refspec):

dulwich/patch.py

@@ -185,8 +185,8 @@ def write_object_diff(f, store, old_file, new_file, diff_binary=False):
     """
     (old_path, old_mode, old_id) = old_file
     (new_path, new_mode, new_id) = new_file
-    old_path = patch_filename(old_path, b"a")
-    new_path = patch_filename(new_path, b"b")
+    patched_old_path = patch_filename(old_path, b"a")
+    patched_new_path = patch_filename(new_path, b"b")
 
     def content(mode, hexsha):
         if hexsha is None:
@@ -207,11 +207,17 @@ def write_object_diff(f, store, old_file, new_file, diff_binary=False):
     new_content = content(new_mode, new_id)
     if not diff_binary and (
             is_binary(old_content.data) or is_binary(new_content.data)):
-        f.write(b"Binary files " + old_path + b" and " + new_path +
-                b" differ\n")
+        binary_diff = (
+            b"Binary files "
+            + patched_old_path
+            + b" and "
+            + patched_new_path
+            + b" differ\n"
+        )
+        f.write(binary_diff)
     else:
         f.writelines(unified_diff(lines(old_content), lines(new_content),
-                     old_path, new_path))
+                     patched_old_path, patched_new_path))
 
 
 # TODO(jelmer): Support writing unicode, rather than bytes.
@@ -225,16 +231,23 @@ def gen_diff_header(paths, modes, shas):
     (old_path, new_path) = paths
     (old_mode, new_mode) = modes
     (old_sha, new_sha) = shas
+    if old_path is None and new_path is not None:
+        old_path = new_path
+    if new_path is None and old_path is not None:
+        new_path = old_path
+    old_path = patch_filename(old_path, b"a")
+    new_path = patch_filename(new_path, b"b")
     yield b"diff --git " + old_path + b" " + new_path + b"\n"
+
     if old_mode != new_mode:
         if new_mode is not None:
             if old_mode is not None:
-                yield ("old mode %o\n" % old_mode).encode('ascii')
-            yield ("new mode %o\n" % new_mode).encode('ascii')
+                yield ("old file mode %o\n" % old_mode).encode('ascii')
+            yield ("new file mode %o\n" % new_mode).encode('ascii')
         else:
-            yield ("deleted mode %o\n" % old_mode).encode('ascii')
+            yield ("deleted file mode %o\n" % old_mode).encode('ascii')
     yield b"index " + shortid(old_sha) + b".." + shortid(new_sha)
-    if new_mode is not None:
+    if new_mode is not None and old_mode is not None:
         yield (" %o" % new_mode).encode('ascii')
     yield b"\n"
 
@@ -251,8 +264,8 @@ def write_blob_diff(f, old_file, new_file):
     """
     (old_path, old_mode, old_blob) = old_file
     (new_path, new_mode, new_blob) = new_file
-    old_path = patch_filename(old_path, b"a")
-    new_path = patch_filename(new_path, b"b")
+    patched_old_path = patch_filename(old_path, b"a")
+    patched_new_path = patch_filename(new_path, b"b")
 
     def lines(blob):
         if blob is not None:
@@ -265,7 +278,7 @@ def write_blob_diff(f, old_file, new_file):
     old_contents = lines(old_blob)
     new_contents = lines(new_blob)
     f.writelines(unified_diff(old_contents, new_contents,
-                 old_path, new_path))
+                 patched_old_path, patched_new_path))
 
 
 def write_tree_diff(f, store, old_tree, new_tree, diff_binary=False):

dulwich/porcelain.py

@@ -154,12 +154,14 @@ class NoneStream(RawIOBase):
         return None
 
 
-default_bytes_out_stream = getattr(
-        sys.stdout, 'buffer', sys.stdout
-    ) or NoneStream()
-default_bytes_err_stream = getattr(
-        sys.stderr, 'buffer', sys.stderr
-    ) or NoneStream()
+if sys.version_info[0] == 2:
+    default_bytes_out_stream = sys.stdout or NoneStream()
+    default_bytes_err_stream = sys.stderr or NoneStream()
+else:
+    default_bytes_out_stream = (
+        getattr(sys.stdout, 'buffer', None) or NoneStream())
+    default_bytes_err_stream = (
+        getattr(sys.stderr, 'buffer', None) or NoneStream())
 
 
 DEFAULT_ENCODING = 'utf-8'
@@ -406,6 +408,57 @@ def add(repo=".", paths=None):
     return (relpaths, ignored)
 
 
+def _is_subdir(subdir, parentdir):
+    """Check whether subdir is parentdir or a subdir of parentdir
+
+        If parentdir or subdir is a relative path, it will be disamgibuated
+        relative to the pwd.
+    """
+    parentdir_abs = os.path.realpath(parentdir) + os.path.sep
+    subdir_abs = os.path.realpath(subdir) + os.path.sep
+    return subdir_abs.startswith(parentdir_abs)
+
+
+# TODO: option to remove ignored files also, in line with `git clean -fdx`
+def clean(repo=".", target_dir=None):
+    """Remove any untracked files from the target directory recursively
+
+    Equivalent to running `git clean -fd` in target_dir.
+
+    :param repo: Repository where the files may be tracked
+    :param target_dir: Directory to clean - current directory if None
+    """
+    if target_dir is None:
+        target_dir = os.getcwd()
+
+    with open_repo_closing(repo) as r:
+        if not _is_subdir(target_dir, r.path):
+            raise ValueError("target_dir must be in the repo's working dir")
+
+        index = r.open_index()
+        ignore_manager = IgnoreFilterManager.from_repo(r)
+
+        paths_in_wd = _walk_working_dir_paths(target_dir, r.path)
+        # Reverse file visit order, so that files and subdirectories are
+        # removed before containing directory
+        for ap, is_dir in reversed(list(paths_in_wd)):
+            if is_dir:
+                # All subdirectories and files have been removed if untracked,
+                # so dir contains no tracked files iff it is empty.
+                is_empty = len(os.listdir(ap)) == 0
+                if is_empty:
+                    os.rmdir(ap)
+            else:
+                ip = path_to_tree_path(r.path, ap)
+                is_tracked = ip in index
+
+                rp = os.path.relpath(ap, r.path)
+                is_ignored = ignore_manager.is_ignored(rp)
+
+                if not is_tracked and not is_ignored:
+                    os.remove(ap)
+
+
 def remove(repo=".", paths=None, cached=False):
     """Remove files from the staging area.
 
@@ -494,7 +547,10 @@ def print_tag(tag, decode, outstream=sys.stdout):
     :param outstream: A stream to write to
     """
     outstream.write("Tagger: " + decode(tag.tagger) + "\n")
-    outstream.write("Date:   " + decode(tag.tag_time) + "\n")
+    time_tuple = time.gmtime(tag.tag_time + tag.tag_timezone)
+    time_str = time.strftime("%a %b %d %Y %H:%M:%S", time_tuple)
+    timezone_str = format_timezone(tag.tag_timezone).decode('ascii')
+    outstream.write("Date:   " + time_str + " " + timezone_str + "\n")
     outstream.write("\n")
     outstream.write(decode(tag.message) + "\n")
     outstream.write("\n")
@@ -556,7 +612,7 @@ def show_tag(repo, tag, decode, outstream=sys.stdout):
     :param outstream: Stream to write to
     """
     print_tag(tag, decode, outstream)
-    show_object(repo, repo[tag.object[1]], outstream)
+    show_object(repo, repo[tag.object[1]], decode, outstream)
 
 
 def show_object(repo, obj, decode, outstream):
@@ -721,7 +777,8 @@ def tag_create(
             if sign:
                 import gpg
                 with gpg.Context(armor=True) as c:
-                    tag_obj.signature, result = c.sign(tag_obj.as_raw_string())
+                    tag_obj.signature, unused_result = c.sign(
+                        tag_obj.as_raw_string())
             r.object_store.add_object(tag_obj)
             tag_id = tag_obj.id
         else:
@@ -873,7 +930,7 @@ def status(repo=".", ignored=False):
     :param repo: Path to repository or repository object
     :param ignored: Whether to include ignored files in `untracked`
     :return: GitStatus tuple,
-        staged -    list of staged paths (diff index/HEAD)
+        staged -  dict with lists of staged paths (diff index/HEAD)
         unstaged -  list of unstaged paths (diff index/working-tree)
         untracked - list of untracked, un-ignored & non-.git paths
     """
@@ -898,14 +955,12 @@ def status(repo=".", ignored=False):
         return GitStatus(tracked_changes, unstaged_changes, untracked_changes)
 
 
-def get_untracked_paths(frompath, basepath, index):
-    """Get untracked paths.
+def _walk_working_dir_paths(frompath, basepath):
+    """Get path, is_dir for files in working dir from frompath
 
-    ;param frompath: Path to walk
+    :param frompath: Path to begin walk
     :param basepath: Path to compare to
-    :param index: Index to check against
     """
-    # If nothing is specified, add all non-ignored files.
     for dirpath, dirnames, filenames in os.walk(frompath):
         # Skip .git and below.
         if '.git' in dirnames:
@@ -916,8 +971,24 @@ def get_untracked_paths(frompath, basepath, index):
             filenames.remove('.git')
             if dirpath != basepath:
                 continue
+
+        if dirpath != frompath:
+            yield dirpath, True
+
         for filename in filenames:
-            ap = os.path.join(dirpath, filename)
+            filepath = os.path.join(dirpath, filename)
+            yield filepath, False
+
+
+def get_untracked_paths(frompath, basepath, index):
+    """Get untracked paths.
+
+    ;param frompath: Path to walk
+    :param basepath: Path to compare to
+    :param index: Index to check against
+    """
+    for ap, is_dir in _walk_working_dir_paths(frompath, basepath):
+        if not is_dir:
             ip = path_to_tree_path(basepath, ap)
             if ip not in index:
                 yield os.path.relpath(ap, frompath)

dulwich/refs.py

@@ -158,6 +158,9 @@ class RefsContainer(object):
         """All refs present in this container."""
         raise NotImplementedError(self.allkeys)
 
+    def __iter__(self):
+        return iter(self.allkeys())
+
     def keys(self, base=None):
         """Refs present in this container.
 

dulwich/repo.py

@@ -168,9 +168,13 @@ def get_user_identity(config, kind=None):
             email = None
     default_user, default_email = _get_default_identity()
     if user is None:
-        user = default_user.encode('utf-8')
+        user = default_user
+        if not isinstance(user, bytes):
+            user = user.encode('utf-8')
     if email is None:
-        email = default_email.encode('utf-8')
+        email = default_email
+        if not isinstance(email, bytes):
+            email = email.encode('utf-8')
     return (user + b" <" + email + b">")
 
 
@@ -239,6 +243,28 @@ def serialize_graftpoints(graftpoints):
     return b'\n'.join(graft_lines)
 
 
+def _set_filesystem_hidden(path):
+    """Mark path as to be hidden if supported by platform and filesystem.
+
+    On win32 uses SetFileAttributesW api:
+    <https://docs.microsoft.com/windows/desktop/api/fileapi/nf-fileapi-setfileattributesw>
+    """
+    if sys.platform == 'win32':
+        import ctypes
+        from ctypes.wintypes import BOOL, DWORD, LPCWSTR
+
+        FILE_ATTRIBUTE_HIDDEN = 2
+        SetFileAttributesW = ctypes.WINFUNCTYPE(BOOL, LPCWSTR, DWORD)(
+            ("SetFileAttributesW", ctypes.windll.kernel32))
+
+        if isinstance(path, bytes):
+            path = path.decode(sys.getfilesystemencoding())
+        if not SetFileAttributesW(path, FILE_ATTRIBUTE_HIDDEN):
+            pass  # Could raise or log `ctypes.WinError()` here
+
+    # Could implement other platform specific filesytem hiding here
+
+
 class BaseRepo(object):
     """Base class for a git repository.
 
@@ -443,7 +469,9 @@ class BaseRepo(object):
         :return: A graph walker object
         """
         if heads is None:
-            heads = self.refs.as_dict(b'refs/heads').values()
+            heads = [
+                sha for sha in self.refs.as_dict(b'refs/heads').values()
+                if sha in self.object_store]
         return ObjectStoreGraphWalker(
             heads, self.get_parents, shallow=self.get_shallow())
 
@@ -966,7 +994,10 @@ class Repo(BaseRepo):
             f.write('')
 
         st1 = os.lstat(fname)
-        os.chmod(fname, st1.st_mode ^ stat.S_IXUSR)
+        try:
+            os.chmod(fname, st1.st_mode ^ stat.S_IXUSR)
+        except PermissionError:
+            return False
         st2 = os.lstat(fname)
 
         os.unlink(fname)
@@ -1228,6 +1259,7 @@ class Repo(BaseRepo):
             os.mkdir(path)
         controldir = os.path.join(path, CONTROLDIR)
         os.mkdir(controldir)
+        _set_filesystem_hidden(controldir)
         cls._init_maybe_bare(controldir, False)
         return cls(path)
 
@@ -1355,7 +1387,7 @@ class MemoryRepo(BaseRepo):
         except KeyError:
             pass
 
-    def get_named_file(self, path):
+    def get_named_file(self, path, basedir=None):
         """Get a file from the control dir with a specific name.
 
         Although the filename should be interpreted as a filename relative to

dulwich/tests/compat/__init__.py

@@ -27,6 +27,7 @@ def test_suite():
     names = [
         'client',
         'pack',
+        'patch',
         'repository',
         'server',
         'utils',

dulwich/tests/compat/test_patch.py

@@ -0,0 +1,120 @@
+# test_patch.py -- test patch compatibility with CGit
+# Copyright (C) 2019 Boris Feld <boris@comet.ml>
+#
+# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+# General Public License as public by the Free Software Foundation; version 2.0
+# or (at your option) any later version. You can redistribute it and/or
+# modify it under the terms of either of these two licenses.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# You should have received a copy of the licenses; if not, see
+# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+# License, Version 2.0.
+#
+
+"""Tests related to patch compatibility with CGit."""
+from io import BytesIO
+import os
+import shutil
+import tempfile
+
+from dulwich import porcelain
+from dulwich.repo import (
+    Repo,
+    )
+from dulwich.tests.compat.utils import (
+    CompatTestCase,
+    run_git_or_fail,
+    )
+
+
+class CompatPatchTestCase(CompatTestCase):
+
+    def setUp(self):
+        super(CompatPatchTestCase, self).setUp()
+        self.test_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.test_dir)
+        self.repo_path = os.path.join(self.test_dir, 'repo')
+        self.repo = Repo.init(self.repo_path, mkdir=True)
+        self.addCleanup(self.repo.close)
+
+    def test_patch_apply(self):
+        # Prepare the repository
+
+        # Create some files and commit them
+        file_list = ["to_exists", "to_modify", "to_delete"]
+        for file in file_list:
+            file_path = os.path.join(self.repo_path, file)
+
+            # Touch the files
+            with open(file_path, "w"):
+                pass
+
+        self.repo.stage(file_list)
+
+        first_commit = self.repo.do_commit(b"The first commit")
+
+        # Make a copy of the repository so we can apply the diff later
+        copy_path = os.path.join(self.test_dir, "copy")
+        shutil.copytree(self.repo_path, copy_path)
+
+        # Do some changes
+        with open(os.path.join(self.repo_path, "to_modify"), "w") as f:
+            f.write("Modified!")
+
+        os.remove(os.path.join(self.repo_path, "to_delete"))
+
+        with open(os.path.join(self.repo_path, "to_add"), "w"):
+            pass
+
+        self.repo.stage(["to_modify", "to_delete", "to_add"])
+
+        second_commit = self.repo.do_commit(b"The second commit")
+
+        # Get the patch
+        first_tree = self.repo[first_commit].tree
+        second_tree = self.repo[second_commit].tree
+
+        outstream = BytesIO()
+        porcelain.diff_tree(self.repo.path, first_tree, second_tree,
+                            outstream=outstream)
+
+        # Save it on disk
+        patch_path = os.path.join(self.test_dir, "patch.patch")
+        with open(patch_path, "wb") as patch:
+            patch.write(outstream.getvalue())
+
+        # And try to apply it to the copy directory
+        git_command = ["-C", copy_path, "apply", patch_path]
+        run_git_or_fail(git_command)
+
+        # And now check that the files contents are exactly the same between
+        # the two repositories
+        original_files = set(os.listdir(self.repo_path))
+        new_files = set(os.listdir(copy_path))
+
+        # Check that we have the exact same files in both repositories
+        self.assertEquals(original_files, new_files)
+
+        for file in original_files:
+            if file == ".git":
+                continue
+
+            original_file_path = os.path.join(self.repo_path, file)
+            copy_file_path = os.path.join(copy_path, file)
+
+            self.assertTrue(os.path.isfile(copy_file_path))
+
+            with open(original_file_path, "rb") as original_file:
+                original_content = original_file.read()
+
+            with open(copy_file_path, "rb") as copy_file:
+                copy_content = copy_file.read()
+
+            self.assertEquals(original_content, copy_content)

dulwich/tests/test_client.py

@@ -48,6 +48,7 @@ from dulwich.client import (
     TCPGitClient,
     SSHGitClient,
     HttpGitClient,
+    FetchPackResult,
     ReportStatusParser,
     SendPackError,
     StrangeHostname,
@@ -1304,3 +1305,17 @@ class CheckWantsTests(TestCase):
             {b'refs/heads/blah': b'3f3dc7a53fb752a6961d3a56683df46d4d3bf262',
              b'refs/heads/blah^{}':
                 b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'})
+
+
+class FetchPackResultTests(TestCase):
+
+    def test_eq(self):
+        self.assertEqual(
+            FetchPackResult(
+                 {b'refs/heads/master':
+                  b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'}, {},
+                 b'user/agent'),
+            FetchPackResult(
+                {b'refs/heads/master':
+                 b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'}, {},
+                b'user/agent'))

dulwich/tests/test_ignore.py

@@ -65,6 +65,7 @@ NEGATIVE_MATCH_TESTS = [
     (b"foo/foo.c", b"/*.c"),
     (b"foo/bar/", b"/bar/"),
     (b"foo/bar/", b"foo/bar/*"),
+    (b"foo/bar", b"foo?bar")
 ]
 
 

dulwich/tests/test_index.py

@@ -671,6 +671,53 @@ class GetUnstagedChangesTests(TestCase):
 
             self.assertEqual(list(changes), [b'foo1'])
 
+    def test_get_unstaged_changes_removed_replaced_by_directory(self):
+        """Unit test for get_unstaged_changes."""
+
+        repo_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, repo_dir)
+        with Repo.init(repo_dir) as repo:
+
+            # Commit a dummy file then modify it
+            foo1_fullpath = os.path.join(repo_dir, 'foo1')
+            with open(foo1_fullpath, 'wb') as f:
+                f.write(b'origstuff')
+
+            repo.stage(['foo1'])
+            repo.do_commit(b'test status', author=b'author <email>',
+                           committer=b'committer <email>')
+
+            os.remove(foo1_fullpath)
+            os.mkdir(foo1_fullpath)
+
+            changes = get_unstaged_changes(repo.open_index(), repo_dir)
+
+            self.assertEqual(list(changes), [b'foo1'])
+
+    @skipIf(not can_symlink(), 'Requires symlink support')
+    def test_get_unstaged_changes_removed_replaced_by_link(self):
+        """Unit test for get_unstaged_changes."""
+
+        repo_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, repo_dir)
+        with Repo.init(repo_dir) as repo:
+
+            # Commit a dummy file then modify it
+            foo1_fullpath = os.path.join(repo_dir, 'foo1')
+            with open(foo1_fullpath, 'wb') as f:
+                f.write(b'origstuff')
+
+            repo.stage(['foo1'])
+            repo.do_commit(b'test status', author=b'author <email>',
+                           committer=b'committer <email>')
+
+            os.remove(foo1_fullpath)
+            os.symlink(os.path.dirname(foo1_fullpath), foo1_fullpath)
+
+            changes = get_unstaged_changes(repo.open_index(), repo_dir)
+
+            self.assertEqual(list(changes), [b'foo1'])
+
 
 class TestValidatePathElement(TestCase):
 

dulwich/tests/test_patch.py

@@ -276,9 +276,9 @@ class DiffTests(TestCase):
             f, (None, None, None),
             (b"bar.txt", 0o644, Blob.from_string(b"new\nsame\n")))
         self.assertEqual([
-             b'diff --git /dev/null b/bar.txt',
-             b'new mode 644',
-             b'index 0000000..a116b51 644',
+             b'diff --git a/bar.txt b/bar.txt',
+             b'new file mode 644',
+             b'index 0000000..a116b51',
              b'--- /dev/null',
              b'+++ b/bar.txt',
              b'@@ -0,0 +1,2 @@',
@@ -292,8 +292,8 @@ class DiffTests(TestCase):
             f, (b"bar.txt", 0o644, Blob.from_string(b"new\nsame\n")),
             (None, None, None))
         self.assertEqual([
-            b'diff --git a/bar.txt /dev/null',
-            b'deleted mode 644',
+            b'diff --git a/bar.txt b/bar.txt',
+            b'deleted file mode 644',
             b'index a116b51..0000000',
             b'--- a/bar.txt',
             b'+++ /dev/null',
@@ -322,9 +322,9 @@ class DiffTests(TestCase):
             tree1, tree2, added, removed, changed1, changed2, unchanged]])
         write_tree_diff(f, store, tree1.id, tree2.id)
         self.assertEqual([
-            b'diff --git /dev/null b/added.txt',
-            b'new mode 644',
-            b'index 0000000..76d4bb8 644',
+            b'diff --git a/added.txt b/added.txt',
+            b'new file mode 644',
+            b'index 0000000..76d4bb8',
             b'--- /dev/null',
             b'+++ b/added.txt',
             b'@@ -0,0 +1 @@',
@@ -337,8 +337,8 @@ class DiffTests(TestCase):
             b' unchanged',
             b'-removed',
             b'+added',
-            b'diff --git a/removed.txt /dev/null',
-            b'deleted mode 644',
+            b'diff --git a/removed.txt b/removed.txt',
+            b'deleted file mode 644',
             b'index 2c3f0b3..0000000',
             b'--- a/removed.txt',
             b'+++ /dev/null',
@@ -394,9 +394,9 @@ class DiffTests(TestCase):
         write_object_diff(f, store, (None, None, None),
                                     (b"bar.txt", 0o644, b2.id))
         self.assertEqual([
-             b'diff --git /dev/null b/bar.txt',
-             b'new mode 644',
-             b'index 0000000..a116b51 644',
+             b'diff --git a/bar.txt b/bar.txt',
+             b'new file mode 644',
+             b'index 0000000..a116b51',
              b'--- /dev/null',
              b'+++ b/bar.txt',
              b'@@ -0,0 +1,2 @@',
@@ -412,8 +412,8 @@ class DiffTests(TestCase):
         write_object_diff(f, store, (b"bar.txt", 0o644, b1.id),
                                     (None, None, None))
         self.assertEqual([
-            b'diff --git a/bar.txt /dev/null',
-            b'deleted mode 644',
+            b'diff --git a/bar.txt b/bar.txt',
+            b'deleted file mode 644',
             b'index a116b51..0000000',
             b'--- a/bar.txt',
             b'+++ /dev/null',
@@ -492,9 +492,9 @@ class DiffTests(TestCase):
         write_object_diff(f, store, (None, None, None),
                                     (b'bar.png', 0o644, b2.id))
         self.assertEqual([
-            b'diff --git /dev/null b/bar.png',
-            b'new mode 644',
-            b'index 0000000..06364b7 644',
+            b'diff --git a/bar.png b/bar.png',
+            b'new file mode 644',
+            b'index 0000000..06364b7',
             b'Binary files /dev/null and b/bar.png differ'
             ], f.getvalue().splitlines())
 
@@ -510,8 +510,8 @@ class DiffTests(TestCase):
         write_object_diff(f, store, (b'foo.png', 0o644, b1.id),
                                     (None, None, None))
         self.assertEqual([
-            b'diff --git a/foo.png /dev/null',
-            b'deleted mode 644',
+            b'diff --git a/foo.png b/foo.png',
+            b'deleted file mode 644',
             b'index f73e47d..0000000',
             b'Binary files a/foo.png and /dev/null differ'
             ], f.getvalue().splitlines())
@@ -527,8 +527,8 @@ class DiffTests(TestCase):
                 b"06d0bdd9e2e20377b3180e4986b14c8549b393e4"))
         self.assertEqual([
             b'diff --git a/bar.txt b/bar.txt',
-            b'old mode 644',
-            b'new mode 160000',
+            b'old file mode 644',
+            b'new file mode 160000',
             b'index a116b51..06d0bdd 160000',
             b'--- a/bar.txt',
             b'+++ b/bar.txt',

dulwich/tests/test_porcelain.py

@@ -25,6 +25,7 @@ try:
     from StringIO import StringIO
 except ImportError:
     from io import StringIO
+import errno
 import os
 import shutil
 import tarfile
@@ -51,6 +52,21 @@ from dulwich.tests.utils import (
     make_commit,
     make_object,
     )
+from dulwich.tests.compat.utils import (
+    run_git_or_fail,
+    )
+
+
+def flat_walk_dir(dir_to_walk):
+    for dirpath, _, filenames in os.walk(dir_to_walk):
+        rel_dirpath = os.path.relpath(dirpath, dir_to_walk)
+        if not dirpath == dir_to_walk:
+            yield rel_dirpath
+        for filename in filenames:
+            if dirpath == dir_to_walk:
+                yield filename
+            else:
+                yield os.path.join(rel_dirpath, filename)
 
 
 class PorcelainTestCase(TestCase):
@@ -59,7 +75,8 @@ class PorcelainTestCase(TestCase):
         super(PorcelainTestCase, self).setUp()
         self.test_dir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.test_dir)
-        self.repo = Repo.init(os.path.join(self.test_dir, 'repo'), mkdir=True)
+        self.repo_path = os.path.join(self.test_dir, 'repo')
+        self.repo = Repo.init(self.repo_path, mkdir=True)
         self.addCleanup(self.repo.close)
 
 
@@ -116,6 +133,105 @@ class CommitTests(PorcelainTestCase):
         self.assertEqual(len(sha), 40)
 
 
+class CleanTests(PorcelainTestCase):
+
+    def put_files(self, tracked, ignored, untracked, empty_dirs):
+        """Put the described files in the wd
+        """
+        all_files = tracked | ignored | untracked
+        for file_path in all_files:
+            abs_path = os.path.join(self.repo.path, file_path)
+            # File may need to be written in a dir that doesn't exist yet, so
+            # create the parent dir(s) as necessary
+            parent_dir = os.path.dirname(abs_path)
+            try:
+                os.makedirs(parent_dir)
+            except OSError as err:
+                if not err.errno == errno.EEXIST:
+                    raise err
+            with open(abs_path, 'w') as f:
+                f.write('')
+
+        with open(os.path.join(self.repo.path, '.gitignore'), 'w') as f:
+            f.writelines(ignored)
+
+        for dir_path in empty_dirs:
+            os.mkdir(os.path.join(self.repo.path, 'empty_dir'))
+
+        files_to_add = [os.path.join(self.repo.path, t) for t in tracked]
+        porcelain.add(repo=self.repo.path, paths=files_to_add)
+        porcelain.commit(repo=self.repo.path, message="init commit")
+
+    def assert_wd(self, expected_paths):
+        """Assert paths of files and dirs in wd are same as expected_paths
+        """
+        control_dir_rel = os.path.relpath(
+            self.repo._controldir, self.repo.path)
+
+        # normalize paths to simplify comparison across platforms
+        found_paths = {
+            os.path.normpath(p)
+            for p in flat_walk_dir(self.repo.path)
+            if not p.split(os.sep)[0] == control_dir_rel}
+        norm_expected_paths = {os.path.normpath(p) for p in expected_paths}
+        self.assertEqual(found_paths, norm_expected_paths)
+
+    def test_from_root(self):
+        self.put_files(
+            tracked={
+                'tracked_file',
+                'tracked_dir/tracked_file',
+                '.gitignore'},
+            ignored={
+                'ignored_file'},
+            untracked={
+                'untracked_file',
+                'tracked_dir/untracked_dir/untracked_file',
+                'untracked_dir/untracked_dir/untracked_file'},
+            empty_dirs={
+                'empty_dir'})
+
+        porcelain.clean(repo=self.repo.path, target_dir=self.repo.path)
+
+        self.assert_wd({
+            'tracked_file',
+            'tracked_dir/tracked_file',
+            '.gitignore',
+            'ignored_file',
+            'tracked_dir'})
+
+    def test_from_subdir(self):
+        self.put_files(
+            tracked={
+                'tracked_file',
+                'tracked_dir/tracked_file',
+                '.gitignore'},
+            ignored={
+                'ignored_file'},
+            untracked={
+                'untracked_file',
+                'tracked_dir/untracked_dir/untracked_file',
+                'untracked_dir/untracked_dir/untracked_file'},
+            empty_dirs={
+                'empty_dir'})
+
+        porcelain.clean(
+            repo=self.repo,
+            target_dir=os.path.join(self.repo.path, 'untracked_dir'))
+
+        self.assert_wd({
+            'tracked_file',
+            'tracked_dir/tracked_file',
+            '.gitignore',
+            'ignored_file',
+            'untracked_file',
+            'tracked_dir/untracked_dir/untracked_file',
+            'empty_dir',
+            'untracked_dir',
+            'tracked_dir',
+            'tracked_dir/untracked_dir'})
+
+
 class CloneTests(PorcelainTestCase):
 
     def test_simple_local(self):
@@ -456,9 +572,46 @@ Date:   Fri Jan 01 2010 00:00:00 +0000
 
 Test message.
 
-diff --git /dev/null b/somename
-new mode 100644
-index 0000000..ea5c7bf 100644
+diff --git a/somename b/somename
+new file mode 100644
+index 0000000..ea5c7bf
+--- /dev/null
++++ b/somename
+@@ -0,0 +1 @@
++The Foo
+""")
+
+    def test_tag(self):
+        a = Blob.from_string(b"The Foo\n")
+        ta = Tree()
+        ta.add(b"somename", 0o100644, a.id)
+        ca = make_commit(tree=ta.id)
+        self.repo.object_store.add_objects([(a, None), (ta, None), (ca, None)])
+        porcelain.tag_create(
+            self.repo.path, b"tryme", b'foo <foo@bar.com>', b'bar',
+            annotated=True, objectish=ca.id, tag_time=1552854211,
+            tag_timezone=0)
+        outstream = StringIO()
+        porcelain.show(self.repo, objects=[b'refs/tags/tryme'],
+                       outstream=outstream)
+        self.maxDiff = None
+        self.assertMultiLineEqual(outstream.getvalue(), """\
+Tagger: foo <foo@bar.com>
+Date:   Sun Mar 17 2019 20:23:31 +0000
+
+bar
+
+--------------------------------------------------
+commit: 344da06c1bb85901270b3e8875c988a027ec087d
+Author: Test Author <test@nodomain.com>
+Committer: Test Committer <test@nodomain.com>
+Date:   Fri Jan 01 2010 00:00:00 +0000
+
+Test message.
+
+diff --git a/somename b/somename
+new file mode 100644
+index 0000000..ea5c7bf
 --- /dev/null
 +++ b/somename
 @@ -0,0 +1 @@
@@ -553,6 +706,81 @@ class DiffTreeTests(PorcelainTestCase):
                             outstream=outstream)
         self.assertEqual(outstream.getvalue(), b"")
 
+    def test_diff_apply(self):
+        # Prepare the repository
+
+        # Create some files and commit them
+        file_list = ["to_exists", "to_modify", "to_delete"]
+        for file in file_list:
+            file_path = os.path.join(self.repo_path, file)
+
+            # Touch the files
+            with open(file_path, "w"):
+                pass
+
+        self.repo.stage(file_list)
+
+        first_commit = self.repo.do_commit(b"The first commit")
+
+        # Make a copy of the repository so we can apply the diff later
+        copy_path = os.path.join(self.test_dir, "copy")
+        shutil.copytree(self.repo_path, copy_path)
+
+        # Do some changes
+        with open(os.path.join(self.repo_path, "to_modify"), "w") as f:
+            f.write("Modified!")
+
+        os.remove(os.path.join(self.repo_path, "to_delete"))
+
+        with open(os.path.join(self.repo_path, "to_add"), "w"):
+            pass
+
+        self.repo.stage(["to_modify", "to_delete", "to_add"])
+
+        second_commit = self.repo.do_commit(b"The second commit")
+
+        # Get the patch
+        first_tree = self.repo[first_commit].tree
+        second_tree = self.repo[second_commit].tree
+
+        outstream = BytesIO()
+        porcelain.diff_tree(self.repo.path, first_tree, second_tree,
+                            outstream=outstream)
+
+        # Save it on disk
+        patch_path = os.path.join(self.test_dir, "patch.patch")
+        with open(patch_path, "wb") as patch:
+            patch.write(outstream.getvalue())
+
+        # And try to apply it to the copy directory
+        git_command = ["-C", copy_path, "apply", patch_path]
+        run_git_or_fail(git_command)
+
+        # And now check that the files contents are exactly the same between
+        # the two repositories
+        original_files = set(os.listdir(self.repo_path))
+        new_files = set(os.listdir(copy_path))
+
+        # Check that we have the exact same files in both repositories
+        assert original_files == new_files
+
+        for file in original_files:
+            if file == ".git":
+                continue
+
+            original_file_path = os.path.join(self.repo_path, file)
+            copy_file_path = os.path.join(copy_path, file)
+
+            assert os.path.isfile(copy_file_path)
+
+            with open(original_file_path, "rb") as original_file:
+                original_content = original_file.read()
+
+            with open(copy_file_path, "rb") as copy_file:
+                copy_content = copy_file.read()
+
+            assert original_content == copy_content
+
 
 class CommitTreeTests(PorcelainTestCase):
 

dulwich/tests/test_refs.py

@@ -185,6 +185,11 @@ class RefsContainerTests(object):
         self.assertEqual([b'refs-0.1', b'refs-0.2'],
                          sorted(self._refs.keys(b'refs/tags')))
 
+    def test_iter(self):
+        actual_keys = set(self._refs.keys())
+        self.assertEqual(set(self._refs), actual_keys)
+        self.assertEqual(set(_TEST_REFS.keys()), actual_keys)
+
     def test_as_dict(self):
         # refs/heads/loop does not show up even if it exists
         expected_refs = dict(_TEST_REFS)

dulwich/tests/test_repository.py

@@ -266,6 +266,49 @@ class RepositoryRootTests(TestCase):
                 r.get_walker(b'2a72d929692c41d8554c07f6301757ba18a65d91')],
             [b'2a72d929692c41d8554c07f6301757ba18a65d91'])
 
+    def assertFilesystemHidden(self, path):
+        if sys.platform != 'win32':
+            return
+        import ctypes
+        from ctypes.wintypes import DWORD, LPCWSTR
+        GetFileAttributesW = ctypes.WINFUNCTYPE(DWORD, LPCWSTR)(
+            ('GetFileAttributesW', ctypes.windll.kernel32))
+        self.assertTrue(2 & GetFileAttributesW(path))
+
+    def test_init_existing(self):
+        tmp_dir = self.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmp_dir)
+        t = Repo.init(tmp_dir)
+        self.addCleanup(t.close)
+        self.assertEqual(os.listdir(tmp_dir), ['.git'])
+        self.assertFilesystemHidden(os.path.join(tmp_dir, '.git'))
+
+    def test_init_mkdir(self):
+        tmp_dir = self.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmp_dir)
+        repo_dir = os.path.join(tmp_dir, 'a-repo')
+
+        t = Repo.init(repo_dir, mkdir=True)
+        self.addCleanup(t.close)
+        self.assertEqual(os.listdir(repo_dir), ['.git'])
+        self.assertFilesystemHidden(os.path.join(repo_dir, '.git'))
+
+    def test_init_mkdir_unicode(self):
+        repo_name = u'\xa7'
+        try:
+            repo_name.encode(sys.getfilesystemencoding())
+        except UnicodeEncodeError:
+            self.skipTest('filesystem lacks unicode support')
+        tmp_dir = self.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmp_dir)
+        repo_dir = os.path.join(tmp_dir, repo_name)
+
+        t = Repo.init(repo_dir, mkdir=True)
+        self.addCleanup(t.close)
+        self.assertEqual(os.listdir(repo_dir), ['.git'])
+        self.assertFilesystemHidden(os.path.join(repo_dir, '.git'))
+
+    @skipIf(sys.platform == 'win32', 'fails on Windows')
     def test_fetch(self):
         r = self.open_repo('a.git')
         tmp_dir = self.mkdtemp()
@@ -279,6 +322,7 @@ class RepositoryRootTests(TestCase):
         self.assertIn(b'28237f4dc30d0d462658d6b937b08a0f0b6ef55a', t)
         self.assertIn(b'b0931cadc54336e78a1d980420e3268903b57a50', t)
 
+    @skipIf(sys.platform == 'win32', 'fails on Windows')
     def test_fetch_ignores_missing_refs(self):
         r = self.open_repo('a.git')
         missing = b'1234566789123456789123567891234657373833'

dulwich/tests/test_web.py

@@ -277,6 +277,20 @@ class DumbHandlersTestCase(WebTestCase):
         self.assertContentTypeEquals('text/plain')
         self.assertFalse(self._req.cached)
 
+    def test_get_info_refs_not_found(self):
+        self._environ['QUERY_STRING'] = ''
+
+        objects = []
+        refs = {}
+        backend = _test_backend(objects, refs=refs)
+
+        mat = re.search('info/refs', '/foo/info/refs')
+        self.assertEqual(
+            [b'No git repository was found at /foo'],
+            list(get_info_refs(self._req, backend, mat)))
+        self.assertEqual(HTTP_NOT_FOUND, self._status)
+        self.assertContentTypeEquals('text/plain')
+
     def test_get_info_packs(self):
         class TestPackData(object):
 
@@ -341,8 +355,12 @@ class SmartHandlersTestCase(WebTestCase):
         if content_length is not None:
             self._environ['CONTENT_LENGTH'] = content_length
         mat = re.search('.*', '/git-upload-pack')
+
+        class Backend(object):
+            def open_repository(self, path):
+                return None
         handler_output = b''.join(
-          handle_service_request(self._req, 'backend', mat))
+          handle_service_request(self._req, Backend(), mat))
         write_output = self._output.getvalue()
         # Ensure all output was written via the write callback.
         self.assertEqual(b'', handler_output)
@@ -363,7 +381,13 @@ class SmartHandlersTestCase(WebTestCase):
 
     def test_get_info_refs_unknown(self):
         self._environ['QUERY_STRING'] = 'service=git-evil-handler'
-        content = list(get_info_refs(self._req, b'backend', None))
+
+        class Backend(object):
+            def open_repository(self, url):
+                return None
+
+        mat = re.search('.*', '/git-evil-pack')
+        content = list(get_info_refs(self._req, Backend(), mat))
         self.assertFalse(b'git-evil-handler' in b"".join(content))
         self.assertEqual(HTTP_FORBIDDEN, self._status)
         self.assertFalse(self._req.cached)
@@ -372,8 +396,13 @@ class SmartHandlersTestCase(WebTestCase):
         self._environ['wsgi.input'] = BytesIO(b'foo')
         self._environ['QUERY_STRING'] = 'service=git-upload-pack'
 
+        class Backend(object):
+
+            def open_repository(self, url):
+                return None
+
         mat = re.search('.*', '/git-upload-pack')
-        handler_output = b''.join(get_info_refs(self._req, b'backend', mat))
+        handler_output = b''.join(get_info_refs(self._req, Backend(), mat))
         write_output = self._output.getvalue()
         self.assertEqual((b'001e# service=git-upload-pack\n'
                           b'0000'

dulwich/walk.py

@@ -21,8 +21,6 @@
 """General implementation of walking commits and their contents."""
 
 
-from collections import defaultdict
-
 import collections
 import heapq
 from itertools import chain
@@ -394,7 +392,7 @@ def _topo_reorder(entries, get_parents=lambda commit: commit.parents):
     """
     todo = collections.deque()
     pending = {}
-    num_children = defaultdict(int)
+    num_children = collections.defaultdict(int)
     for entry in entries:
         todo.append(entry)
         for p in get_parents(entry.commit):

dulwich/web.py

@@ -47,6 +47,7 @@ from dulwich.protocol import (
     ReceivableProtocol,
     )
 from dulwich.repo import (
+    NotGitRepository,
     Repo,
     )
 from dulwich.server import (
@@ -173,6 +174,11 @@ def get_idx_file(req, backend, mat):
 def get_info_refs(req, backend, mat):
     params = parse_qs(req.environ['QUERY_STRING'])
     service = params.get('service', [None])[0]
+    try:
+        repo = get_repo(backend, mat)
+    except NotGitRepository as e:
+        yield req.not_found(str(e))
+        return
     if service and not req.dumb:
         handler_cls = req.handlers.get(service.encode('ascii'), None)
         if handler_cls is None:
@@ -194,7 +200,6 @@ def get_info_refs(req, backend, mat):
         req.nocache()
         req.respond(HTTP_OK, 'text/plain')
         logger.info('Emulating dumb info/refs')
-        repo = get_repo(backend, mat)
         for text in generate_info_refs(repo):
             yield text
 
@@ -236,9 +241,16 @@ def handle_service_request(req, backend, mat):
     if handler_cls is None:
         yield req.forbidden('Unsupported service')
         return
+    try:
+        get_repo(backend, mat)
+    except NotGitRepository as e:
+        yield req.not_found(str(e))
+        return
     req.nocache()
     write = req.respond(HTTP_OK, 'application/x-%s-result' % service)
     proto = ReceivableProtocol(req.environ['wsgi.input'].read, write)
+    # TODO(jelmer): Find a way to pass in repo, rather than having handler_cls
+    # reopen.
     handler = handler_cls(backend, [url_prefix(mat)], proto, http_req=req)
     handler.handle()
 

setup.py

@@ -15,7 +15,7 @@ import io
 import os
 import sys
 
-dulwich_version_string = '0.19.11'
+dulwich_version_string = '0.19.12'
 
 include_dirs = []
 # Windows MSVC support
@@ -77,10 +77,10 @@ setup_kwargs = {}
 if has_setuptools:
     setup_kwargs['extras_require'] = {
         'fastimport': ['fastimport'],
-        'https': ['urllib3[secure]>=1.23'],
+        'https': ['urllib3[secure]>=1.24.1'],
         'pgp': ['gpg'],
         }
-    setup_kwargs['install_requires'] = ['urllib3>=1.23', 'certifi']
+    setup_kwargs['install_requires'] = ['urllib3>=1.24.1', 'certifi']
     setup_kwargs['include_package_data'] = True
     setup_kwargs['test_suite'] = 'dulwich.tests.test_suite'
     setup_kwargs['tests_require'] = tests_require