Import upstream version 0.20.42

.github/workflows/pythonpackage.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6, 3.7, 3.8, 3.9, 3.10-dev, pypy3]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", pypy3]
         exclude:
           # sqlite3 exit handling seems to get in the way
           - os: macos-latest
@@ -38,7 +38,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -U pip coverage codecov flake8 fastimport
+        pip install -U pip coverage codecov flake8 fastimport paramiko
     - name: Install gpg on supported platforms
       run: pip install -U gpg
       if: "matrix.os != 'windows-latest' && matrix.python-version != 'pypy3'"

.github/workflows/pythonpublish.yml → .github/workflows/pythonwheels.yml

@@ -1,12 +1,13 @@
-name: Upload Python Package
+name: Build Python Wheels
 
 on:
   push:
-    tags:
-      - dulwich-*
+  pull_request:
+  schedule:
+    - cron: '0 6 * * *'  # Daily 6AM UTC build
 
 jobs:
-  deploy:
+  build:
 
     runs-on: ${{ matrix.os }}
     strategy:
@@ -17,7 +18,7 @@ jobs:
           - os: ubuntu-latest
             python-version: '3.x'
           # path encoding
-      fail-fast: false
+      fail-fast: true
 
     steps:
     - uses: actions/checkout@v2
@@ -34,7 +35,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine fastimport
+        pip install setuptools wheel fastimport paramiko urllib3
     - name: Install gpg on supported platforms
       run: pip install -U gpg
       if: "matrix.os != 'windows-latest' && matrix.python-version != 'pypy3'"
@@ -48,12 +49,12 @@ jobs:
     - uses: docker/setup-qemu-action@v1
       name: Set up QEMU
       if: "matrix.os == 'ubuntu-latest'"
-    - name: Build and publish (Linux aarch64)
+    - name: Build (Linux aarch64)
       uses: RalfG/python-wheels-manylinux-build@v0.3.3-manylinux2014_aarch64
       with:
         python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310'
       if: "matrix.os == 'ubuntu-latest'"
-    - name: Build and publish (Linux)
+    - name: Build (Linux)
       uses: RalfG/python-wheels-manylinux-build@v0.3.1
       with:
         python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310'
@@ -62,19 +63,39 @@ jobs:
         # https://github.com/RalfG/python-wheels-manylinux-build/issues/26
         LD_LIBRARY_PATH: /usr/local/lib:${{ env.LD_LIBRARY_PATH }}
       if: "matrix.os == 'ubuntu-latest'"
-    - name: Publish (Linux)
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        # Only include *manylinux* wheels; the other wheels files are built but
-        # rejected by pip.
-        twine upload dist/*manylinux*.whl
+    - name: Upload wheels (Linux)
+      uses: actions/upload-artifact@v2
+      # Only include *manylinux* wheels; the other wheels files are built but
+      # rejected by pip.
       if: "matrix.os == 'ubuntu-latest'"
-    - name: Publish
+      with:
+        name: dist
+        path: dist/*manylinux*.whl
+    - name: Upload wheels (non-Linux)
+      uses: actions/upload-artifact@v2
+      with:
+        name: dist
+        path: dist/*.whl
+      if: "matrix.os != 'ubuntu-latest'"
+
+  publish:
+    runs-on: ubuntu-latest
+
+    needs: build
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/dulwich-')
+    steps:
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.x"
+    - name: Install twine
+      run: |
+        python -m pip install --upgrade pip
+        pip install twine
+    - name: Download wheels
+      uses: actions/download-artifact@v2
+    - name: Publish wheels
       env:
         TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
         TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        twine upload dist/*.whl
-      if: "matrix.os != 'ubuntu-latest'"
+      run: twine upload dist/*.whl

Makefile

@@ -41,6 +41,9 @@ check-pypy:: clean
 check-noextensions:: clean
 	$(RUNTEST) dulwich.tests.test_suite
 
+check-contrib:: clean
+	$(RUNTEST) -v dulwich.contrib.test_suite
+
 check-all: check check-pypy check-noextensions
 
 typing:

NEWS

@@ -1,3 +1,48 @@
+0.20.42	2022-05-24
+
+ * Drop ``RefsContainer.watch`` that was always flaky.
+   (Jelmer Vernooij, #886)
+
+0.20.41	2022-05-24
+
+ * Fix wheel uploading, properly. (Ruslan Kuprieiev)
+
+0.20.40	2022-05-19
+
+ * Fix wheel uploading. (Daniele Trifirò, Jelmer Vernooij)
+
+0.20.39	2022-05-19
+
+0.20.38	2022-05-17
+
+ * Disable paramiko tests if paramiko is not available. (Michał Górny)
+
+ * Set flag to re-enable paramiko server side on gentoo for running paramiko
+   tests. (Michał Górny)
+
+ * Increase tolerance when comparing time stamps; fixes some
+   spurious test failures on slow CI systems. (Jelmer Vernooij)
+
+ * Revert removal of caching of full HTTP response. This breaks
+   access to some HTTP servers.
+   (Jelmer Vernooij)
+
+0.20.37	2022-05-16
+
+ * Avoid making an extra copy when fetching pack files.
+   (Jelmer Vernooij)
+
+ * Add ``porcelain.remote_remove``.
+   (Jelmer Vernooij, #923)
+
+0.20.36	2022-05-15
+
+ * Add ``walk_untracked`` argument to ``porcelain.status``.
+   (Daniele Trifirò)
+
+ * Add tests for paramiko SSH Vendor.
+   (Filipp Frizzy)
+
 0.20.35	2022-03-20
 
  * Document the ``path`` attribute for ``Repo``.

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.20.35
+Version: 0.20.42
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij
@@ -26,8 +26,8 @@ Classifier: Topic :: Software Development :: Version Control
 Requires-Python: >=3.6
 Provides-Extra: fastimport
 Provides-Extra: https
+Provides-Extra: paramiko
 Provides-Extra: pgp
-Provides-Extra: watch
 License-File: COPYING
 License-File: AUTHORS
 
@@ -106,9 +106,8 @@ Help
 ----
 
 There is a *#dulwich* IRC channel on the `OFTC <https://www.oftc.net/>`_, and
-`dulwich-announce <https://groups.google.com/forum/#!forum/dulwich-announce>`_
-and `dulwich-discuss <https://groups.google.com/forum/#!forum/dulwich-discuss>`_
-mailing lists.
+a `dulwich-discuss <https://groups.google.com/forum/#!forum/dulwich-discuss>`_
+mailing list.
 
 Contributing
 ------------

README.rst

@@ -73,9 +73,8 @@ Help
 ----
 
 There is a *#dulwich* IRC channel on the `OFTC <https://www.oftc.net/>`_, and
-`dulwich-announce <https://groups.google.com/forum/#!forum/dulwich-announce>`_
-and `dulwich-discuss <https://groups.google.com/forum/#!forum/dulwich-discuss>`_
-mailing lists.
+a `dulwich-discuss <https://groups.google.com/forum/#!forum/dulwich-discuss>`_
+mailing list.
 
 Contributing
 ------------

dulwich.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.20.35
+Version: 0.20.42
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij
@@ -26,8 +26,8 @@ Classifier: Topic :: Software Development :: Version Control
 Requires-Python: >=3.6
 Provides-Extra: fastimport
 Provides-Extra: https
+Provides-Extra: paramiko
 Provides-Extra: pgp
-Provides-Extra: watch
 License-File: COPYING
 License-File: AUTHORS
 
@@ -106,9 +106,8 @@ Help
 ----
 
 There is a *#dulwich* IRC channel on the `OFTC <https://www.oftc.net/>`_, and
-`dulwich-announce <https://groups.google.com/forum/#!forum/dulwich-announce>`_
-and `dulwich-discuss <https://groups.google.com/forum/#!forum/dulwich-discuss>`_
-mailing lists.
+a `dulwich-discuss <https://groups.google.com/forum/#!forum/dulwich-discuss>`_
+mailing list.
 
 Contributing
 ------------

dulwich.egg-info/SOURCES.txt

@@ -25,7 +25,7 @@ status.yaml
 tox.ini
 .github/FUNDING.yml
 .github/workflows/pythonpackage.yml
-.github/workflows/pythonpublish.yml
+.github/workflows/pythonwheels.yml
 bin/dul-receive-pack
 bin/dul-upload-pack
 bin/dulwich

dulwich.egg-info/requires.txt

@@ -7,8 +7,8 @@ fastimport
 [https]
 urllib3[secure]>=1.24.1
 
+[paramiko]
+paramiko
+
 [pgp]
 gpg
-
-[watch]
-pyinotify

dulwich/__init__.py

@@ -22,4 +22,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 20, 35)
+__version__ = (0, 20, 42)

dulwich/client.py

@@ -47,7 +47,7 @@ import shlex
 import socket
 import subprocess
 import sys
-from typing import Optional, Dict, Callable, Set
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, IO
 
 from urllib.parse import (
     quote as urlquote,
@@ -57,6 +57,8 @@ from urllib.parse import (
     urlunsplit,
     urlunparse,
 )
+from urllib.request import url2pathname
+
 
 import dulwich
 from dulwich.config import get_xdg_config_home_path
@@ -102,16 +104,18 @@ from dulwich.protocol import (
     ZERO_SHA,
     extract_capabilities,
     parse_capability,
+    pkt_line,
 )
 from dulwich.pack import (
-    write_pack_data,
     write_pack_objects,
+    PackChunkGenerator,
 )
 from dulwich.refs import (
     read_info_refs,
     ANNOTATED_TAG_SUFFIX,
     _import_remote_refs,
 )
+from dulwich.repo import Repo
 
 
 logger = logging.getLogger(__name__)
@@ -227,11 +231,11 @@ class ReportStatusParser(object):
             self._ref_statuses.append(ref_status)
 
 
-def read_pkt_refs(proto):
+def read_pkt_refs(pkt_seq):
     server_capabilities = None
     refs = {}
     # Receive refs from server
-    for pkt in proto.read_pkt_seq():
+    for pkt in pkt_seq:
         (sha, ref) = pkt.rstrip(b"\n").split(None, 1)
         if sha == b"ERR":
             raise GitProtocolError(ref.decode("utf-8", "replace"))
@@ -402,10 +406,10 @@ class SendPackResult(object):
         return "%s(%r, %r)" % (self.__class__.__name__, self.refs, self.agent)
 
 
-def _read_shallow_updates(proto):
+def _read_shallow_updates(pkt_seq):
     new_shallow = set()
     new_unshallow = set()
-    for pkt in proto.read_pkt_seq():
+    for pkt in pkt_seq:
         cmd, sha = pkt.split(b" ", 1)
         if cmd == COMMAND_SHALLOW:
             new_shallow.add(sha.strip())
@@ -416,6 +420,89 @@ def _read_shallow_updates(proto):
     return (new_shallow, new_unshallow)
 
 
+class _v1ReceivePackHeader(object):
+
+    def __init__(self, capabilities, old_refs, new_refs):
+        self.want = []
+        self.have = []
+        self._it = self._handle_receive_pack_head(capabilities, old_refs, new_refs)
+        self.sent_capabilities = False
+
+    def __iter__(self):
+        return self._it
+
+    def _handle_receive_pack_head(self, capabilities, old_refs, new_refs):
+        """Handle the head of a 'git-receive-pack' request.
+
+        Args:
+          proto: Protocol object to read from
+          capabilities: List of negotiated capabilities
+          old_refs: Old refs, as received from the server
+          new_refs: Refs to change
+
+        Returns:
+          (have, want) tuple
+        """
+        self.have = [x for x in old_refs.values() if not x == ZERO_SHA]
+
+        for refname in new_refs:
+            if not isinstance(refname, bytes):
+                raise TypeError("refname is not a bytestring: %r" % refname)
+            old_sha1 = old_refs.get(refname, ZERO_SHA)
+            if not isinstance(old_sha1, bytes):
+                raise TypeError(
+                    "old sha1 for %s is not a bytestring: %r" % (refname, old_sha1)
+                )
+            new_sha1 = new_refs.get(refname, ZERO_SHA)
+            if not isinstance(new_sha1, bytes):
+                raise TypeError(
+                    "old sha1 for %s is not a bytestring %r" % (refname, new_sha1)
+                )
+
+            if old_sha1 != new_sha1:
+                logger.debug(
+                    'Sending updated ref %r: %r -> %r',
+                    refname, old_sha1, new_sha1)
+                if self.sent_capabilities:
+                    yield old_sha1 + b" " + new_sha1 + b" " + refname
+                else:
+                    yield (
+                        old_sha1
+                        + b" "
+                        + new_sha1
+                        + b" "
+                        + refname
+                        + b"\0"
+                        + b" ".join(sorted(capabilities))
+                    )
+                    self.sent_capabilities = True
+            if new_sha1 not in self.have and new_sha1 != ZERO_SHA:
+                self.want.append(new_sha1)
+        yield None
+
+
+def _read_side_band64k_data(pkt_seq, channel_callbacks):
+    """Read per-channel data.
+
+    This requires the side-band-64k capability.
+
+    Args:
+      pkt_seq: Sequence of packets to read
+      channel_callbacks: Dictionary mapping channels to packet
+        handlers to use. None for a callback discards channel data.
+    """
+    for pkt in pkt_seq:
+        channel = ord(pkt[:1])
+        pkt = pkt[1:]
+        try:
+            cb = channel_callbacks[channel]
+        except KeyError:
+            raise AssertionError("Invalid sideband channel %d" % channel)
+        else:
+            if cb is not None:
+                cb(pkt)
+
+
 # TODO(durin42): this doesn't correctly degrade if the server doesn't
 # support some capabilities. This should work properly with servers
 # that don't support multi_ack.
@@ -500,7 +587,6 @@ class GitClient(object):
               checkout=None, branch=None, progress=None, depth=None):
         """Clone a repository."""
         from .refs import _set_origin_head, _set_default_branch, _set_head
-        from .repo import Repo
 
         if mkdir:
             os.mkdir(target_path)
@@ -522,6 +608,7 @@ class GitClient(object):
             else:
                 encoded_path = self.get_url(path).encode('utf-8')
 
+            assert target is not None
             target_config = target.get_config()
             target_config.set((b"remote", origin.encode('utf-8')), b"url", encoded_path)
             target_config.set(
@@ -564,7 +651,16 @@ class GitClient(object):
             raise
         return target
 
-    def fetch(self, path, target, determine_wants=None, progress=None, depth=None):
+    def fetch(
+        self,
+        path: str,
+        target: Repo,
+        determine_wants: Optional[
+            Callable[[Dict[bytes, bytes], Optional[int]], List[bytes]]
+        ] = None,
+        progress: Optional[Callable[[bytes], None]] = None,
+        depth: Optional[int] = None
+    ) -> FetchPackResult:
         """Fetch into a target repository.
 
         Args:
@@ -585,15 +681,17 @@ class GitClient(object):
         if CAPABILITY_THIN_PACK in self._fetch_capabilities:
             # TODO(jelmer): Avoid reading entire file into memory and
             # only processing it after the whole file has been fetched.
-            f = BytesIO()
+            from tempfile import SpooledTemporaryFile
+            f = SpooledTemporaryFile()  # type: IO[bytes]
 
             def commit():
                 if f.tell():
                     f.seek(0)
                     target.object_store.add_thin_pack(f.read, None)
+                f.close()
 
             def abort():
-                pass
+                f.close()
 
         else:
             f, commit, abort = target.object_store.add_pack()
@@ -652,84 +750,11 @@ class GitClient(object):
         """
         raise NotImplementedError(self.get_refs)
 
-    def _read_side_band64k_data(self, proto, channel_callbacks):
-        """Read per-channel data.
-
-        This requires the side-band-64k capability.
-
-        Args:
-          proto: Protocol object to read from
-          channel_callbacks: Dictionary mapping channels to packet
-            handlers to use. None for a callback discards channel data.
-        """
-        for pkt in proto.read_pkt_seq():
-            channel = ord(pkt[:1])
-            pkt = pkt[1:]
-            try:
-                cb = channel_callbacks[channel]
-            except KeyError:
-                raise AssertionError("Invalid sideband channel %d" % channel)
-            else:
-                if cb is not None:
-                    cb(pkt)
-
     @staticmethod
     def _should_send_pack(new_refs):
         # The packfile MUST NOT be sent if the only command used is delete.
         return any(sha != ZERO_SHA for sha in new_refs.values())
 
-    def _handle_receive_pack_head(self, proto, capabilities, old_refs, new_refs):
-        """Handle the head of a 'git-receive-pack' request.
-
-        Args:
-          proto: Protocol object to read from
-          capabilities: List of negotiated capabilities
-          old_refs: Old refs, as received from the server
-          new_refs: Refs to change
-
-        Returns:
-          (have, want) tuple
-        """
-        want = []
-        have = [x for x in old_refs.values() if not x == ZERO_SHA]
-        sent_capabilities = False
-
-        for refname in new_refs:
-            if not isinstance(refname, bytes):
-                raise TypeError("refname is not a bytestring: %r" % refname)
-            old_sha1 = old_refs.get(refname, ZERO_SHA)
-            if not isinstance(old_sha1, bytes):
-                raise TypeError(
-                    "old sha1 for %s is not a bytestring: %r" % (refname, old_sha1)
-                )
-            new_sha1 = new_refs.get(refname, ZERO_SHA)
-            if not isinstance(new_sha1, bytes):
-                raise TypeError(
-                    "old sha1 for %s is not a bytestring %r" % (refname, new_sha1)
-                )
-
-            if old_sha1 != new_sha1:
-                logger.debug(
-                    'Sending updated ref %r: %r -> %r',
-                    refname, old_sha1, new_sha1)
-                if sent_capabilities:
-                    proto.write_pkt_line(old_sha1 + b" " + new_sha1 + b" " + refname)
-                else:
-                    proto.write_pkt_line(
-                        old_sha1
-                        + b" "
-                        + new_sha1
-                        + b" "
-                        + refname
-                        + b"\0"
-                        + b" ".join(sorted(capabilities))
-                    )
-                    sent_capabilities = True
-            if new_sha1 not in have and new_sha1 != ZERO_SHA:
-                want.append(new_sha1)
-        proto.write_pkt_line(None)
-        return (have, want)
-
     def _negotiate_receive_pack_capabilities(self, server_capabilities):
         negotiated_capabilities = self._send_capabilities & server_capabilities
         agent = None
@@ -772,7 +797,7 @@ class GitClient(object):
                 channel_callbacks[1] = PktLineParser(
                     self._report_status_parser.handle_packet
                 ).parse
-            self._read_side_band64k_data(proto, channel_callbacks)
+            _read_side_band64k_data(proto.read_pkt_seq(), channel_callbacks)
         else:
             if CAPABILITY_REPORT_STATUS in capabilities:
                 for pkt in proto.read_pkt_seq():
@@ -841,7 +866,7 @@ class GitClient(object):
                 )
             proto.write_pkt_line(None)
             if can_read is not None:
-                (new_shallow, new_unshallow) = _read_shallow_updates(proto)
+                (new_shallow, new_unshallow) = _read_shallow_updates(proto.read_pkt_seq())
             else:
                 new_shallow = new_unshallow = None
         else:
@@ -908,8 +933,8 @@ class GitClient(object):
                 def progress(x):
                     pass
 
-            self._read_side_band64k_data(
-                proto,
+            _read_side_band64k_data(
+                proto.read_pkt_seq(),
                 {
                     SIDE_BAND_CHANNEL_DATA: pack_data,
                     SIDE_BAND_CHANNEL_PROGRESS: progress,
@@ -996,7 +1021,7 @@ class TraditionalGitClient(GitClient):
         proto, unused_can_read, stderr = self._connect(b"receive-pack", path)
         with proto:
             try:
-                old_refs, server_capabilities = read_pkt_refs(proto)
+                old_refs, server_capabilities = read_pkt_refs(proto.read_pkt_seq())
             except HangupException:
                 raise _remote_error_from_stderr(stderr)
             (
@@ -1042,18 +1067,20 @@ class TraditionalGitClient(GitClient):
                     ref_status = None
                 return SendPackResult(old_refs, agent=agent, ref_status=ref_status)
 
-            (have, want) = self._handle_receive_pack_head(
-                proto, negotiated_capabilities, old_refs, new_refs
-            )
+            header_handler = _v1ReceivePackHeader(negotiated_capabilities, old_refs, new_refs)
+
+            for pkt in header_handler:
+                proto.write_pkt_line(pkt)
 
             pack_data_count, pack_data = generate_pack_data(
-                have,
-                want,
+                header_handler.have,
+                header_handler.want,
                 ofs_delta=(CAPABILITY_OFS_DELTA in negotiated_capabilities),
             )
 
             if self._should_send_pack(new_refs):
-                write_pack_data(proto.write_file(), pack_data_count, pack_data)
+                for chunk in PackChunkGenerator(pack_data_count, pack_data):
+                    proto.write(chunk)
 
             ref_status = self._handle_receive_pack_tail(
                 proto, negotiated_capabilities, progress
@@ -1088,7 +1115,7 @@ class TraditionalGitClient(GitClient):
         proto, can_read, stderr = self._connect(b"upload-pack", path)
         with proto:
             try:
-                refs, server_capabilities = read_pkt_refs(proto)
+                refs, server_capabilities = read_pkt_refs(proto.read_pkt_seq())
             except HangupException:
                 raise _remote_error_from_stderr(stderr)
             (
@@ -1137,7 +1164,7 @@ class TraditionalGitClient(GitClient):
         proto, _, stderr = self._connect(b"upload-pack", path)
         with proto:
             try:
-                refs, _ = read_pkt_refs(proto)
+                refs, _ = read_pkt_refs(proto.read_pkt_seq())
             except HangupException:
                 raise _remote_error_from_stderr(stderr)
             proto.write_pkt_line(None)
@@ -1180,8 +1207,8 @@ class TraditionalGitClient(GitClient):
             ret = proto.read_pkt_line()
             if ret is not None:
                 raise AssertionError("expected pkt tail")
-            self._read_side_band64k_data(
-                proto,
+            _read_side_band64k_data(
+                proto.read_pkt_seq(),
                 {
                     SIDE_BAND_CHANNEL_DATA: write_data,
                     SIDE_BAND_CHANNEL_PROGRESS: progress,
@@ -1285,7 +1312,7 @@ class SubprocessWrapper(object):
         self.proc.wait()
 
 
-def find_git_command():
+def find_git_command() -> List[str]:
     """Find command to run for system Git (usually C Git)."""
     if sys.platform == "win32":  # support .exe, .bat and .cmd
         try:  # to avoid overhead
@@ -1359,7 +1386,6 @@ class LocalGitClient(GitClient):
 
     @classmethod
     def _open_repo(cls, path):
-        from dulwich.repo import Repo
 
         if not isinstance(path, str):
             path = os.fsdecode(path)
@@ -1929,7 +1955,7 @@ class AbstractHttpGitClient(GitClient):
                     raise GitProtocolError(
                         "unexpected first line %r from smart server" % pkt
                     )
-                return read_pkt_refs(proto) + (base_url,)
+                return read_pkt_refs(proto.read_pkt_seq()) + (base_url,)
             else:
                 return read_info_refs(resp), set(), base_url
         finally:
@@ -1947,8 +1973,9 @@ class AbstractHttpGitClient(GitClient):
         headers = {
             "Content-Type": "application/x-%s-request" % service,
             "Accept": result_content_type,
-            "Content-Length": str(len(data)),
         }
+        if isinstance(data, bytes):
+            headers["Content-Length"] = str(len(data))
         resp, read = self._http_request(url, headers, data)
         if resp.content_type != result_content_type:
             raise GitProtocolError(
@@ -1996,20 +2023,21 @@ class AbstractHttpGitClient(GitClient):
             return SendPackResult(new_refs, agent=agent, ref_status={})
         if self.dumb:
             raise NotImplementedError(self.fetch_pack)
-        req_data = BytesIO()
-        req_proto = Protocol(None, req_data.write)
-        (have, want) = self._handle_receive_pack_head(
-            req_proto, negotiated_capabilities, old_refs, new_refs
-        )
-        pack_data_count, pack_data = generate_pack_data(
-            have,
-            want,
-            ofs_delta=(CAPABILITY_OFS_DELTA in negotiated_capabilities),
-        )
-        if self._should_send_pack(new_refs):
-            write_pack_data(req_proto.write_file(), pack_data_count, pack_data)
+
+        def body_generator():
+            header_handler = _v1ReceivePackHeader(negotiated_capabilities, old_refs, new_refs)
+            for pkt in header_handler:
+                yield pkt_line(pkt)
+            pack_data_count, pack_data = generate_pack_data(
+                header_handler.have,
+                header_handler.want,
+                ofs_delta=(CAPABILITY_OFS_DELTA in negotiated_capabilities),
+            )
+            if self._should_send_pack(new_refs):
+                yield from PackChunkGenerator(pack_data_count, pack_data)
+
         resp, read = self._smart_request(
-            "git-receive-pack", url, data=req_data.getvalue()
+            "git-receive-pack", url, data=body_generator()
         )
         try:
             resp_proto = Protocol(read, None)
@@ -2078,7 +2106,7 @@ class AbstractHttpGitClient(GitClient):
         try:
             resp_proto = Protocol(read, None)
             if new_shallow is None and new_unshallow is None:
-                (new_shallow, new_unshallow) = _read_shallow_updates(resp_proto)
+                (new_shallow, new_unshallow) = _read_shallow_updates(resp_proto.read_pkt_seq())
             self._handle_upload_pack_tail(
                 resp_proto,
                 negotiated_capabilities,
@@ -2177,10 +2205,11 @@ class Urllib3HttpGitClient(AbstractHttpGitClient):
             req_headers["Accept-Encoding"] = "identity"
 
         if data is None:
-            resp = self.pool_manager.request("GET", url, headers=req_headers)
+            resp = self.pool_manager.request(
+                "GET", url, headers=req_headers, preload_content=False)
         else:
             resp = self.pool_manager.request(
-                "POST", url, headers=req_headers, body=data
+                "POST", url, headers=req_headers, body=data, preload_content=False
             )
 
         if resp.status == 404:
@@ -2194,13 +2223,6 @@ class Urllib3HttpGitClient(AbstractHttpGitClient):
                 "unexpected http resp %d for %s" % (resp.status, url)
             )
 
-        # TODO: Optimization available by adding `preload_content=False` to the
-        # request and just passing the `read` method on instead of going via
-        # `BytesIO`, if we can guarantee that the entire response is consumed
-        # before issuing the next to still allow for connection reuse from the
-        # pool.
-        read = BytesIO(resp.data).read
-
         resp.content_type = resp.getheader("Content-Type")
         # Check if geturl() is available (urllib3 version >= 1.23)
         try:
@@ -2210,12 +2232,41 @@ class Urllib3HttpGitClient(AbstractHttpGitClient):
             resp.redirect_location = resp.get_redirect_location()
         else:
             resp.redirect_location = resp_url if resp_url != url else ""
-        return resp, read
+        # TODO(jelmer): Remove BytesIO() call that caches entire response in
+        # memory. See https://github.com/jelmer/dulwich/issues/966
+        return resp, BytesIO(resp.data).read
 
 
 HttpGitClient = Urllib3HttpGitClient
 
 
+def _win32_url_to_path(parsed) -> str:
+    """
+    Convert a file: URL to a path.
+
+    https://datatracker.ietf.org/doc/html/rfc8089
+    """
+    assert sys.platform == "win32" or os.name == "nt"
+    assert parsed.scheme == "file"
+
+    _, netloc, path, _, _, _ = parsed
+
+    if netloc == "localhost" or not netloc:
+        netloc = ""
+    elif (
+        netloc
+        and len(netloc) >= 2
+        and netloc[0].isalpha()
+        and netloc[1:2] in (":", ":/")
+    ):
+        # file://C:/foo.bar/baz or file://C://foo.bar//baz
+        netloc = netloc[:2]
+    else:
+        raise NotImplementedError("Non-local file URLs are not supported")
+
+    return url2pathname(netloc + path)
+
+
 def get_transport_and_path_from_url(url, config=None, **kwargs):
     """Obtain a git client from a URL.
 
@@ -2241,6 +2292,8 @@ def get_transport_and_path_from_url(url, config=None, **kwargs):
             parsed.path,
         )
     elif parsed.scheme == "file":
+        if sys.platform == "win32" or os.name == "nt":
+            return default_local_git_client_cls(**kwargs), _win32_url_to_path(parsed)
         return (
             default_local_git_client_cls.from_parsedurl(parsed, **kwargs),
             parsed.path,
@@ -2268,7 +2321,10 @@ def parse_rsync_url(location):
     return (user, host, path)
 
 
-def get_transport_and_path(location, **kwargs):
+def get_transport_and_path(
+    location: str,
+    **kwargs: Any
+) -> Tuple[GitClient, str]:
     """Obtain a git client from a URL.
 
     Args:

dulwich/config.py

@@ -30,7 +30,7 @@ import os
 import sys
 import warnings
 
-from typing import BinaryIO, Tuple, Optional
+from typing import BinaryIO, Iterator, KeysView, Optional, Tuple, Union
 
 try:
     from collections.abc import (
@@ -87,7 +87,7 @@ class CaseInsensitiveOrderedMultiDict(MutableMapping):
     def __len__(self):
         return len(self._keyed)
 
-    def keys(self):
+    def keys(self) -> KeysView[Tuple[bytes, ...]]:
         return self._keyed.keys()
 
     def items(self):
@@ -241,7 +241,7 @@ class Config(object):
         """
         raise NotImplementedError(self.sections)
 
-    def has_section(self, name):
+    def has_section(self, name: Tuple[bytes, ...]) -> bool:
         """Check if a specified section exists.
 
         Args:
@@ -320,7 +320,11 @@ class ConfigDict(Config, MutableMapping):
 
         return self._values[(section[0],)].get_all(name)
 
-    def get(self, section, name):
+    def get(  # type: ignore[override]
+        self,
+        section: Union[bytes, str, Tuple[Union[bytes, str], ...]],
+        name: Union[str, bytes]
+    ) -> Optional[bytes]:
         section, name = self._check_section_and_name(section, name)
 
         if len(section) > 1:
@@ -679,7 +683,7 @@ class StackedConfig(Config):
         return self.writable.set(section, name, value)
 
 
-def parse_submodules(config):
+def parse_submodules(config: ConfigFile) -> Iterator[Tuple[bytes, bytes, bytes]]:
     """Parse a gitmodules GitConfig file, returning submodules.
 
     Args:
@@ -692,5 +696,7 @@ def parse_submodules(config):
         section_kind, section_name = section
         if section_kind == b"submodule":
             sm_path = config.get(section, b"path")
+            assert sm_path is not None
             sm_url = config.get(section, b"url")
+            assert sm_url is not None
             yield (sm_path, sm_url, section_name)

dulwich/contrib/__init__.py

@@ -23,6 +23,7 @@ def test_suite():
     import unittest
 
     names = [
+        "paramiko_vendor",
         "release_robot",
         "swift",
     ]

dulwich/contrib/test_paramiko_vendor.py

@@ -20,16 +20,49 @@
 """Tests for paramiko_vendor."""
 
 import socket
-import paramiko
 import threading
 
 from dulwich.tests import TestCase
-from dulwich.contrib.paramiko_vendor import ParamikoSSHVendor
+
+from io import StringIO
+from unittest import skipIf
 
 try:
-    from StringIO import StringIO
+    import paramiko
 except ImportError:
-    from io import StringIO
+    has_paramiko = False
+else:
+    has_paramiko = True
+    from dulwich.contrib.paramiko_vendor import ParamikoSSHVendor
+
+    class Server(paramiko.ServerInterface):
+        """http://docs.paramiko.org/en/2.4/api/server.html"""
+        def __init__(self, commands, *args, **kwargs):
+            super(Server, self).__init__(*args, **kwargs)
+            self.commands = commands
+
+        def check_channel_exec_request(self, channel, command):
+            self.commands.append(command)
+            return True
+
+        def check_auth_password(self, username, password):
+            if username == USER and password == PASSWORD:
+                return paramiko.AUTH_SUCCESSFUL
+            return paramiko.AUTH_FAILED
+
+        def check_auth_publickey(self, username, key):
+            pubkey = paramiko.RSAKey.from_private_key(StringIO(CLIENT_KEY))
+            if username == USER and key == pubkey:
+                return paramiko.AUTH_SUCCESSFUL
+            return paramiko.AUTH_FAILED
+
+        def check_channel_request(self, kind, chanid):
+            if kind == "session":
+                return paramiko.OPEN_SUCCEEDED
+            return paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED
+
+        def get_allowed_auths(self, username):
+            return "password,publickey"
 
 
 USER = 'testuser'
@@ -60,7 +93,8 @@ cNj+6W2guZ2tyHuPhZ64/4SJVyE2hKDSKD4xTb2nVjsMeN0bLD2UWXC9mwbx8nWa
 R6legDG2e/50ph7yc8gwAaA1kUXMiuLi8Nfkw/3yyvmJwklNegi4aRzRbA2Mzhi2
 4q9WMQKBgQCb0JNyxHG4pvLWCF/j0Sm1FfvrpnqSv5678n1j4GX7Ka/TubOK1Y4K
 U+Oib7dKa/zQMWehVFNTayrsq6bKVZ6q7zG+IHiRLw4wjeAxREFH6WUjDrn9vl2l
-D48DKbBuBwuVOJWyq3qbfgJXojscgNQklrsPdXVhDwOF0dYxP89HnA=="""
+D48DKbBuBwuVOJWyq3qbfgJXojscgNQklrsPdXVhDwOF0dYxP89HnA==
+-----END RSA PRIVATE KEY-----"""
 CLIENT_KEY = """\
 -----BEGIN RSA PRIVATE KEY-----
 MIIEpAIBAAKCAQEAxvREKSElPOm/0z/nPO+j5rk2tjdgGcGc7We1QZ6TRXYLu7nN
@@ -91,38 +125,16 @@ WxtWBWHwxfSmqgTXilEA3ALJp0kNolLnEttnhENwJpZHlqtes0ZA4w==
 -----END RSA PRIVATE KEY-----"""
 
 
-class Server(paramiko.ServerInterface):
-    """http://docs.paramiko.org/en/2.4/api/server.html"""
-    def __init__(self, commands, *args, **kwargs):
-        super(Server, self).__init__(*args, **kwargs)
-        self.commands = commands
-
-    def check_channel_exec_request(self, channel, command):
-        self.commands.append(command)
-        return True
-
-    def check_auth_password(self, username, password):
-        if username == USER and password == PASSWORD:
-            return paramiko.AUTH_SUCCESSFUL
-        return paramiko.AUTH_FAILED
-
-    def check_auth_publickey(self, username, key):
-        pubkey = paramiko.RSAKey.from_private_key(StringIO(CLIENT_KEY))
-        if username == USER and key == pubkey:
-            return paramiko.AUTH_SUCCESSFUL
-        return paramiko.AUTH_FAILED
-
-    def check_channel_request(self, kind, chanid):
-        if kind == "session":
-            return paramiko.OPEN_SUCCEEDED
-        return paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED
+@skipIf(not has_paramiko, "paramiko is not installed")
+class ParamikoSSHVendorTests(TestCase):
 
-    def get_allowed_auths(self, username):
-        return "password,publickey"
+    def setUp(self):
+        import paramiko.transport
 
+        # reenable server functionality for tests
+        if hasattr(paramiko.transport, "SERVER_DISABLED_BY_GENTOO"):
+            paramiko.transport.SERVER_DISABLED_BY_GENTOO = False
 
-class ParamikoSSHVendorTests(TestCase):
-    def setUp(self):
         self.commands = []
         socket.setdefaulttimeout(10)
         self.addCleanup(socket.setdefaulttimeout, None)
@@ -135,7 +147,7 @@ class ParamikoSSHVendorTests(TestCase):
         self.thread.start()
 
     def tearDown(self):
-        pass
+        self.thread.join()
 
     def _run(self):
         try:

dulwich/contrib/test_swift.py

@@ -49,17 +49,17 @@ missing_libs = []
 
 try:
     import gevent  # noqa:F401
-except ImportError:
+except ModuleNotFoundError:
     missing_libs.append("gevent")
 
 try:
     import geventhttpclient  # noqa:F401
-except ImportError:
+except ModuleNotFoundError:
     missing_libs.append("geventhttpclient")
 
 try:
     from unittest.mock import patch
-except ImportError:
+except ModuleNotFoundError:
     missing_libs.append("mock")
 
 skipmsg = "Required libraries are not installed (%r)" % missing_libs

dulwich/object_store.py

@@ -27,6 +27,8 @@ import os
 import stat
 import sys
 
+from typing import Callable, Dict, List, Optional, Tuple
+
 from dulwich.diff_tree import (
     tree_changes,
     walk_trees,
@@ -79,7 +81,11 @@ PACK_MODE = 0o444 if sys.platform != "win32" else 0o644
 class BaseObjectStore(object):
     """Object store interface."""
 
-    def determine_wants_all(self, refs, depth=None):
+    def determine_wants_all(
+        self,
+        refs: Dict[bytes, bytes],
+        depth: Optional[int] = None
+    ) -> List[bytes]:
         def _want_deepen(sha):
             if not depth:
                 return False
@@ -142,6 +148,12 @@ class BaseObjectStore(object):
         """Iterate over the SHAs that are present in this store."""
         raise NotImplementedError(self.__iter__)
 
+    def add_pack(
+        self
+    ) -> Tuple[BytesIO, Callable[[], None], Callable[[], None]]:
+        """Add a new pack to this object store."""
+        raise NotImplementedError(self.add_pack)
+
     def add_object(self, obj):
         """Add a single object to this object store."""
         raise NotImplementedError(self.add_object)

dulwich/objects.py

@@ -564,12 +564,6 @@ class ShaFile(object):
             raise TypeError
         return self.id <= other.id
 
-    def __cmp__(self, other):
-        """Compare the SHA of this object with that of the other object."""
-        if not isinstance(other, ShaFile):
-            raise TypeError
-        return cmp(self.id, other.id)  # noqa: F821
-
 
 class Blob(ShaFile):
     """A Git Blob object."""

dulwich/pack.py

@@ -1713,6 +1713,66 @@ def write_pack_objects(
     )
 
 
+class PackChunkGenerator(object):
+
+    def __init__(self, num_records=None, records=None, progress=None, compression_level=-1):
+        self.cs = sha1(b"")
+        self.entries = {}
+        self._it = self._pack_data_chunks(
+            num_records=num_records, records=records, progress=progress, compression_level=compression_level)
+
+    def sha1digest(self):
+        return self.cs.digest()
+
+    def __iter__(self):
+        return self._it
+
+    def _pack_data_chunks(self, num_records=None, records=None, progress=None, compression_level=-1):
+        """Iterate pack data file chunks..
+
+        Args:
+          num_records: Number of records (defaults to len(records) if None)
+          records: Iterator over type_num, object_id, delta_base, raw
+          progress: Function to report progress to
+          compression_level: the zlib compression level
+        Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
+        """
+        # Write the pack
+        if num_records is None:
+            num_records = len(records)
+        f = BytesIO()
+        write_pack_header(f, num_records)
+        self.cs.update(f.getvalue())
+        yield f.getvalue()
+        offset = f.tell()
+        actual_num_records = 0
+        for i, (type_num, object_id, delta_base, raw) in enumerate(records):
+            if progress is not None:
+                progress(("writing pack data: %d/%d\r" % (i, num_records)).encode("ascii"))
+            if delta_base is not None:
+                try:
+                    base_offset, base_crc32 = self.entries[delta_base]
+                except KeyError:
+                    type_num = REF_DELTA
+                    raw = (delta_base, raw)
+                else:
+                    type_num = OFS_DELTA
+                    raw = (offset - base_offset, raw)
+            f = BytesIO()
+            crc32 = write_pack_object(f, type_num, raw, compression_level=compression_level)
+            self.cs.update(f.getvalue())
+            yield f.getvalue()
+            actual_num_records += 1
+            self.entries[object_id] = (offset, crc32)
+            offset += f.tell()
+        if actual_num_records != num_records:
+            raise AssertionError(
+                'actual records written differs: %d != %d' % (
+                    actual_num_records, num_records))
+
+        yield self.cs.digest()
+
+
 def write_pack_data(f, num_records=None, records=None, progress=None, compression_level=-1):
     """Write a new pack data file.
 
@@ -1724,34 +1784,12 @@ def write_pack_data(f, num_records=None, records=None, progress=None, compressio
       compression_level: the zlib compression level
     Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
     """
-    # Write the pack
-    entries = {}
-    f = SHA1Writer(f)
-    if num_records is None:
-        num_records = len(records)
-    write_pack_header(f, num_records)
-    actual_num_records = 0
-    for i, (type_num, object_id, delta_base, raw) in enumerate(records):
-        if progress is not None:
-            progress(("writing pack data: %d/%d\r" % (i, num_records)).encode("ascii"))
-        offset = f.offset()
-        if delta_base is not None:
-            try:
-                base_offset, base_crc32 = entries[delta_base]
-            except KeyError:
-                type_num = REF_DELTA
-                raw = (delta_base, raw)
-            else:
-                type_num = OFS_DELTA
-                raw = (offset - base_offset, raw)
-        crc32 = write_pack_object(f, type_num, raw, compression_level=compression_level)
-        actual_num_records += 1
-        entries[object_id] = (offset, crc32)
-    if actual_num_records != num_records:
-        raise AssertionError(
-            'actual records written differs: %d != %d' % (
-                actual_num_records, num_records))
-    return entries, f.write_sha()
+    chunk_generator = PackChunkGenerator(
+        num_records=num_records, records=records, progress=progress,
+        compression_level=compression_level)
+    for chunk in chunk_generator:
+        f.write(chunk)
+    return chunk_generator.entries, chunk_generator.sha1digest()
 
 
 def write_pack_index_v1(f, entries, pack_checksum):

dulwich/porcelain.py

@@ -438,7 +438,7 @@ def clone(
 
     mkdir = not os.path.exists(target)
 
-    (client, path) = get_transport_and_path(source)
+    (client, path) = get_transport_and_path(source, **kwargs)
 
     return client.clone(
         path,
@@ -1002,6 +1002,7 @@ def get_remote_repo(
     if config.has_section(section):
         remote_name = encoded_location.decode()
         url = config.get(section, "url")
+        assert url is not None
         encoded_location = url
     else:
         remote_name = None
@@ -1155,12 +1156,20 @@ def pull(
             _import_remote_refs(r.refs, remote_name, fetch_result.refs)
 
 
-def status(repo=".", ignored=False):
+def status(repo=".", ignored=False, untracked_files="all"):
     """Returns staged, unstaged, and untracked changes relative to the HEAD.
 
     Args:
       repo: Path to repository or repository object
       ignored: Whether to include ignored files in untracked
+      untracked_files: How to handle untracked files, defaults to "all":
+          "no": do not return untracked files
+          "all": include all files in untracked directories
+        Using `untracked_files="no"` can be faster than "all" when the worktreee
+          contains many untracked files/directories.
+
+    Note: `untracked_files="normal" (`git`'s default) is not implemented.
+
     Returns: GitStatus tuple,
         staged -  dict with lists of staged paths (diff index/HEAD)
         unstaged -  list of unstaged paths (diff index/working-tree)
@@ -1176,7 +1185,11 @@ def status(repo=".", ignored=False):
         unstaged_changes = list(get_unstaged_changes(index, r.path, filter_callback))
 
         untracked_paths = get_untracked_paths(
-            r.path, r.path, index, exclude_ignored=not ignored
+            r.path,
+            r.path,
+            index,
+            exclude_ignored=not ignored,
+            untracked_files=untracked_files,
         )
         untracked_changes = list(untracked_paths)
 
@@ -1215,7 +1228,9 @@ def _walk_working_dir_paths(frompath, basepath, prune_dirnames=None):
             dirnames[:] = prune_dirnames(dirpath, dirnames)
 
 
-def get_untracked_paths(frompath, basepath, index, exclude_ignored=False):
+def get_untracked_paths(
+    frompath, basepath, index, exclude_ignored=False, untracked_files="all"
+):
     """Get untracked paths.
 
     Args:
@@ -1223,11 +1238,24 @@ def get_untracked_paths(frompath, basepath, index, exclude_ignored=False):
       basepath: Path to compare to
       index: Index to check against
       exclude_ignored: Whether to exclude ignored paths
+      untracked_files: How to handle untracked files:
+        - "no": return an empty list
+        - "all": return all files in untracked directories
+        - "normal": Not implemented
 
     Note: ignored directories will never be walked for performance reasons.
       If exclude_ignored is False, only the path to an ignored directory will
       be yielded, no files inside the directory will be returned
     """
+    if untracked_files == "normal":
+        raise NotImplementedError("normal is not yet supported")
+
+    if untracked_files not in ("no", "all"):
+        raise ValueError("untracked_files must be one of (no, all)")
+
+    if untracked_files == "no":
+        return
+
     with open_repo_closing(basepath) as r:
         ignore_manager = IgnoreFilterManager.from_repo(r)
 
@@ -1251,11 +1279,8 @@ def get_untracked_paths(frompath, basepath, index, exclude_ignored=False):
         if not is_dir:
             ip = path_to_tree_path(basepath, ap)
             if ip not in index:
-                if (
-                    not exclude_ignored
-                    or not ignore_manager.is_ignored(
-                        os.path.relpath(ap, basepath)
-                    )
+                if not exclude_ignored or not ignore_manager.is_ignored(
+                    os.path.relpath(ap, basepath)
                 ):
                     yield os.path.relpath(ap, frompath)
 
@@ -1614,7 +1639,7 @@ def ls_tree(
         list_tree(r.object_store, tree.id, "")
 
 
-def remote_add(repo, name, url):
+def remote_add(repo: Repo, name: Union[bytes, str], url: Union[bytes, str]):
     """Add a remote.
 
     Args:
@@ -1635,6 +1660,22 @@ def remote_add(repo, name, url):
         c.write_to_path()
 
 
+def remote_remove(repo: Repo, name: Union[bytes, str]):
+    """Remove a remote
+
+    Args:
+      repo: Path to the repository
+      name: Remote name
+    """
+    if not isinstance(name, bytes):
+        name = name.encode(DEFAULT_ENCODING)
+    with open_repo_closing(repo) as r:
+        c = r.get_config()
+        section = (b"remote", name)
+        del c[section]
+        c.write_to_path()
+
+
 def check_ignore(repo, paths, no_index=False):
     """Debug gitignore files.
 

dulwich/refs.py

@@ -158,13 +158,13 @@ class RefsContainer(object):
 
     def import_refs(
         self,
-        base,
-        other,
-        committer=None,
-        timestamp=None,
-        timezone=None,
-        message=None,
-        prune=False,
+        base: bytes,
+        other: Dict[bytes, bytes],
+        committer: Optional[bytes] = None,
+        timestamp: Optional[bytes] = None,
+        timezone: Optional[bytes] = None,
+        message: Optional[bytes] = None,
+        prune: bool = False,
     ):
         if prune:
             to_delete = set(self.subkeys(base))
@@ -430,35 +430,6 @@ class RefsContainer(object):
                 ret[src] = dst
         return ret
 
-    def watch(self):
-        """Watch for changes to the refs in this container.
-
-        Returns a context manager that yields tuples with (refname, new_sha)
-        """
-        raise NotImplementedError(self.watch)
-
-
-class _DictRefsWatcher(object):
-    def __init__(self, refs):
-        self._refs = refs
-
-    def __enter__(self):
-        from queue import Queue
-
-        self.queue = Queue()
-        self._refs._watchers.add(self)
-        return self
-
-    def __next__(self):
-        return self.queue.get()
-
-    def _notify(self, entry):
-        self.queue.put_nowait(entry)
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self._refs._watchers.remove(self)
-        return False
-
 
 class DictRefsContainer(RefsContainer):
     """RefsContainer backed by a simple dict.
@@ -486,9 +457,6 @@ class DictRefsContainer(RefsContainer):
         for watcher in self._watchers:
             watcher._notify((ref, newsha))
 
-    def watch(self):
-        return _DictRefsWatcher(self)
-
     def set_symbolic_ref(
         self,
         name,
@@ -642,50 +610,6 @@ class InfoRefsContainer(RefsContainer):
             return self._refs[name]
 
 
-class _InotifyRefsWatcher(object):
-    def __init__(self, path):
-        import pyinotify
-        from queue import Queue
-
-        self.path = os.fsdecode(path)
-        self.manager = pyinotify.WatchManager()
-        self.manager.add_watch(
-            self.path,
-            pyinotify.IN_DELETE | pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO,
-            rec=True,
-            auto_add=True,
-        )
-
-        self.notifier = pyinotify.ThreadedNotifier(
-            self.manager, default_proc_fun=self._notify
-        )
-        self.queue = Queue()
-
-    def _notify(self, event):
-        if event.dir:
-            return
-        if event.pathname.endswith(".lock"):
-            return
-        ref = os.fsencode(os.path.relpath(event.pathname, self.path))
-        if event.maskname == "IN_DELETE":
-            self.queue.put_nowait((ref, None))
-        elif event.maskname in ("IN_CLOSE_WRITE", "IN_MOVED_TO"):
-            with open(event.pathname, "rb") as f:
-                sha = f.readline().rstrip(b"\n\r")
-                self.queue.put_nowait((ref, sha))
-
-    def __next__(self):
-        return self.queue.get()
-
-    def __enter__(self):
-        self.notifier.start()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.notifier.stop()
-        return False
-
-
 class DiskRefsContainer(RefsContainer):
     """Refs container that reads refs from disk."""
 
@@ -1085,11 +1009,6 @@ class DiskRefsContainer(RefsContainer):
 
         return True
 
-    def watch(self):
-        import pyinotify  # noqa: F401
-
-        return _InotifyRefsWatcher(self.path)
-
 
 def _split_ref_line(line):
     """Split a single ref line into a tuple of SHA1 and name."""

dulwich/repo.py

@@ -1057,7 +1057,12 @@ class Repo(BaseRepo):
       bare (bool): Whether this is a bare repository
     """
 
-    def __init__(self, root, object_store=None, bare=None):
+    def __init__(
+        self,
+        root: str,
+        object_store: Optional[BaseObjectStore] = None,
+        bare: Optional[bool] = None
+    ) -> None:
         hidden_path = os.path.join(root, CONTROLDIR)
         if bare is None:
             if (os.path.isfile(hidden_path) or
@@ -1093,9 +1098,18 @@ class Repo(BaseRepo):
         self.path = root
         config = self.get_config()
         try:
-            format_version = int(config.get("core", "repositoryformatversion"))
+            repository_format_version = config.get(
+                "core",
+                "repositoryformatversion"
+            )
+            format_version = (
+                0
+                if repository_format_version is None
+                else int(repository_format_version)
+            )
         except KeyError:
             format_version = 0
+
         if format_version != 0:
             raise UnsupportedVersion(format_version)
         if object_store is None:
@@ -1485,7 +1499,7 @@ class Repo(BaseRepo):
             raise
         return target
 
-    def reset_index(self, tree=None):
+    def reset_index(self, tree: Optional[Tree] = None):
         """Reset the index back to a specific tree.
 
         Args:
@@ -1569,7 +1583,7 @@ class Repo(BaseRepo):
         return ret
 
     @classmethod
-    def init(cls, path, mkdir=False):
+    def init(cls, path: str, mkdir: bool = False) -> "Repo":
         """Create a new repository.
 
         Args:

dulwich/tests/compat/test_client.py

@@ -602,9 +602,23 @@ class GitHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
         try:
             nbytes = int(length)
         except (TypeError, ValueError):
-            nbytes = 0
-        if self.command.lower() == "post" and nbytes > 0:
-            data = self.rfile.read(nbytes)
+            nbytes = -1
+        if self.command.lower() == "post":
+            if nbytes > 0:
+                data = self.rfile.read(nbytes)
+            elif self.headers.get('transfer-encoding') == 'chunked':
+                chunks = []
+                while True:
+                    line = self.rfile.readline()
+                    length = int(line.rstrip(), 16)
+                    chunk = self.rfile.read(length + 2)
+                    chunks.append(chunk[:-2])
+                    if length == 0:
+                        break
+                data = b''.join(chunks)
+                env["CONTENT_LENGTH"] = str(len(data))
+            else:
+                raise AssertionError
         else:
             data = None
             env["CONTENT_LENGTH"] = "0"

dulwich/tests/test_client.py

@@ -31,6 +31,8 @@ from urllib.parse import (
     urlparse,
 )
 
+from unittest.mock import patch
+
 import dulwich
 from dulwich import (
     client,
@@ -682,11 +684,41 @@ class TestGetTransportAndPathFromUrl(TestCase):
         self.assertIsInstance(c, HttpGitClient)
         self.assertEqual("/jelmer/dulwich", path)
 
+    @patch("os.name", "posix")
+    @patch("sys.platform", "linux")
     def test_file(self):
         c, path = get_transport_and_path_from_url("file:///home/jelmer/foo")
         self.assertIsInstance(c, LocalGitClient)
         self.assertEqual("/home/jelmer/foo", path)
 
+    @patch("os.name", "nt")
+    @patch("sys.platform", "win32")
+    def test_file_win(self):
+        # `_win32_url_to_path` uses urllib.request.url2pathname, which is set to
+        # `ntutl2path.url2pathname`  when `os.name==nt`
+        from nturl2path import url2pathname
+
+        with patch("dulwich.client.url2pathname", url2pathname):
+            expected = "C:\\foo.bar\\baz"
+            for file_url in [
+                "file:C:/foo.bar/baz",
+                "file:/C:/foo.bar/baz",
+                "file://C:/foo.bar/baz",
+                "file://C://foo.bar//baz",
+                "file:///C:/foo.bar/baz",
+            ]:
+                c, path = get_transport_and_path(file_url)
+                self.assertIsInstance(c, LocalGitClient)
+                self.assertEqual(path, expected)
+
+            for remote_url in [
+                "file://host.example.com/C:/foo.bar/baz"
+                "file://host.example.com/C:/foo.bar/baz"
+                "file:////host.example/foo.bar/baz",
+            ]:
+                with self.assertRaises(NotImplementedError):
+                    c, path = get_transport_and_path(remote_url)
+
 
 class TestSSHVendor(object):
     def __init__(self):
@@ -1048,7 +1080,7 @@ class HttpGitClientTests(TestCase):
             def __init__(self):
                 self.headers = {}
 
-            def request(self, method, url, fields=None, headers=None, redirect=True):
+            def request(self, method, url, fields=None, headers=None, redirect=True, preload_content=True):
                 base_url = url[: -len(tail)]
                 redirect_base_url = test_data[base_url]["redirect_url"]
                 redirect_url = redirect_base_url + tail
@@ -1063,14 +1095,15 @@ class HttpGitClientTests(TestCase):
                 if redirect is False:
                     request_url = url
                     if redirect_base_url != base_url:
-                        body = ""
+                        body = b""
                         headers["location"] = redirect_url
                         status = 301
                 return HTTPResponse(
-                    body=body,
+                    body=BytesIO(body),
                     headers=headers,
                     request_method=method,
                     request_url=request_url,
+                    preload_content=preload_content,
                     status=status,
                 )
 

dulwich/tests/test_porcelain.py

@@ -79,6 +79,11 @@ class PorcelainTestCase(TestCase):
         self.repo = Repo.init(self.repo_path, mkdir=True)
         self.addCleanup(self.repo.close)
 
+    def assertRecentTimestamp(self, ts):
+        # On some slow CIs it does actually take more than 5 seconds to go from
+        # creating the tag to here.
+        self.assertLess(time.time() - ts, 50)
+
 
 class PorcelainGpgTestCase(PorcelainTestCase):
     DEFAULT_KEY = """
@@ -1112,6 +1117,7 @@ class RevListTests(PorcelainTestCase):
 
 @skipIf(platform.python_implementation() == "PyPy" or sys.platform == "win32", "gpgme not easily available or supported on Windows and PyPy")
 class TagCreateSignTests(PorcelainGpgTestCase):
+
     def test_default_key(self):
         import gpg
 
@@ -1138,7 +1144,7 @@ class TagCreateSignTests(PorcelainGpgTestCase):
         self.assertIsInstance(tag, Tag)
         self.assertEqual(b"foo <foo@bar.com>", tag.tagger)
         self.assertEqual(b"bar\n", tag.message)
-        self.assertLess(time.time() - tag.tag_time, 5)
+        self.assertRecentTimestamp(tag.tag_time)
         tag = self.repo[b'refs/tags/tryme']
         # GPG Signatures aren't deterministic, so we can't do a static assertion.
         tag.verify()
@@ -1181,13 +1187,14 @@ class TagCreateSignTests(PorcelainGpgTestCase):
         self.assertIsInstance(tag, Tag)
         self.assertEqual(b"foo <foo@bar.com>", tag.tagger)
         self.assertEqual(b"bar\n", tag.message)
-        self.assertLess(time.time() - tag.tag_time, 5)
+        self.assertRecentTimestamp(tag.tag_time)
         tag = self.repo[b'refs/tags/tryme']
         # GPG Signatures aren't deterministic, so we can't do a static assertion.
         tag.verify()
 
 
 class TagCreateTests(PorcelainTestCase):
+
     def test_annotated(self):
         c1, c2, c3 = build_commit_graph(
             self.repo.object_store, [[1], [2, 1], [3, 1, 2]]
@@ -1208,7 +1215,7 @@ class TagCreateTests(PorcelainTestCase):
         self.assertIsInstance(tag, Tag)
         self.assertEqual(b"foo <foo@bar.com>", tag.tagger)
         self.assertEqual(b"bar\n", tag.message)
-        self.assertLess(time.time() - tag.tag_time, 5)
+        self.assertRecentTimestamp(tag.tag_time)
 
     def test_unannotated(self):
         c1, c2, c3 = build_commit_graph(
@@ -1874,7 +1881,12 @@ class StatusTests(PorcelainTestCase):
             results.staged,
         )
         self.assertListEqual(results.unstaged, [b"blye"])
-        self.assertListEqual(results.untracked, ["blyat"])
+        results_no_untracked = porcelain.status(self.repo.path, untracked_files="no")
+        self.assertListEqual(results_no_untracked.untracked, [])
+
+    def test_status_wrong_untracked_files_value(self):
+        with self.assertRaises(ValueError):
+            porcelain.status(self.repo.path, untracked_files="antani")
 
     def test_status_crlf_mismatch(self):
         # First make a commit as if the file has been added on a Linux system
@@ -2170,6 +2182,22 @@ class StatusTests(PorcelainTestCase):
             )
         )
 
+    def test_get_untracked_paths_invalid_untracked_files(self):
+        with self.assertRaises(ValueError):
+            list(
+                porcelain.get_untracked_paths(
+                    self.repo.path,
+                    self.repo.path,
+                    self.repo.open_index(),
+                    untracked_files="invalid_value",
+                )
+            )
+
+    def test_get_untracked_paths_normal(self):
+        with self.assertRaises(NotImplementedError):
+            _, _, _ = porcelain.status(
+                repo=self.repo.path, untracked_files="normal"
+            )
 
 # TODO(jelmer): Add test for dulwich.porcelain.daemon
 
@@ -2524,6 +2552,20 @@ class RemoteAddTests(PorcelainTestCase):
         )
 
 
+class RemoteRemoveTests(PorcelainTestCase):
+    def test_remove(self):
+        porcelain.remote_add(self.repo, "jelmer", "git://jelmer.uk/code/dulwich")
+        c = self.repo.get_config()
+        self.assertEqual(
+            c.get((b"remote", b"jelmer"), b"url"),
+            b"git://jelmer.uk/code/dulwich",
+        )
+        porcelain.remote_remove(self.repo, "jelmer")
+        self.assertRaises(KeyError, porcelain.remote_remove, self.repo, "jelmer")
+        c = self.repo.get_config()
+        self.assertRaises(KeyError, c.get, (b"remote", b"jelmer"), b"url")
+
+
 class CheckIgnoreTests(PorcelainTestCase):
     def test_check_ignored(self):
         with open(os.path.join(self.repo.path, ".gitignore"), "w") as f:

dulwich/tests/test_refs.py

@@ -355,38 +355,6 @@ class RefsContainerTests(object):
         )
         self.assertNotIn(b"refs/remotes/origin/other", self._refs)
 
-    def test_watch(self):
-        try:
-            watcher = self._refs.watch()
-        except (NotImplementedError, ImportError):
-            self.skipTest("watching not supported")
-        with watcher:
-            self._refs[
-                b"refs/remotes/origin/other"
-            ] = b"48d01bd4b77fed026b154d16493e5deab78f02ec"
-            change = next(watcher)
-            self.assertEqual(
-                (
-                    b"refs/remotes/origin/other",
-                    b"48d01bd4b77fed026b154d16493e5deab78f02ec",
-                ),
-                change,
-            )
-            self._refs[
-                b"refs/remotes/origin/other"
-            ] = b"48d01bd4b77fed026b154d16493e5deab78f02ed"
-            change = next(watcher)
-            self.assertEqual(
-                (
-                    b"refs/remotes/origin/other",
-                    b"48d01bd4b77fed026b154d16493e5deab78f02ed",
-                ),
-                change,
-            )
-            del self._refs[b"refs/remotes/origin/other"]
-            change = next(watcher)
-            self.assertEqual((b"refs/remotes/origin/other", None), change)
-
 
 class DictRefsContainerTests(RefsContainerTests, TestCase):
     def setUp(self):

setup.py

@@ -23,7 +23,7 @@ if sys.version_info < (3, 6):
         'For 2.7 support, please install a version prior to 0.20')
 
 
-dulwich_version_string = '0.20.35'
+dulwich_version_string = '0.20.42'
 
 
 class DulwichDistribution(Distribution):
@@ -78,7 +78,7 @@ if has_setuptools:
         'fastimport': ['fastimport'],
         'https': ['urllib3[secure]>=1.24.1'],
         'pgp': ['gpg'],
-        'watch': ['pyinotify'],
+        'paramiko': ['paramiko'],
         }
     setup_kwargs['install_requires'] = ['urllib3>=1.24.1', 'certifi']
     setup_kwargs['include_package_data'] = True