Import upstream version 0.20.43

NEWS

@@ -1,3 +1,11 @@
+0.20.43	2022-06-07
+
+ * Lazily import url2pathname.
+   (Jelmer Vernooij)
+
+ * Drop caching of full HTTP response. Attempt #2.
+   (jelmer Vernooij, Antoine Lambert, #966)
+
 0.20.42	2022-05-24
 
  * Drop ``RefsContainer.watch`` that was always flaky.

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.20.42
+Version: 0.20.43
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij

dulwich.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.20.42
+Version: 0.20.43
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij

dulwich/__init__.py

@@ -22,4 +22,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 20, 42)
+__version__ = (0, 20, 43)

dulwich/client.py

@@ -43,7 +43,6 @@ from io import BytesIO, BufferedReader
 import logging
 import os
 import select
-import shlex
 import socket
 import subprocess
 import sys
@@ -57,7 +56,6 @@ from urllib.parse import (
     urlunsplit,
     urlunparse,
 )
-from urllib.request import url2pathname
 
 
 import dulwich
@@ -118,6 +116,10 @@ from dulwich.refs import (
 from dulwich.repo import Repo
 
 
+# url2pathname is lazily imported
+url2pathname = None
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -503,6 +505,129 @@ def _read_side_band64k_data(pkt_seq, channel_callbacks):
                 cb(pkt)
 
 
+def _handle_upload_pack_head(
+    proto, capabilities, graph_walker, wants, can_read, depth
+):
+    """Handle the head of a 'git-upload-pack' request.
+
+    Args:
+      proto: Protocol object to read from
+      capabilities: List of negotiated capabilities
+      graph_walker: GraphWalker instance to call .ack() on
+      wants: List of commits to fetch
+      can_read: function that returns a boolean that indicates
+    whether there is extra graph data to read on proto
+      depth: Depth for request
+
+    Returns:
+
+    """
+    assert isinstance(wants, list) and isinstance(wants[0], bytes)
+    proto.write_pkt_line(
+        COMMAND_WANT
+        + b" "
+        + wants[0]
+        + b" "
+        + b" ".join(sorted(capabilities))
+        + b"\n"
+    )
+    for want in wants[1:]:
+        proto.write_pkt_line(COMMAND_WANT + b" " + want + b"\n")
+    if depth not in (0, None) or getattr(graph_walker, "shallow", None):
+        if CAPABILITY_SHALLOW not in capabilities:
+            raise GitProtocolError(
+                "server does not support shallow capability required for " "depth"
+            )
+        for sha in graph_walker.shallow:
+            proto.write_pkt_line(COMMAND_SHALLOW + b" " + sha + b"\n")
+        if depth is not None:
+            proto.write_pkt_line(
+                COMMAND_DEEPEN + b" " + str(depth).encode("ascii") + b"\n"
+            )
+        proto.write_pkt_line(None)
+        if can_read is not None:
+            (new_shallow, new_unshallow) = _read_shallow_updates(proto.read_pkt_seq())
+        else:
+            new_shallow = new_unshallow = None
+    else:
+        new_shallow = new_unshallow = set()
+        proto.write_pkt_line(None)
+    have = next(graph_walker)
+    while have:
+        proto.write_pkt_line(COMMAND_HAVE + b" " + have + b"\n")
+        if can_read is not None and can_read():
+            pkt = proto.read_pkt_line()
+            parts = pkt.rstrip(b"\n").split(b" ")
+            if parts[0] == b"ACK":
+                graph_walker.ack(parts[1])
+                if parts[2] in (b"continue", b"common"):
+                    pass
+                elif parts[2] == b"ready":
+                    break
+                else:
+                    raise AssertionError(
+                        "%s not in ('continue', 'ready', 'common)" % parts[2]
+                    )
+        have = next(graph_walker)
+    proto.write_pkt_line(COMMAND_DONE + b"\n")
+    return (new_shallow, new_unshallow)
+
+
+def _handle_upload_pack_tail(
+    proto,
+    capabilities,
+    graph_walker,
+    pack_data,
+    progress=None,
+    rbufsize=_RBUFSIZE,
+):
+    """Handle the tail of a 'git-upload-pack' request.
+
+    Args:
+      proto: Protocol object to read from
+      capabilities: List of negotiated capabilities
+      graph_walker: GraphWalker instance to call .ack() on
+      pack_data: Function to call with pack data
+      progress: Optional progress reporting function
+      rbufsize: Read buffer size
+
+    Returns:
+
+    """
+    pkt = proto.read_pkt_line()
+    while pkt:
+        parts = pkt.rstrip(b"\n").split(b" ")
+        if parts[0] == b"ACK":
+            graph_walker.ack(parts[1])
+        if len(parts) < 3 or parts[2] not in (
+            b"ready",
+            b"continue",
+            b"common",
+        ):
+            break
+        pkt = proto.read_pkt_line()
+    if CAPABILITY_SIDE_BAND_64K in capabilities:
+        if progress is None:
+            # Just ignore progress data
+
+            def progress(x):
+                pass
+
+        _read_side_band64k_data(
+            proto.read_pkt_seq(),
+            {
+                SIDE_BAND_CHANNEL_DATA: pack_data,
+                SIDE_BAND_CHANNEL_PROGRESS: progress,
+            },
+        )
+    else:
+        while True:
+            data = proto.read(rbufsize)
+            if data == b"":
+                break
+            pack_data(data)
+
+
 # TODO(durin42): this doesn't correctly degrade if the server doesn't
 # support some capabilities. This should work properly with servers
 # that don't support multi_ack.
@@ -825,128 +950,6 @@ class GitClient(object):
         negotiated_capabilities = self._fetch_capabilities & server_capabilities
         return (negotiated_capabilities, symrefs, agent)
 
-    def _handle_upload_pack_head(
-        self, proto, capabilities, graph_walker, wants, can_read, depth
-    ):
-        """Handle the head of a 'git-upload-pack' request.
-
-        Args:
-          proto: Protocol object to read from
-          capabilities: List of negotiated capabilities
-          graph_walker: GraphWalker instance to call .ack() on
-          wants: List of commits to fetch
-          can_read: function that returns a boolean that indicates
-        whether there is extra graph data to read on proto
-          depth: Depth for request
-
-        Returns:
-
-        """
-        assert isinstance(wants, list) and isinstance(wants[0], bytes)
-        proto.write_pkt_line(
-            COMMAND_WANT
-            + b" "
-            + wants[0]
-            + b" "
-            + b" ".join(sorted(capabilities))
-            + b"\n"
-        )
-        for want in wants[1:]:
-            proto.write_pkt_line(COMMAND_WANT + b" " + want + b"\n")
-        if depth not in (0, None) or getattr(graph_walker, "shallow", None):
-            if CAPABILITY_SHALLOW not in capabilities:
-                raise GitProtocolError(
-                    "server does not support shallow capability required for " "depth"
-                )
-            for sha in graph_walker.shallow:
-                proto.write_pkt_line(COMMAND_SHALLOW + b" " + sha + b"\n")
-            if depth is not None:
-                proto.write_pkt_line(
-                    COMMAND_DEEPEN + b" " + str(depth).encode("ascii") + b"\n"
-                )
-            proto.write_pkt_line(None)
-            if can_read is not None:
-                (new_shallow, new_unshallow) = _read_shallow_updates(proto.read_pkt_seq())
-            else:
-                new_shallow = new_unshallow = None
-        else:
-            new_shallow = new_unshallow = set()
-            proto.write_pkt_line(None)
-        have = next(graph_walker)
-        while have:
-            proto.write_pkt_line(COMMAND_HAVE + b" " + have + b"\n")
-            if can_read is not None and can_read():
-                pkt = proto.read_pkt_line()
-                parts = pkt.rstrip(b"\n").split(b" ")
-                if parts[0] == b"ACK":
-                    graph_walker.ack(parts[1])
-                    if parts[2] in (b"continue", b"common"):
-                        pass
-                    elif parts[2] == b"ready":
-                        break
-                    else:
-                        raise AssertionError(
-                            "%s not in ('continue', 'ready', 'common)" % parts[2]
-                        )
-            have = next(graph_walker)
-        proto.write_pkt_line(COMMAND_DONE + b"\n")
-        return (new_shallow, new_unshallow)
-
-    def _handle_upload_pack_tail(
-        self,
-        proto,
-        capabilities,
-        graph_walker,
-        pack_data,
-        progress=None,
-        rbufsize=_RBUFSIZE,
-    ):
-        """Handle the tail of a 'git-upload-pack' request.
-
-        Args:
-          proto: Protocol object to read from
-          capabilities: List of negotiated capabilities
-          graph_walker: GraphWalker instance to call .ack() on
-          pack_data: Function to call with pack data
-          progress: Optional progress reporting function
-          rbufsize: Read buffer size
-
-        Returns:
-
-        """
-        pkt = proto.read_pkt_line()
-        while pkt:
-            parts = pkt.rstrip(b"\n").split(b" ")
-            if parts[0] == b"ACK":
-                graph_walker.ack(parts[1])
-            if len(parts) < 3 or parts[2] not in (
-                b"ready",
-                b"continue",
-                b"common",
-            ):
-                break
-            pkt = proto.read_pkt_line()
-        if CAPABILITY_SIDE_BAND_64K in capabilities:
-            if progress is None:
-                # Just ignore progress data
-
-                def progress(x):
-                    pass
-
-            _read_side_band64k_data(
-                proto.read_pkt_seq(),
-                {
-                    SIDE_BAND_CHANNEL_DATA: pack_data,
-                    SIDE_BAND_CHANNEL_PROGRESS: progress,
-                },
-            )
-        else:
-            while True:
-                data = proto.read(rbufsize)
-                if data == b"":
-                    break
-                pack_data(data)
-
 
 def check_wants(wants, refs):
     """Check that a set of wants is valid.
@@ -1141,7 +1144,7 @@ class TraditionalGitClient(GitClient):
             if not wants:
                 proto.write_pkt_line(None)
                 return FetchPackResult(refs, symrefs, agent)
-            (new_shallow, new_unshallow) = self._handle_upload_pack_head(
+            (new_shallow, new_unshallow) = _handle_upload_pack_head(
                 proto,
                 negotiated_capabilities,
                 graph_walker,
@@ -1149,7 +1152,7 @@ class TraditionalGitClient(GitClient):
                 can_read,
                 depth=depth,
             )
-            self._handle_upload_pack_tail(
+            _handle_upload_pack_tail(
                 proto,
                 negotiated_capabilities,
                 graph_walker,
@@ -1612,6 +1615,7 @@ class SubprocessSSHVendor(SSHVendor):
             )
 
         if ssh_command:
+            import shlex
             args = shlex.split(ssh_command) + ["-x"]
         else:
             args = ["ssh", "-x"]
@@ -1653,6 +1657,7 @@ class PLinkSSHVendor(SSHVendor):
     ):
 
         if ssh_command:
+            import shlex
             args = shlex.split(ssh_command) + ["-ssh"]
         elif sys.platform == "win32":
             args = ["plink.exe", "-ssh"]
@@ -1905,14 +1910,13 @@ class AbstractHttpGitClient(GitClient):
         self.dumb = dumb
         GitClient.__init__(self, **kwargs)
 
-    def _http_request(self, url, headers=None, data=None, allow_compression=False):
+    def _http_request(self, url, headers=None, data=None):
         """Perform HTTP request.
 
         Args:
           url: Request URL.
           headers: Optional custom headers to override defaults.
           data: Request data.
-          allow_compression: Allow GZipped communication.
 
         Returns:
           Tuple (`response`, `read`), where response is an `urllib3`
@@ -1931,7 +1935,7 @@ class AbstractHttpGitClient(GitClient):
         if self.dumb is not True:
             tail += "?service=%s" % service.decode("ascii")
         url = urljoin(base_url, tail)
-        resp, read = self._http_request(url, headers, allow_compression=True)
+        resp, read = self._http_request(url, headers)
 
         if resp.redirect_location:
             # Something changed (redirect!), so let's update the base URL
@@ -2092,7 +2096,7 @@ class AbstractHttpGitClient(GitClient):
             raise NotImplementedError(self.fetch_pack)
         req_data = BytesIO()
         req_proto = Protocol(None, req_data.write)
-        (new_shallow, new_unshallow) = self._handle_upload_pack_head(
+        (new_shallow, new_unshallow) = _handle_upload_pack_head(
             req_proto,
             negotiated_capabilities,
             graph_walker,
@@ -2106,8 +2110,9 @@ class AbstractHttpGitClient(GitClient):
         try:
             resp_proto = Protocol(read, None)
             if new_shallow is None and new_unshallow is None:
-                (new_shallow, new_unshallow) = _read_shallow_updates(resp_proto.read_pkt_seq())
-            self._handle_upload_pack_tail(
+                (new_shallow, new_unshallow) = _read_shallow_updates(
+                    resp_proto.read_pkt_seq())
+            _handle_upload_pack_tail(
                 resp_proto,
                 negotiated_capabilities,
                 graph_walker,
@@ -2194,15 +2199,11 @@ class Urllib3HttpGitClient(AbstractHttpGitClient):
             path = path.decode("utf-8")
         return urljoin(self._base_url, path).rstrip("/") + "/"
 
-    def _http_request(self, url, headers=None, data=None, allow_compression=False):
+    def _http_request(self, url, headers=None, data=None):
         req_headers = self.pool_manager.headers.copy()
         if headers is not None:
             req_headers.update(headers)
         req_headers["Pragma"] = "no-cache"
-        if allow_compression:
-            req_headers["Accept-Encoding"] = "gzip"
-        else:
-            req_headers["Accept-Encoding"] = "identity"
 
         if data is None:
             resp = self.pool_manager.request(
@@ -2232,17 +2233,14 @@ class Urllib3HttpGitClient(AbstractHttpGitClient):
             resp.redirect_location = resp.get_redirect_location()
         else:
             resp.redirect_location = resp_url if resp_url != url else ""
-        # TODO(jelmer): Remove BytesIO() call that caches entire response in
-        # memory. See https://github.com/jelmer/dulwich/issues/966
-        return resp, BytesIO(resp.data).read
+        return resp, resp.read
 
 
 HttpGitClient = Urllib3HttpGitClient
 
 
 def _win32_url_to_path(parsed) -> str:
-    """
-    Convert a file: URL to a path.
+    """Convert a file: URL to a path.
 
     https://datatracker.ietf.org/doc/html/rfc8089
     """
@@ -2264,7 +2262,10 @@ def _win32_url_to_path(parsed) -> str:
     else:
         raise NotImplementedError("Non-local file URLs are not supported")
 
-    return url2pathname(netloc + path)
+    global url2pathname
+    if url2pathname is None:
+        from urllib.request import url2pathname  # type: ignore
+    return url2pathname(netloc + path)  # type: ignore
 
 
 def get_transport_and_path_from_url(url, config=None, **kwargs):

dulwich/config.py

@@ -30,18 +30,16 @@ import os
 import sys
 import warnings
 
-from typing import BinaryIO, Iterator, KeysView, Optional, Tuple, Union
-
-try:
-    from collections.abc import (
-        Iterable,
-        MutableMapping,
-    )
-except ImportError:  # python < 3.7
-    from collections import (  # type: ignore
-        Iterable,
-        MutableMapping,
-    )
+from typing import (
+    BinaryIO,
+    Iterable,
+    Iterator,
+    KeysView,
+    MutableMapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from dulwich.file import GitFile
 
@@ -252,35 +250,52 @@ class Config(object):
         return name in self.sections()
 
 
-class ConfigDict(Config, MutableMapping):
+BytesLike = Union[bytes, str]
+Key = Tuple[bytes, ...]
+KeyLike = Union[bytes, str, Tuple[BytesLike, ...]]
+Value = Union[bytes, bool]
+ValueLike = Union[bytes, str, bool]
+
+
+class ConfigDict(Config, MutableMapping[Key, MutableMapping[bytes, Value]]):
     """Git configuration stored in a dictionary."""
 
-    def __init__(self, values=None, encoding=None):
+    def __init__(
+        self,
+        values: Union[
+            MutableMapping[Key, MutableMapping[bytes, Value]], None
+        ] = None,
+        encoding: Union[str, None] = None
+    ) -> None:
         """Create a new ConfigDict."""
         if encoding is None:
             encoding = sys.getdefaultencoding()
         self.encoding = encoding
         self._values = CaseInsensitiveOrderedMultiDict.make(values)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return "%s(%r)" % (self.__class__.__name__, self._values)
 
-    def __eq__(self, other):
+    def __eq__(self, other: object) -> bool:
         return isinstance(other, self.__class__) and other._values == self._values
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: Key) -> MutableMapping[bytes, Value]:
         return self._values.__getitem__(key)
 
-    def __setitem__(self, key, value):
+    def __setitem__(
+        self,
+        key: Key,
+        value: MutableMapping[bytes, Value]
+    ) -> None:
         return self._values.__setitem__(key, value)
 
-    def __delitem__(self, key):
+    def __delitem__(self, key: Key) -> None:
         return self._values.__delitem__(key)
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator[Key]:
         return self._values.__iter__()
 
-    def __len__(self):
+    def __len__(self) -> int:
         return self._values.__len__()
 
     @classmethod
@@ -291,11 +306,15 @@ class ConfigDict(Config, MutableMapping):
         else:
             return (parts[0], None, parts[1])
 
-    def _check_section_and_name(self, section, name):
+    def _check_section_and_name(
+        self,
+        section: KeyLike,
+        name: BytesLike
+    ) -> Tuple[Key, bytes]:
         if not isinstance(section, tuple):
             section = (section,)
 
-        section = tuple(
+        checked_section = tuple(
             [
                 subsection.encode(self.encoding)
                 if not isinstance(subsection, bytes)
@@ -307,9 +326,13 @@ class ConfigDict(Config, MutableMapping):
         if not isinstance(name, bytes):
             name = name.encode(self.encoding)
 
-        return section, name
+        return checked_section, name
 
-    def get_multivar(self, section, name):
+    def get_multivar(
+        self,
+        section: KeyLike,
+        name: BytesLike
+    ) -> Iterator[Value]:
         section, name = self._check_section_and_name(section, name)
 
         if len(section) > 1:
@@ -322,9 +345,9 @@ class ConfigDict(Config, MutableMapping):
 
     def get(  # type: ignore[override]
         self,
-        section: Union[bytes, str, Tuple[Union[bytes, str], ...]],
-        name: Union[str, bytes]
-    ) -> Optional[bytes]:
+        section: KeyLike,
+        name: BytesLike,
+    ) -> Optional[Value]:
         section, name = self._check_section_and_name(section, name)
 
         if len(section) > 1:
@@ -335,18 +358,26 @@ class ConfigDict(Config, MutableMapping):
 
         return self._values[(section[0],)][name]
 
-    def set(self, section, name, value):
+    def set(
+        self,
+        section: KeyLike,
+        name: BytesLike,
+        value: ValueLike,
+    ) -> None:
         section, name = self._check_section_and_name(section, name)
 
-        if type(value) not in (bool, bytes):
+        if not isinstance(value, (bytes, bool)):
             value = value.encode(self.encoding)
 
         self._values.setdefault(section)[name] = value
 
-    def items(self, section):
+    def items(  # type: ignore[override]
+        self,
+        section: Key
+    ) -> Iterator[Value]:
         return self._values.get(section).items()
 
-    def sections(self):
+    def sections(self) -> Iterator[Key]:
         return self._values.keys()
 
 
@@ -696,7 +727,9 @@ def parse_submodules(config: ConfigFile) -> Iterator[Tuple[bytes, bytes, bytes]]
         section_kind, section_name = section
         if section_kind == b"submodule":
             sm_path = config.get(section, b"path")
+            assert isinstance(sm_path, bytes)
             assert sm_path is not None
             sm_url = config.get(section, b"url")
             assert sm_url is not None
+            assert isinstance(sm_url, bytes)
             yield (sm_path, sm_url, section_name)

dulwich/porcelain.py

@@ -1003,6 +1003,7 @@ def get_remote_repo(
         remote_name = encoded_location.decode()
         url = config.get(section, "url")
         assert url is not None
+        assert isinstance(url, bytes)
         encoded_location = url
     else:
         remote_name = None

setup.py

@@ -23,7 +23,7 @@ if sys.version_info < (3, 6):
         'For 2.7 support, please install a version prior to 0.20')
 
 
-dulwich_version_string = '0.20.42'
+dulwich_version_string = '0.20.43'
 
 
 class DulwichDistribution(Distribution):