New upstream version 0.18.3

AUTHORS

@@ -128,5 +128,7 @@ Segev Finer <segev208@gmail.com>
 fviolette <fviolette@talend.com>
 dzhuang <dzhuang.scut@gmail.com>
 Antoine Pietri <antoine.pietri1@gmail.com>
+Taras Postument <trane9991@gmail.com>
+Earl Chew <earl_chew@yahoo.com>
 
 If you contributed but are missing from this list, please send me an e-mail.

NEWS

@@ -1,3 +1,30 @@
+0.18.3	2017-09-03
+
+ BUG FIXES
+
+  * Read config during porcelain operations that involve remotes.
+    (Jelmer Vernooij, #545)
+
+  * Fix headers of empty chunks in unified diffs. (Taras Postument, #543)
+
+  * Properly follow redirects over HTTP. (Jelmer Vernooij, #117)
+
+ IMPROVEMENTS
+
+  * Add ``dulwich.porcelain.update_head``. (Jelmer Vernooij, #439)
+
+  * ``GitClient.fetch_pack`` now returns symrefs.
+    (Jelmer Vernooij, #485)
+
+  * The server now supports providing symrefs.
+    (Jelmer Vernooij, #485)
+
+  * Add ``dulwich.object_store.commit_tree_changes`` to incrementally
+    commit changes to a tree structure. (Jelmer Vernooij)
+
+  * Add basic ``PackBasedObjectStore.repack`` method.
+    (Jelmer Vernooij, Earl Chew, #296, #549, #552)
+
 0.18.2	2017-08-01
 
  TEST FIXES
@@ -671,7 +698,7 @@ API CHANGES
   * Add a basic `dulwich.porcelain` module. (Jelmer Vernooij, Marcin Kuzminski)
 
   * Various performance improvements for object access.
-   (Jelmer Vernooij)
+    (Jelmer Vernooij)
 
   * New function `get_transport_and_path_from_url`,
     similar to `get_transport_and_path` but only

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: dulwich
-Version: 0.18.2
+Version: 0.18.3
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: UNKNOWN

README.md

@@ -1,5 +1,5 @@
 [![Build Status](https://travis-ci.org/jelmer/dulwich.png?branch=master)](https://travis-ci.org/jelmer/dulwich)
-[![Windows Build status](https://ci.appveyor.com/api/projects/status/cnothr6pxprfx2lf/branch/master?svg=true)](https://ci.appveyor.com/project/jelmer/dulwich-njb6g/branch/master)
+[![Windows Build status](https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true)](https://ci.appveyor.com/project/jelmer/dulwich/branch/master)
 
 This is the Dulwich project.
 
@@ -60,7 +60,7 @@ And to print it using porcelain:
 Further documentation
 ---------------------
 
-The dulwich documentation can be found in doc/ and
+The dulwich documentation can be found in docs/ and
 [on the web](https://www.dulwich.io/docs/).
 
 The API reference can be generated using pydoctor, by running "make pydoctor",

bin/dulwich

@@ -480,6 +480,19 @@ class cmd_pull(Command):
         porcelain.pull('.', from_location)
 
 
+class cmd_push(Command):
+
+    def run(self, args):
+        parser = optparse.OptionParser()
+        options, args = parser.parse_args(args)
+        if len(args) < 2:
+            print("Usage: dulwich push TO-LOCATION REFSPEC..")
+            sys.exit(1)
+        to_location = args[0]
+        refspecs = args[1:]
+        porcelain.push('.', to_location, refspecs)
+
+
 class cmd_remote_add(Command):
 
     def run(self, args):
@@ -562,6 +575,7 @@ commands = {
     "ls-tree": cmd_ls_tree,
     "pack-objects": cmd_pack_objects,
     "pull": cmd_pull,
+    "push": cmd_push,
     "receive-pack": cmd_receive_pack,
     "remote": cmd_remote,
     "repack": cmd_repack,

docs/tutorial/object-store.txt

@@ -175,7 +175,7 @@ write_tree_diff::
   index c55063a..16ee268 100644
   --- a/spam
   +++ b/spam
-  @@ -1,1 +1,1 @@
+  @@ -1 +1 @@
   -My file content
   +My new file content
 

docs/tutorial/remote.txt

@@ -60,7 +60,7 @@ which we will write to a ``BytesIO`` object::
 
    >>> from io import BytesIO
    >>> f = BytesIO()
-   >>> remote_refs = client.fetch_pack(b"/", determine_wants,
+   >>> result = client.fetch_pack(b"/", determine_wants,
    ...    DummyGraphWalker(), pack_data=f.write)
 
 ``f`` will now contain a full pack file::

dulwich.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: dulwich
-Version: 0.18.2
+Version: 0.18.3
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: UNKNOWN

dulwich/__init__.py

@@ -22,4 +22,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 18, 2)
+__version__ = (0, 18, 3)

dulwich/client.py

@@ -40,7 +40,7 @@ Known capabilities that are not supported:
 
 from contextlib import closing
 from io import BytesIO, BufferedReader
-import dulwich
+import gzip
 import select
 import socket
 import subprocess
@@ -60,6 +60,7 @@ except ImportError:
     import urllib.request as urllib2
     import urllib.parse as urlparse
 
+import dulwich
 from dulwich.errors import (
     GitProtocolError,
     NotGitRepository,
@@ -68,16 +69,22 @@ from dulwich.errors import (
     )
 from dulwich.protocol import (
     _RBUFSIZE,
+    agent_string,
     capability_agent,
+    extract_capability_names,
+    CAPABILITY_AGENT,
     CAPABILITY_DELETE_REFS,
     CAPABILITY_MULTI_ACK,
     CAPABILITY_MULTI_ACK_DETAILED,
     CAPABILITY_OFS_DELTA,
     CAPABILITY_QUIET,
     CAPABILITY_REPORT_STATUS,
+    CAPABILITY_SYMREF,
     CAPABILITY_SIDE_BAND_64K,
     CAPABILITY_THIN_PACK,
     CAPABILITIES_REF,
+    KNOWN_RECEIVE_CAPABILITIES,
+    KNOWN_UPLOAD_CAPABILITIES,
     COMMAND_DONE,
     COMMAND_HAVE,
     COMMAND_WANT,
@@ -90,6 +97,7 @@ from dulwich.protocol import (
     TCP_GIT_PORT,
     ZERO_SHA,
     extract_capabilities,
+    parse_capability,
     )
 from dulwich.pack import (
     write_pack_objects,
@@ -118,10 +126,9 @@ def _win32_peek_avail(handle):
 
 
 COMMON_CAPABILITIES = [CAPABILITY_OFS_DELTA, CAPABILITY_SIDE_BAND_64K]
-FETCH_CAPABILITIES = ([CAPABILITY_THIN_PACK, CAPABILITY_MULTI_ACK,
-                       CAPABILITY_MULTI_ACK_DETAILED] +
-                      COMMON_CAPABILITIES)
-SEND_CAPABILITIES = [CAPABILITY_REPORT_STATUS] + COMMON_CAPABILITIES
+UPLOAD_CAPABILITIES = ([CAPABILITY_THIN_PACK, CAPABILITY_MULTI_ACK,
+                        CAPABILITY_MULTI_ACK_DETAILED] + COMMON_CAPABILITIES)
+RECEIVE_CAPABILITIES = [CAPABILITY_REPORT_STATUS] + COMMON_CAPABILITIES
 
 
 class ReportStatusParser(object):
@@ -202,6 +209,62 @@ def read_pkt_refs(proto):
     return refs, set(server_capabilities)
 
 
+class FetchPackResult(object):
+    """Result of a fetch-pack operation.
+
+    :var refs: Dictionary with all remote refs
+    :var symrefs: Dictionary with remote symrefs
+    :var agent: User agent string
+    """
+
+    _FORWARDED_ATTRS = [
+            'clear', 'copy', 'fromkeys', 'get', 'has_key', 'items',
+            'iteritems', 'iterkeys', 'itervalues', 'keys', 'pop', 'popitem',
+            'setdefault', 'update', 'values', 'viewitems', 'viewkeys',
+            'viewvalues']
+
+    def __init__(self, refs, symrefs, agent):
+        self.refs = refs
+        self.symrefs = symrefs
+        self.agent = agent
+
+    def _warn_deprecated(self):
+        import warnings
+        warnings.warn(
+            "Use FetchPackResult.refs instead.",
+            DeprecationWarning, stacklevel=3)
+
+    def __eq__(self, other):
+        if isinstance(other, dict):
+            self._warn_deprecated()
+            return (self.refs == other)
+        return (self.refs == other.refs and
+                self.symrefs == other.symrefs and
+                self.agent == other.agent)
+
+    def __contains__(self, name):
+        self._warn_deprecated()
+        return name in self.refs
+
+    def __getitem__(self, name):
+        self._warn_deprecated()
+        return self.refs[name]
+
+    def __len__(self):
+        self._warn_deprecated()
+        return len(self.refs)
+
+    def __iter__(self):
+        self._warn_deprecated()
+        return iter(self.refs)
+
+    def __getattribute__(self, name):
+        if name in type(self)._FORWARDED_ATTRS:
+            self._warn_deprecated()
+            return getattr(self.refs, name)
+        return super(FetchPackResult, self).__getattribute__(name)
+
+
 # TODO(durin42): this doesn't correctly degrade if the server doesn't
 # support some capabilities. This should work properly with servers
 # that don't support multi_ack.
@@ -219,9 +282,9 @@ class GitClient(object):
         """
         self._report_activity = report_activity
         self._report_status_parser = None
-        self._fetch_capabilities = set(FETCH_CAPABILITIES)
+        self._fetch_capabilities = set(UPLOAD_CAPABILITIES)
         self._fetch_capabilities.add(capability_agent())
-        self._send_capabilities = set(SEND_CAPABILITIES)
+        self._send_capabilities = set(RECEIVE_CAPABILITIES)
         self._send_capabilities.add(capability_agent())
         if quiet:
             self._send_capabilities.add(CAPABILITY_QUIET)
@@ -316,7 +379,7 @@ class GitClient(object):
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
-        :return: Dictionary with all remote refs (not just those fetched)
+        :return: FetchPackResult object
         """
         raise NotImplementedError(self.fetch_pack)
 
@@ -424,6 +487,15 @@ class GitClient(object):
         proto.write_pkt_line(None)
         return (have, want)
 
+    def _negotiate_receive_pack_capabilities(self, server_capabilities):
+        negotiated_capabilities = (
+            self._send_capabilities & server_capabilities)
+        unknown_capabilities = (  # noqa: F841
+            extract_capability_names(server_capabilities) -
+            KNOWN_RECEIVE_CAPABILITIES)
+        # TODO(jelmer): warn about unknown capabilities
+        return negotiated_capabilities
+
     def _handle_receive_pack_tail(self, proto, capabilities, progress=None):
         """Handle the tail of a 'git-receive-pack' request.
 
@@ -431,7 +503,7 @@ class GitClient(object):
         :param capabilities: List of negotiated capabilities
         :param progress: Optional progress reporting function
         """
-        if b"side-band-64k" in capabilities:
+        if CAPABILITY_SIDE_BAND_64K in capabilities:
             if progress is None:
                 def progress(x):
                     pass
@@ -447,6 +519,25 @@ class GitClient(object):
         if self._report_status_parser is not None:
             self._report_status_parser.check()
 
+    def _negotiate_upload_pack_capabilities(self, server_capabilities):
+        unknown_capabilities = (  # noqa: F841
+            extract_capability_names(server_capabilities) -
+            KNOWN_UPLOAD_CAPABILITIES)
+        # TODO(jelmer): warn about unknown capabilities
+        symrefs = {}
+        agent = None
+        for capability in server_capabilities:
+            k, v = parse_capability(capability)
+            if k == CAPABILITY_SYMREF:
+                (src, dst) = v.split(b':', 1)
+                symrefs[src] = dst
+            if k == CAPABILITY_AGENT:
+                agent = v
+
+        negotiated_capabilities = (
+            self._fetch_capabilities & server_capabilities)
+        return (negotiated_capabilities, symrefs, agent)
+
     def _handle_upload_pack_head(self, proto, capabilities, graph_walker,
                                  wants, can_read):
         """Handle the head of a 'git-upload-pack' request.
@@ -567,9 +658,8 @@ class TraditionalGitClient(GitClient):
         proto, unused_can_read = self._connect(b'receive-pack', path)
         with proto:
             old_refs, server_capabilities = read_pkt_refs(proto)
-            negotiated_capabilities = (
-                self._send_capabilities & server_capabilities)
-
+            negotiated_capabilities = \
+                self._negotiate_receive_pack_capabilities(server_capabilities)
             if CAPABILITY_REPORT_STATUS in negotiated_capabilities:
                 self._report_status_parser = ReportStatusParser()
             report_status_parser = self._report_status_parser
@@ -632,17 +722,18 @@ class TraditionalGitClient(GitClient):
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
-        :return: Dictionary with all remote refs (not just those fetched)
+        :return: FetchPackResult object
         """
         proto, can_read = self._connect(b'upload-pack', path)
         with proto:
             refs, server_capabilities = read_pkt_refs(proto)
-            negotiated_capabilities = (
-                self._fetch_capabilities & server_capabilities)
+            negotiated_capabilities, symrefs, agent = (
+                    self._negotiate_upload_pack_capabilities(
+                            server_capabilities))
 
             if refs is None:
                 proto.write_pkt_line(None)
-                return refs
+                return FetchPackResult(refs, symrefs, agent)
 
             try:
                 wants = determine_wants(refs)
@@ -653,13 +744,13 @@ class TraditionalGitClient(GitClient):
                 wants = [cid for cid in wants if cid != ZERO_SHA]
             if not wants:
                 proto.write_pkt_line(None)
-                return refs
+                return FetchPackResult(refs, symrefs, agent)
             self._handle_upload_pack_head(
                 proto, negotiated_capabilities, graph_walker, wants, can_read)
             self._handle_upload_pack_tail(
                 proto, negotiated_capabilities, graph_walker, pack_data,
                 progress)
-            return refs
+            return FetchPackResult(refs, symrefs, agent)
 
     def get_refs(self, path):
         """Retrieve the current refs from a git smart server."""
@@ -833,7 +924,7 @@ class SubprocessGitClient(TraditionalGitClient):
 class LocalGitClient(GitClient):
     """Git Client that just uses a local Repo."""
 
-    def __init__(self, thin_packs=True, report_activity=None):
+    def __init__(self, thin_packs=True, report_activity=None, config=None):
         """Create a new LocalGitClient instance.
 
         :param thin_packs: Whether or not thin packs should be retrieved
@@ -938,18 +1029,20 @@ class LocalGitClient(GitClient):
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
-        :return: Dictionary with all remote refs (not just those fetched)
+        :return: FetchPackResult object
         """
         with self._open_repo(path) as r:
             objects_iter = r.fetch_objects(
                 determine_wants, graph_walker, progress)
+            symrefs = r.refs.get_symrefs()
+            agent = agent_string()
 
             # Did the process short-circuit (e.g. in a stateless RPC call)?
             # Note that the client still expects a 0-object pack in most cases.
             if objects_iter is None:
-                return
+                return FetchPackResult(None, symrefs, agent)
             write_pack_objects(ProtocolFile(None, pack_data), objects_iter)
-            return r.get_refs()
+            return FetchPackResult(r.get_refs(), symrefs, agent)
 
     def get_refs(self, path):
         """Retrieve the current refs from a git smart server."""
@@ -1019,7 +1112,8 @@ get_ssh_vendor = SubprocessSSHVendor
 
 class SSHGitClient(TraditionalGitClient):
 
-    def __init__(self, host, port=None, username=None, vendor=None, **kwargs):
+    def __init__(self, host, port=None, username=None, vendor=None,
+                 config=None, **kwargs):
         self.host = host
         self.port = port
         self.username = username
@@ -1072,7 +1166,10 @@ def default_user_agent_string():
 
 def default_urllib2_opener(config):
     if config is not None:
-        proxy_server = config.get("http", "proxy")
+        try:
+            proxy_server = config.get(b"http", b"proxy")
+        except KeyError:
+            proxy_server = None
     else:
         proxy_server = None
     handlers = []
@@ -1080,7 +1177,10 @@ def default_urllib2_opener(config):
         handlers.append(urllib2.ProxyHandler({"http": proxy_server}))
     opener = urllib2.build_opener(*handlers)
     if config is not None:
-        user_agent = config.get("http", "useragent")
+        try:
+            user_agent = config.get(b"http", b"useragent")
+        except KeyError:
+            user_agent = None
     else:
         user_agent = None
     if user_agent is None:
@@ -1136,7 +1236,15 @@ class HttpGitClient(GitClient):
             path = path.decode(sys.getfilesystemencoding())
         return urlparse.urljoin(self._base_url, path).rstrip("/") + "/"
 
-    def _http_request(self, url, headers={}, data=None):
+    def _http_request(self, url, headers={}, data=None,
+                      allow_compression=False):
+        if headers is None:
+            headers = dict(headers.items())
+        headers["Pragma"] = "no-cache"
+        if allow_compression:
+            headers["Accept-Encoding"] = "gzip"
+        else:
+            headers["Accept-Encoding"] = "identity"
         req = urllib2.Request(url, headers=headers, data=data)
         try:
             resp = self.opener.open(req)
@@ -1144,18 +1252,32 @@ class HttpGitClient(GitClient):
             if e.code == 404:
                 raise NotGitRepository()
             if e.code != 200:
-                raise GitProtocolError("unexpected http response %d" % e.code)
-        return resp
+                raise GitProtocolError("unexpected http response %d for %s" %
+                                       (e.code, url))
+        if resp.info().get('Content-Encoding') == 'gzip':
+            read = gzip.GzipFile(fileobj=BytesIO(resp.read())).read
+        else:
+            read = resp.read
 
-    def _discover_references(self, service, url):
-        assert url[-1] == "/"
-        url = urlparse.urljoin(url, "info/refs")
-        headers = {}
+        return resp, read
+
+    def _discover_references(self, service, base_url):
+        assert base_url[-1] == "/"
+        tail = "info/refs"
+        headers = {"Accept": "*/*"}
         if self.dumb is not False:
-            url += "?service=%s" % service.decode('ascii')
-            headers["Content-Type"] = "application/x-%s-request" % (
-                service.decode('ascii'))
-        resp = self._http_request(url, headers)
+            tail += "?service=%s" % service.decode('ascii')
+        url = urlparse.urljoin(base_url, tail)
+        resp, read = self._http_request(url, headers, allow_compression=True)
+
+        if url != resp.geturl():
+            # Something changed (redirect!), so let's update the base URL
+            if not resp.geturl().endswith(tail):
+                raise GitProtocolError(
+                        "Redirected from URL %s to URL %s without %s" % (
+                            url, resp.geturl(), tail))
+            base_url = resp.geturl()[:-len(tail)]
+
         try:
             content_type = resp.info().gettype()
         except AttributeError:
@@ -1163,7 +1285,7 @@ class HttpGitClient(GitClient):
         try:
             self.dumb = (not content_type.startswith("application/x-git-"))
             if not self.dumb:
-                proto = Protocol(resp.read, None)
+                proto = Protocol(read, None)
                 # The first line should mention the service
                 try:
                     [pkt] = list(proto.read_pkt_seq())
@@ -1173,9 +1295,9 @@ class HttpGitClient(GitClient):
                 if pkt.rstrip(b'\n') != (b'# service=' + service):
                     raise GitProtocolError(
                         "unexpected first line %r from smart server" % pkt)
-                return read_pkt_refs(proto)
+                return read_pkt_refs(proto) + (base_url, )
             else:
-                return read_info_refs(resp), set()
+                return read_info_refs(resp), set(), base_url
         finally:
             resp.close()
 
@@ -1185,7 +1307,7 @@ class HttpGitClient(GitClient):
         headers = {
             "Content-Type": "application/x-%s-request" % service
         }
-        resp = self._http_request(url, headers, data)
+        resp, read = self._http_request(url, headers, data)
         try:
             content_type = resp.info().gettype()
         except AttributeError:
@@ -1194,7 +1316,7 @@ class HttpGitClient(GitClient):
                 "application/x-%s-result" % service):
             raise GitProtocolError("Invalid content-type from server: %s"
                                    % content_type)
-        return resp
+        return resp, read
 
     def send_pack(self, path, update_refs, generate_pack_contents,
                   progress=None, write_pack=write_pack_objects):
@@ -1217,9 +1339,10 @@ class HttpGitClient(GitClient):
             {refname: new_ref}, including deleted refs.
         """
         url = self._get_url(path)
-        old_refs, server_capabilities = self._discover_references(
+        old_refs, server_capabilities, url = self._discover_references(
             b"git-receive-pack", url)
-        negotiated_capabilities = self._send_capabilities & server_capabilities
+        negotiated_capabilities = self._negotiate_receive_pack_capabilities(
+                server_capabilities)
 
         if CAPABILITY_REPORT_STATUS in negotiated_capabilities:
             self._report_status_parser = ReportStatusParser()
@@ -1239,8 +1362,8 @@ class HttpGitClient(GitClient):
         objects = generate_pack_contents(have, want)
         if len(objects) > 0:
             write_pack(req_proto.write_file(), objects)
-        resp = self._smart_request("git-receive-pack", url,
-                                   data=req_data.getvalue())
+        resp, read = self._smart_request("git-receive-pack", url,
+                                         data=req_data.getvalue())
         try:
             resp_proto = Protocol(resp.read, None)
             self._handle_receive_pack_tail(
@@ -1257,18 +1380,19 @@ class HttpGitClient(GitClient):
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
-        :return: Dictionary with all remote refs (not just those fetched)
+        :return: FetchPackResult object
         """
         url = self._get_url(path)
-        refs, server_capabilities = self._discover_references(
+        refs, server_capabilities, url = self._discover_references(
             b"git-upload-pack", url)
-        negotiated_capabilities = (
-            self._fetch_capabilities & server_capabilities)
+        negotiated_capabilities, symrefs, agent = (
+                self._negotiate_upload_pack_capabilities(
+                        server_capabilities))
         wants = determine_wants(refs)
         if wants is not None:
             wants = [cid for cid in wants if cid != ZERO_SHA]
         if not wants:
-            return refs
+            return FetchPackResult(refs, symrefs, agent)
         if self.dumb:
             raise NotImplementedError(self.send_pack)
         req_data = BytesIO()
@@ -1276,21 +1400,21 @@ class HttpGitClient(GitClient):
         self._handle_upload_pack_head(
                 req_proto, negotiated_capabilities, graph_walker, wants,
                 lambda: False)
-        resp = self._smart_request(
+        resp, read = self._smart_request(
             "git-upload-pack", url, data=req_data.getvalue())
         try:
-            resp_proto = Protocol(resp.read, None)
+            resp_proto = Protocol(read, None)
             self._handle_upload_pack_tail(
                 resp_proto, negotiated_capabilities, graph_walker, pack_data,
                 progress)
-            return refs
+            return FetchPackResult(refs, symrefs, agent)
         finally:
             resp.close()
 
     def get_refs(self, path):
         """Retrieve the current refs from a git smart server."""
         url = self._get_url(path)
-        refs, _ = self._discover_references(
+        refs, _, _ = self._discover_references(
             b"git-upload-pack", url)
         return refs
 

dulwich/config.py

@@ -387,6 +387,10 @@ class StackedConfig(Config):
     def __repr__(self):
         return "<%s for %r>" % (self.__class__.__name__, self.backends)
 
+    @classmethod
+    def default(cls):
+        return cls(cls.default_backends())
+
     @classmethod
     def default_backends(cls):
         """Retrieve the default configuration.

dulwich/object_store.py

@@ -293,15 +293,22 @@ class PackBasedObjectStore(BaseObjectStore):
         """Add a newly appeared pack to the cache by path.
 
         """
-        self._pack_cache[base_name] = pack
+        prev_pack = self._pack_cache.get(base_name)
+        if prev_pack is not pack:
+            self._pack_cache[base_name] = pack
+            if prev_pack:
+                prev_pack.close()
 
-    def close(self):
+    def _flush_pack_cache(self):
         pack_cache = self._pack_cache
         self._pack_cache = {}
         while pack_cache:
             (name, pack) = pack_cache.popitem()
             pack.close()
 
+    def close(self):
+        self._flush_pack_cache()
+
     @property
     def packs(self):
         """List with pack objects."""
@@ -326,6 +333,9 @@ class PackBasedObjectStore(BaseObjectStore):
     def _remove_loose_object(self, sha):
         raise NotImplementedError(self._remove_loose_object)
 
+    def _remove_pack(self, name):
+        raise NotImplementedError(self._remove_pack)
+
     def pack_loose_objects(self):
         """Pack loose objects.
 
@@ -339,6 +349,35 @@ class PackBasedObjectStore(BaseObjectStore):
             self._remove_loose_object(obj.id)
         return len(objects)
 
+    def repack(self):
+        """Repack the packs in this repository.
+
+        Note that this implementation is fairly naive and currently keeps all
+        objects in memory while it repacks.
+        """
+        loose_objects = set()
+        for sha in self._iter_loose_objects():
+            loose_objects.add(self._get_loose_object(sha))
+        objects = {(obj, None) for obj in loose_objects}
+        old_packs = {p.name(): p for p in self.packs}
+        for name, pack in old_packs.items():
+            objects.update((obj, None) for obj in pack.iterobjects())
+        self._flush_pack_cache()
+
+        # The name of the consolidated pack might match the name of a
+        # pre-existing pack. Take care not to remove the newly created
+        # consolidated pack.
+
+        consolidated = self.add_objects(objects)
+        old_packs.pop(consolidated.name(), None)
+
+        for obj in loose_objects:
+            self._remove_loose_object(obj.id)
+        for name, pack in old_packs.items():
+            self._remove_pack(pack)
+        self._update_pack_cache()
+        return len(objects)
+
     def __iter__(self):
         """Iterate over the SHAs that are present in this store."""
         iterables = (list(self.packs) + [self._iter_loose_objects()] +
@@ -532,6 +571,10 @@ class DiskObjectStore(PackBasedObjectStore):
     def _remove_loose_object(self, sha):
         os.remove(self._get_shafile_path(sha))
 
+    def _remove_pack(self, pack):
+        os.remove(pack.data.path)
+        os.remove(pack.index.path)
+
     def _get_pack_basepath(self, entries):
         suffix = iter_sha1(entry[0] for entry in entries)
         # TODO: Handle self.pack_dir being bytes
@@ -652,6 +695,7 @@ class DiskObjectStore(PackBasedObjectStore):
         f = os.fdopen(fd, 'wb')
 
         def commit():
+            f.flush()
             os.fsync(fd)
             f.close()
             if os.path.getsize(path) > 0:
@@ -1126,3 +1170,51 @@ class ObjectStoreGraphWalker(object):
         return None
 
     __next__ = next
+
+
+def commit_tree_changes(object_store, tree, changes):
+    """Commit a specified set of changes to a tree structure.
+
+    This will apply a set of changes on top of an existing tree, storing new
+    objects in object_store.
+
+    changes are a list of tuples with (path, mode, object_sha).
+    Paths can be both blobs and trees. See the mode and
+    object sha to None deletes the path.
+
+    This method works especially well if there are only a small
+    number of changes to a big tree. For a large number of changes
+    to a large tree, use e.g. commit_tree.
+
+    :param object_store: Object store to store new objects in
+        and retrieve old ones from.
+    :param tree: Original tree root
+    :param changes: changes to apply
+    :return: New tree root object
+    """
+    # TODO(jelmer): Save up the objects and add them using .add_objects
+    # rather than with individual calls to .add_object.
+    nested_changes = {}
+    for (path, new_mode, new_sha) in changes:
+        try:
+            (dirname, subpath) = path.split(b'/', 1)
+        except ValueError:
+            if new_sha is None:
+                del tree[path]
+            else:
+                tree[path] = (new_mode, new_sha)
+        else:
+            nested_changes.setdefault(dirname, []).append(
+                (subpath, new_mode, new_sha))
+    for name, subchanges in nested_changes.items():
+        try:
+            orig_subtree = object_store[tree[name][1]]
+        except KeyError:
+            orig_subtree = Tree()
+        subtree = commit_tree_changes(object_store, orig_subtree, subchanges)
+        if len(subtree) == 0:
+            del tree[name]
+        else:
+            tree[name] = (stat.S_IFDIR, subtree.id)
+    object_store.add_object(tree)
+    return tree

dulwich/objectspec.py

@@ -172,7 +172,15 @@ def parse_commit(repo, committish):
     :raise ValueError: If the range can not be parsed
     """
     committish = to_bytes(committish)
-    return repo[committish]  # For now..
+    try:
+        return repo[committish]
+    except KeyError:
+        pass
+    try:
+        return repo[parse_ref(repo, committish)]
+    except KeyError:
+        pass
+    raise KeyError(committish)
 
 
 # TODO: parse_path_in_tree(), which handles e.g. v1.0:Documentation

dulwich/pack.py

@@ -473,6 +473,10 @@ class FilePackIndex(PackIndex):
         else:
             self._contents, self._size = (contents, size)
 
+    @property
+    def path(self):
+        return self._filename
+
     def __eq__(self, other):
         # Quick optimization:
         if (isinstance(other, FilePackIndex) and
@@ -1009,6 +1013,10 @@ class PackData(object):
     def filename(self):
         return os.path.basename(self._filename)
 
+    @property
+    def path(self):
+        return self._filename
+
     @classmethod
     def from_file(cls, file, size):
         return cls(str(file), file=file, size=size)

dulwich/patch.py

@@ -86,31 +86,63 @@ def get_summary(commit):
     return commit.message.splitlines()[0].replace(" ", "-")
 
 
-def unified_diff(a, b, fromfile, tofile, n=3):
-    """difflib.unified_diff that doesn't write any dates or trailing spaces.
-
-    Based on the same function in Python2.6.5-rc2's difflib.py
+#  Unified Diff
+def _format_range_unified(start, stop):
+    'Convert range to the "ed" format'
+    # Per the diff spec at http://www.unix.org/single_unix_specification/
+    beginning = start + 1  # lines start numbering with one
+    length = stop - start
+    if length == 1:
+        return '{}'.format(beginning)
+    if not length:
+        beginning -= 1  # empty ranges begin at line just before the range
+    return '{},{}'.format(beginning, length)
+
+
+def unified_diff(a, b, fromfile='', tofile='', fromfiledate='',
+                 tofiledate='', n=3, lineterm='\n'):
+    """difflib.unified_diff that can detect "No newline at end of file" as
+    original "git diff" does.
+
+    Based on the same function in Python2.7 difflib.py
     """
     started = False
     for group in SequenceMatcher(None, a, b).get_grouped_opcodes(n):
         if not started:
-            yield b'--- ' + fromfile + b'\n'
-            yield b'+++ ' + tofile + b'\n'
             started = True
-        i1, i2, j1, j2 = group[0][1], group[-1][2], group[0][3], group[-1][4]
-        sizes = "@@ -%d,%d +%d,%d @@\n" % (i1+1, i2-i1, j1+1, j2-j1)
-        yield sizes.encode('ascii')
+            fromdate = '\t{}'.format(fromfiledate) if fromfiledate else ''
+            todate = '\t{}'.format(tofiledate) if tofiledate else ''
+            yield '--- {}{}{}'.format(
+                fromfile.decode("ascii"),
+                fromdate,
+                lineterm
+                ).encode('ascii')
+            yield '+++ {}{}{}'.format(
+                tofile.decode("ascii"),
+                todate,
+                lineterm
+                ).encode('ascii')
+
+        first, last = group[0], group[-1]
+        file1_range = _format_range_unified(first[1], last[2])
+        file2_range = _format_range_unified(first[3], last[4])
+        yield '@@ -{} +{} @@{}'.format(
+            file1_range,
+            file2_range,
+            lineterm
+             ).encode('ascii')
+
         for tag, i1, i2, j1, j2 in group:
             if tag == 'equal':
                 for line in a[i1:i2]:
                     yield b' ' + line
                 continue
-            if tag == 'replace' or tag == 'delete':
+            if tag in ('replace', 'delete'):
                 for line in a[i1:i2]:
                     if not line[-1:] == b'\n':
                         line += b'\n\\ No newline at end of file\n'
                     yield b'-' + line
-            if tag == 'replace' or tag == 'insert':
+            if tag in ('replace', 'insert'):
                 for line in b[j1:j2]:
                     if not line[-1:] == b'\n':
                         line += b'\n\\ No newline at end of file\n'

dulwich/porcelain.py

@@ -25,6 +25,7 @@ Currently implemented:
  * add
  * branch{_create,_delete,_list}
  * check-ignore
+ * checkout
  * clone
  * commit
  * commit-tree
@@ -69,6 +70,9 @@ from dulwich.archive import (
 from dulwich.client import (
     get_transport_and_path,
     )
+from dulwich.config import (
+    StackedConfig,
+    )
 from dulwich.diff_tree import (
     CHANGE_ADD,
     CHANGE_DELETE,
@@ -97,7 +101,9 @@ from dulwich.objects import (
     pretty_format_tree_entry,
     )
 from dulwich.objectspec import (
+    parse_commit,
     parse_object,
+    parse_ref,
     parse_reftuples,
     parse_tree,
     )
@@ -283,7 +289,9 @@ def clone(source, target=None, bare=False, checkout=None,
         checkout = (not bare)
     if checkout and bare:
         raise ValueError("checkout and bare are incompatible")
-    client, host_path = get_transport_and_path(source)
+
+    config = StackedConfig.default()
+    client, host_path = get_transport_and_path(source, config=config)
 
     if target is None:
         target = host_path.split("/")[-1]
@@ -742,7 +750,8 @@ def push(repo, remote_location, refspecs,
     with open_repo_closing(repo) as r:
 
         # Get the client and path
-        client, path = get_transport_and_path(remote_location)
+        client, path = get_transport_and_path(
+                remote_location, config=r.get_config_stack())
 
         selected_refs = []
 
@@ -796,7 +805,8 @@ def pull(repo, remote_location=None, refspecs=None,
             selected_refs.extend(
                 parse_reftuples(remote_refs, r.refs, refspecs))
             return [remote_refs[lh] for (lh, rh, force) in selected_refs]
-        client, path = get_transport_and_path(remote_location)
+        client, path = get_transport_and_path(
+                remote_location, config=r.get_config_stack())
         remote_refs = client.fetch(
             path, r, progress=errstream.write, determine_wants=determine_wants)
         for (lh, rh, force) in selected_refs:
@@ -1032,7 +1042,8 @@ def fetch(repo, remote_location, outstream=sys.stdout,
     :return: Dictionary with refs on the remote
     """
     with open_repo_closing(repo) as r:
-        client, path = get_transport_and_path(remote_location)
+        client, path = get_transport_and_path(
+                remote_location, config=r.get_config_stack())
         remote_refs = client.fetch(path, r, progress=errstream.write)
     return remote_refs
 
@@ -1043,7 +1054,8 @@ def ls_remote(remote):
     :param remote: Remote repository location
     :return: Dictionary with remote refs
     """
-    client, host_path = get_transport_and_path(remote)
+    config = StackedConfig.default()
+    client, host_path = get_transport_and_path(remote, config=config)
     return client.get_refs(host_path)
 
 
@@ -1139,3 +1151,30 @@ def check_ignore(repo, paths, no_index=False):
                 continue
             if ignore_manager.is_ignored(path):
                 yield path
+
+
+def update_head(repo, target, detached=False, new_branch=None):
+    """Update HEAD to point at a new branch/commit.
+
+    Note that this does not actually update the working tree.
+
+    :param repo: Path to the repository
+    :param detach: Create a detached head
+    :param target: Branch or committish to switch to
+    :param new_branch: New branch to create
+    """
+    with open_repo_closing(repo) as r:
+        if new_branch is not None:
+            to_set = b"refs/heads/" + new_branch.encode(DEFAULT_ENCODING)
+        else:
+            to_set = b"HEAD"
+        if detached:
+            # TODO(jelmer): Provide some way so that the actual ref gets
+            # updated rather than what it points to, so the delete isn't
+            # necessary.
+            del r.refs[to_set]
+            r.refs[to_set] = parse_commit(r, target).id
+        else:
+            r.refs.set_symbolic_ref(to_set, parse_ref(r, target))
+        if new_branch is not None:
+            r.refs.set_symbolic_ref(b"HEAD", to_set)

dulwich/protocol.py

@@ -48,6 +48,9 @@ SIDE_BAND_CHANNEL_PROGRESS = 2
 # fatal error message just before stream aborts
 SIDE_BAND_CHANNEL_FATAL = 3
 
+CAPABILITY_DEEPEN_SINCE = b'deepen-since'
+CAPABILITY_DEEPEN_NOT = b'deepen-not'
+CAPABILITY_DEEPEN_RELATIVE = b'deepen-relative'
 CAPABILITY_DELETE_REFS = b'delete-refs'
 CAPABILITY_INCLUDE_TAG = b'include-tag'
 CAPABILITY_MULTI_ACK = b'multi_ack'
@@ -58,14 +61,36 @@ CAPABILITY_OFS_DELTA = b'ofs-delta'
 CAPABILITY_QUIET = b'quiet'
 CAPABILITY_REPORT_STATUS = b'report-status'
 CAPABILITY_SHALLOW = b'shallow'
+CAPABILITY_SIDE_BAND = b'side-band'
 CAPABILITY_SIDE_BAND_64K = b'side-band-64k'
 CAPABILITY_THIN_PACK = b'thin-pack'
 CAPABILITY_AGENT = b'agent'
+CAPABILITY_SYMREF = b'symref'
 
 # Magic ref that is used to attach capabilities to when
 # there are no refs. Should always be ste to ZERO_SHA.
 CAPABILITIES_REF = b'capabilities^{}'
 
+COMMON_CAPABILITIES = [
+    CAPABILITY_OFS_DELTA,
+    CAPABILITY_SIDE_BAND,
+    CAPABILITY_SIDE_BAND_64K,
+    CAPABILITY_AGENT,
+    CAPABILITY_NO_PROGRESS]
+KNOWN_UPLOAD_CAPABILITIES = set(COMMON_CAPABILITIES + [
+    CAPABILITY_THIN_PACK,
+    CAPABILITY_MULTI_ACK,
+    CAPABILITY_MULTI_ACK_DETAILED,
+    CAPABILITY_INCLUDE_TAG,
+    CAPABILITY_DEEPEN_SINCE,
+    CAPABILITY_SYMREF,
+    CAPABILITY_SHALLOW,
+    CAPABILITY_DEEPEN_NOT,
+    CAPABILITY_DEEPEN_RELATIVE,
+    ])
+KNOWN_RECEIVE_CAPABILITIES = set(COMMON_CAPABILITIES + [
+    CAPABILITY_REPORT_STATUS])
+
 
 def agent_string():
     return ('dulwich/%d.%d.%d' % dulwich.__version__).encode('ascii')
@@ -75,6 +100,25 @@ def capability_agent():
     return CAPABILITY_AGENT + b'=' + agent_string()
 
 
+def capability_symref(from_ref, to_ref):
+    return CAPABILITY_SYMREF + b'=' + from_ref + b':' + to_ref
+
+
+def extract_capability_names(capabilities):
+    return set(parse_capability(c)[0] for c in capabilities)
+
+
+def parse_capability(capability):
+    parts = capability.split(b'=', 1)
+    if len(parts) == 1:
+        return (parts[0], None)
+    return tuple(parts)
+
+
+def symref_capabilities(symrefs):
+    return [capability_symref(*k) for k in symrefs]
+
+
 COMMAND_DEEPEN = b'deepen'
 COMMAND_SHALLOW = b'shallow'
 COMMAND_UNSHALLOW = b'unshallow'

dulwich/refs.py

@@ -47,6 +47,17 @@ BAD_REF_CHARS = set(b'\177 ~^:?*[')
 ANNOTATED_TAG_SUFFIX = b'^{}'
 
 
+def parse_symref_value(contents):
+    """Parse a symref value.
+
+    :param contents: Contents to parse
+    :return: Destination
+    """
+    if contents.startswith(SYMREF):
+        return contents[len(SYMREF):].rstrip(b'\r\n')
+    raise ValueError(contents)
+
+
 def check_ref_format(refname):
     """Check if a refname is correctly formatted.
 
@@ -306,6 +317,21 @@ class RefsContainer(object):
         """
         self.remove_if_equals(name, None)
 
+    def get_symrefs(self):
+        """Get a dict with all symrefs in this container.
+
+        :return: Dictionary mapping source ref to target ref
+        """
+        ret = {}
+        for src in self.allkeys():
+            try:
+                dst = parse_symref_value(self.read_ref(src))
+            except ValueError:
+                pass
+            else:
+                ret[src] = dst
+        return ret
+
 
 class DictRefsContainer(RefsContainer):
     """RefsContainer backed by a simple dict.
@@ -538,7 +564,7 @@ class DiskRefsContainer(RefsContainer):
                     # Read only the first 40 bytes
                     return header + f.read(40 - len(SYMREF))
         except IOError as e:
-            if e.errno == errno.ENOENT:
+            if e.errno in (errno.ENOENT, errno.EISDIR):
                 return None
             raise
 

dulwich/server.py

@@ -39,6 +39,7 @@ Currently supported capabilities:
  * report-status
  * delete-refs
  * shallow
+ * symref
 """
 
 import collections
@@ -104,6 +105,7 @@ from dulwich.protocol import (  # noqa: F401
     ack_type,
     extract_capabilities,
     extract_want_line_capabilities,
+    symref_capabilities,
     )
 from dulwich.refs import (
     ANNOTATED_TAG_SUFFIX,
@@ -229,8 +231,9 @@ class PackHandler(Handler):
         self._done_received = False
 
     @classmethod
-    def capability_line(cls):
-        return b"".join([b" " + c for c in cls.capabilities()])
+    def capability_line(cls, capabilities):
+        logger.info('Sending capabilities: %s', capabilities)
+        return b"".join([b" " + c for c in capabilities])
 
     @classmethod
     def capabilities(cls):
@@ -238,9 +241,9 @@ class PackHandler(Handler):
 
     @classmethod
     def innocuous_capabilities(cls):
-        return (CAPABILITY_INCLUDE_TAG, CAPABILITY_THIN_PACK,
+        return [CAPABILITY_INCLUDE_TAG, CAPABILITY_THIN_PACK,
                 CAPABILITY_NO_PROGRESS, CAPABILITY_OFS_DELTA,
-                capability_agent())
+                capability_agent()]
 
     @classmethod
     def required_capabilities(cls):
@@ -288,10 +291,10 @@ class UploadPackHandler(PackHandler):
 
     @classmethod
     def capabilities(cls):
-        return (CAPABILITY_MULTI_ACK_DETAILED, CAPABILITY_MULTI_ACK,
+        return [CAPABILITY_MULTI_ACK_DETAILED, CAPABILITY_MULTI_ACK,
                 CAPABILITY_SIDE_BAND_64K, CAPABILITY_THIN_PACK,
                 CAPABILITY_OFS_DELTA, CAPABILITY_NO_PROGRESS,
-                CAPABILITY_INCLUDE_TAG, CAPABILITY_SHALLOW, CAPABILITY_NO_DONE)
+                CAPABILITY_INCLUDE_TAG, CAPABILITY_SHALLOW, CAPABILITY_NO_DONE]
 
     @classmethod
     def required_capabilities(cls):
@@ -337,8 +340,9 @@ class UploadPackHandler(PackHandler):
         def write(x):
             return self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, x)
 
-        graph_walker = ProtocolGraphWalker(
-                self, self.repo.object_store, self.repo.get_peeled)
+        graph_walker = _ProtocolGraphWalker(
+                self, self.repo.object_store, self.repo.get_peeled,
+                self.repo.refs.get_symrefs)
         objects_iter = self.repo.fetch_objects(
             graph_walker.determine_wants, graph_walker, self.progress,
             get_tagged=self.get_tagged)
@@ -496,7 +500,7 @@ def _all_wants_satisfied(store, haves, wants):
     return True
 
 
-class ProtocolGraphWalker(object):
+class _ProtocolGraphWalker(object):
     """A graph walker that knows the git protocol.
 
     As a graph walker, this class implements ack(), next(), and reset(). It
@@ -509,10 +513,11 @@ class ProtocolGraphWalker(object):
     call to set_ack_type() is required to set up the implementation, before
     any calls to next() or ack() are made.
     """
-    def __init__(self, handler, object_store, get_peeled):
+    def __init__(self, handler, object_store, get_peeled, get_symrefs):
         self.handler = handler
         self.store = object_store
         self.get_peeled = get_peeled
+        self.get_symrefs = get_symrefs
         self.proto = handler.proto
         self.http_req = handler.http_req
         self.advertise_refs = handler.advertise_refs
@@ -542,12 +547,16 @@ class ProtocolGraphWalker(object):
         :param heads: a dict of refname->SHA1 to advertise
         :return: a list of SHA1s requested by the client
         """
+        symrefs = self.get_symrefs()
         values = set(heads.values())
         if self.advertise_refs or not self.http_req:
             for i, (ref, sha) in enumerate(sorted(heads.items())):
                 line = sha + b' ' + ref
                 if not i:
-                    line += b'\x00' + self.handler.capability_line()
+                    line += (b'\x00' +
+                             self.handler.capability_line(
+                                 self.handler.capabilities() +
+                                 symref_capabilities(symrefs.items())))
                 self.proto.write_pkt_line(line + b'\n')
                 peeled_sha = self.get_peeled(ref)
                 if peeled_sha != sha:
@@ -872,9 +881,9 @@ class ReceivePackHandler(PackHandler):
 
     @classmethod
     def capabilities(cls):
-        return (CAPABILITY_REPORT_STATUS, CAPABILITY_DELETE_REFS,
+        return [CAPABILITY_REPORT_STATUS, CAPABILITY_DELETE_REFS,
                 CAPABILITY_QUIET, CAPABILITY_OFS_DELTA,
-                CAPABILITY_SIDE_BAND_64K, CAPABILITY_NO_DONE)
+                CAPABILITY_SIDE_BAND_64K, CAPABILITY_NO_DONE]
 
     def _apply_pack(self, refs):
         all_exceptions = (IOError, OSError, ChecksumMismatch, ApplyDeltaError,
@@ -954,12 +963,14 @@ class ReceivePackHandler(PackHandler):
     def handle(self):
         if self.advertise_refs or not self.http_req:
             refs = sorted(self.repo.get_refs().items())
+            symrefs = sorted(self.repo.refs.get_symrefs().items())
 
             if not refs:
                 refs = [(CAPABILITIES_REF, ZERO_SHA)]
             self.proto.write_pkt_line(
               refs[0][1] + b' ' + refs[0][0] + b'\0' +
-              self.capability_line() + b'\n')
+              self.capability_line(
+                  self.capabilities() + symref_capabilities(symrefs)) + b'\n')
             for i in range(1, len(refs)):
                 ref = refs[i]
                 self.proto.write_pkt_line(ref[1] + b' ' + ref[0] + b'\n')

dulwich/tests/compat/server_utils.py

@@ -28,6 +28,9 @@ import tempfile
 
 from dulwich.repo import Repo
 from dulwich.objects import hex_to_sha
+from dulwich.protocol import (
+    CAPABILITY_SIDE_BAND_64K,
+    )
 from dulwich.server import (
     ReceivePackHandler,
     )
@@ -300,8 +303,8 @@ class NoSideBand64kReceivePackHandler(ReceivePackHandler):
 
     @classmethod
     def capabilities(cls):
-        return tuple(c for c in ReceivePackHandler.capabilities()
-                     if c != b'side-band-64k')
+        return [c for c in ReceivePackHandler.capabilities()
+                if c != CAPABILITY_SIDE_BAND_64K]
 
 
 def ignore_error(error):

dulwich/tests/compat/test_client.py

@@ -205,8 +205,8 @@ class DulwichClientTestBase(object):
     def test_fetch_pack(self):
         c = self._client()
         with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
-            refs = c.fetch(self._build_path('/server_new.export'), dest)
-            for r in refs.items():
+            result = c.fetch(self._build_path('/server_new.export'), dest)
+            for r in result.refs.items():
                 dest.refs.set_if_equals(r[0], None, r[1])
             self.assertDestEqualsSrc()
 
@@ -217,8 +217,8 @@ class DulwichClientTestBase(object):
         c = self._client()
         repo_dir = os.path.join(self.gitroot, 'server_new.export')
         with repo.Repo(repo_dir) as dest:
-            refs = c.fetch(self._build_path('/dest'), dest)
-            for r in refs.items():
+            result = c.fetch(self._build_path('/dest'), dest)
+            for r in result.refs.items():
                 dest.refs.set_if_equals(r[0], None, r[1])
             self.assertDestEqualsSrc()
 
@@ -226,8 +226,8 @@ class DulwichClientTestBase(object):
         c = self._client()
         c._fetch_capabilities.remove(b'side-band-64k')
         with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
-            refs = c.fetch(self._build_path('/server_new.export'), dest)
-            for r in refs.items():
+            result = c.fetch(self._build_path('/server_new.export'), dest)
+            for r in result.refs.items():
                 dest.refs.set_if_equals(r[0], None, r[1])
             self.assertDestEqualsSrc()
 
@@ -236,10 +236,10 @@ class DulwichClientTestBase(object):
         # be ignored
         c = self._client()
         with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
-            refs = c.fetch(
+            result = c.fetch(
                 self._build_path('/server_new.export'), dest,
                 lambda refs: [protocol.ZERO_SHA])
-            for r in refs.items():
+            for r in result.refs.items():
                 dest.refs.set_if_equals(r[0], None, r[1])
 
     def test_send_remove_branch(self):
@@ -442,7 +442,9 @@ class GitHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
                         if len(authorization) == 2:
                             env['REMOTE_USER'] = authorization[0]
         # XXX REMOTE_IDENT
-        env['CONTENT_TYPE'] = self.headers.get('content-type')
+        content_type = self.headers.get('content-type')
+        if content_type:
+            env['CONTENT_TYPE'] = content_type
         length = self.headers.get('content-length')
         if length:
             env['CONTENT_LENGTH'] = length

dulwich/tests/compat/test_web.py

@@ -111,8 +111,8 @@ def patch_capabilities(handler, caps_removed):
     # Patch a handler's capabilities by specifying a list of them to be
     # removed, and return the original classmethod for restoration.
     original_capabilities = handler.capabilities
-    filtered_capabilities = tuple(
-        i for i in original_capabilities() if i not in caps_removed)
+    filtered_capabilities = [
+        i for i in original_capabilities() if i not in caps_removed]
 
     def capabilities(cls):
         return filtered_capabilities

dulwich/tests/test_client.py

@@ -23,6 +23,11 @@ import sys
 import shutil
 import tempfile
 
+try:
+    import urllib2
+except ImportError:
+    import urllib.request as urllib2
+
 try:
     from urllib import quote as urlquote
 except ImportError:
@@ -46,9 +51,13 @@ from dulwich.client import (
     ReportStatusParser,
     SendPackError,
     UpdateRefsError,
+    default_urllib2_opener,
     get_transport_and_path,
     get_transport_and_path_from_url,
     )
+from dulwich.config import (
+    ConfigDict,
+    )
 from dulwich.tests import (
     TestCase,
     )
@@ -123,7 +132,8 @@ class GitClientTests(TestCase):
             self.assertIs(heads, None)
             return []
         ret = self.client.fetch_pack(b'/', check_heads, None, None)
-        self.assertIs(None, ret)
+        self.assertIs(None, ret.refs)
+        self.assertEqual({}, ret.symrefs)
 
     def test_fetch_pack_ignores_magic_ref(self):
         self.rin.write(
@@ -138,17 +148,23 @@ class GitClientTests(TestCase):
             self.assertEquals({}, heads)
             return []
         ret = self.client.fetch_pack(b'bla', check_heads, None, None, None)
-        self.assertIs(None, ret)
+        self.assertIs(None, ret.refs)
+        self.assertEqual({}, ret.symrefs)
         self.assertEqual(self.rout.getvalue(), b'0000')
 
     def test_fetch_pack_none(self):
         self.rin.write(
-            b'008855dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7 HEAD.multi_ack '
+            b'008855dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7 HEAD\x00multi_ack '
             b'thin-pack side-band side-band-64k ofs-delta shallow no-progress '
             b'include-tag\n'
             b'0000')
         self.rin.seek(0)
-        self.client.fetch_pack(b'bla', lambda heads: [], None, None, None)
+        ret = self.client.fetch_pack(
+                b'bla', lambda heads: [], None, None, None)
+        self.assertEqual(
+                {b'HEAD': b'55dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7'},
+                ret.refs)
+        self.assertEqual({}, ret.symrefs)
         self.assertEqual(self.rout.getvalue(), b'0000')
 
     def test_send_pack_no_sideband64k_with_update_ref_error(self):
@@ -745,7 +761,10 @@ class LocalGitClientTests(TestCase):
             b'refs/tags/mytag': b'28237f4dc30d0d462658d6b937b08a0f0b6ef55a',
             b'refs/tags/mytag-packed':
                 b'b0931cadc54336e78a1d980420e3268903b57a50'
-            }, ret)
+            }, ret.refs)
+        self.assertEqual(
+                {b'HEAD': b'refs/heads/master'},
+                ret.symrefs)
         self.assertEqual(
                 b"PACK\x00\x00\x00\x02\x00\x00\x00\x00\x02\x9d\x08"
                 b"\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e",
@@ -757,10 +776,18 @@ class LocalGitClientTests(TestCase):
         self.addCleanup(tear_down_repo, s)
         out = BytesIO()
         walker = MemoryRepo().get_graph_walker()
-        c.fetch_pack(
+        ret = c.fetch_pack(
             s.path,
             lambda heads: [b"a90fa2d900a17e99b433217e988c4eb4a2e9a097"],
             graph_walker=walker, pack_data=out.write)
+        self.assertEqual({b'HEAD': b'refs/heads/master'}, ret.symrefs)
+        self.assertEqual({
+            b'HEAD': b'a90fa2d900a17e99b433217e988c4eb4a2e9a097',
+            b'refs/heads/master': b'a90fa2d900a17e99b433217e988c4eb4a2e9a097',
+            b'refs/tags/mytag': b'28237f4dc30d0d462658d6b937b08a0f0b6ef55a',
+            b'refs/tags/mytag-packed':
+            b'b0931cadc54336e78a1d980420e3268903b57a50'
+            }, ret.refs)
         # Hardcoding is not ideal, but we'll fix that some other day..
         self.assertTrue(out.getvalue().startswith(
                 b'PACK\x00\x00\x00\x02\x00\x00\x00\x07'))
@@ -899,3 +926,19 @@ class TCPGitClientTests(TestCase):
 
         url = c.get_url(path)
         self.assertEqual('git://github.com:9090/jelmer/dulwich', url)
+
+
+class DefaultUrllib2OpenerTest(TestCase):
+
+    def test_no_config(self):
+        default_urllib2_opener(config=None)
+
+    def test_config_no_proxy(self):
+        default_urllib2_opener(config=ConfigDict())
+
+    def test_config_proxy(self):
+        config = ConfigDict()
+        config.set(b'http', b'proxy', b'http://localhost:3128/')
+        opener = default_urllib2_opener(config=config)
+        self.assertIn(urllib2.ProxyHandler,
+                      list(map(lambda x: x.__class__, opener.handlers)))

dulwich/tests/test_object_store.py

@@ -25,6 +25,7 @@ from contextlib import closing
 from io import BytesIO
 import os
 import shutil
+import stat
 import tempfile
 
 from dulwich.index import (
@@ -43,6 +44,7 @@ from dulwich.object_store import (
     DiskObjectStore,
     MemoryObjectStore,
     ObjectStoreGraphWalker,
+    commit_tree_changes,
     tree_lookup_path,
     )
 from dulwich.pack import (
@@ -270,11 +272,52 @@ class PackBasedObjectStoreTests(ObjectStoreTests):
         self.store.add_object(b1)
         b2 = make_object(Blob, data=b"more yummy data")
         self.store.add_object(b2)
-        self.assertEqual([], list(self.store.packs))
+        b3 = make_object(Blob, data=b"even more yummy data")
+        b4 = make_object(Blob, data=b"and more yummy data")
+        self.store.add_objects([(b3, None), (b4, None)])
+        self.assertEqual({b1.id, b2.id, b3.id, b4.id}, set(self.store))
+        self.assertEqual(1, len(self.store.packs))
         self.assertEqual(2, self.store.pack_loose_objects())
         self.assertNotEqual([], list(self.store.packs))
         self.assertEqual(0, self.store.pack_loose_objects())
 
+    def test_repack(self):
+        b1 = make_object(Blob, data=b"yummy data")
+        self.store.add_object(b1)
+        b2 = make_object(Blob, data=b"more yummy data")
+        self.store.add_object(b2)
+        b3 = make_object(Blob, data=b"even more yummy data")
+        b4 = make_object(Blob, data=b"and more yummy data")
+        self.store.add_objects([(b3, None), (b4, None)])
+        b5 = make_object(Blob, data=b"and more data")
+        b6 = make_object(Blob, data=b"and some more data")
+        self.store.add_objects([(b5, None), (b6, None)])
+        self.assertEqual({b1.id, b2.id, b3.id, b4.id, b5.id, b6.id},
+                         set(self.store))
+        self.assertEqual(2, len(self.store.packs))
+        self.assertEqual(6, self.store.repack())
+        self.assertEqual(1, len(self.store.packs))
+        self.assertEqual(0, self.store.pack_loose_objects())
+
+    def test_repack_existing(self):
+        b1 = make_object(Blob, data=b"yummy data")
+        self.store.add_object(b1)
+        b2 = make_object(Blob, data=b"more yummy data")
+        self.store.add_object(b2)
+        self.store.add_objects([(b1, None), (b2, None)])
+        self.store.add_objects([(b2, None)])
+        self.assertEqual({b1.id, b2.id}, set(self.store))
+        self.assertEqual(2, len(self.store.packs))
+        self.assertEqual(2, self.store.repack())
+        self.assertEqual(1, len(self.store.packs))
+        self.assertEqual(0, self.store.pack_loose_objects())
+
+        self.assertEqual({b1.id, b2.id}, set(self.store))
+        self.assertEqual(1, len(self.store.packs))
+        self.assertEqual(2, self.store.repack())
+        self.assertEqual(1, len(self.store.packs))
+        self.assertEqual(0, self.store.pack_loose_objects())
+
 
 class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
 
@@ -493,3 +536,75 @@ class ObjectStoreGraphWalkerTests(TestCase):
                          sorted(walk))
         self.assertLess(walk.index(b"a" * 40), walk.index(b"c" * 40))
         self.assertLess(walk.index(b"b" * 40), walk.index(b"d" * 40))
+
+
+class CommitTreeChangesTests(TestCase):
+
+    def setUp(self):
+        super(CommitTreeChangesTests, self).setUp()
+        self.store = MemoryObjectStore()
+        self.blob_a = make_object(Blob, data=b'a')
+        self.blob_b = make_object(Blob, data=b'b')
+        self.blob_c = make_object(Blob, data=b'c')
+        for blob in [self.blob_a, self.blob_b, self.blob_c]:
+            self.store.add_object(blob)
+
+        blobs = [
+          (b'a', self.blob_a.id, 0o100644),
+          (b'ad/b', self.blob_b.id, 0o100644),
+          (b'ad/bd/c', self.blob_c.id, 0o100755),
+          (b'ad/c', self.blob_c.id, 0o100644),
+          (b'c', self.blob_c.id, 0o100644),
+          ]
+        self.tree_id = commit_tree(self.store, blobs)
+
+    def test_no_changes(self):
+        self.assertEqual(
+                self.store[self.tree_id],
+                commit_tree_changes(self.store, self.store[self.tree_id], []))
+
+    def test_add_blob(self):
+        blob_d = make_object(Blob, data=b'd')
+        new_tree = commit_tree_changes(
+                self.store, self.store[self.tree_id], [
+                    (b'd', 0o100644, blob_d.id)])
+        self.assertEqual(
+            new_tree[b'd'],
+            (33188, b'c59d9b6344f1af00e504ba698129f07a34bbed8d'))
+
+    def test_add_blob_in_dir(self):
+        blob_d = make_object(Blob, data=b'd')
+        new_tree = commit_tree_changes(
+                self.store, self.store[self.tree_id], [
+                    (b'e/f/d', 0o100644, blob_d.id)])
+        self.assertEqual(
+            new_tree.items(), [
+                TreeEntry(path=b'a', mode=stat.S_IFREG | 0o100644,
+                          sha=self.blob_a.id),
+                TreeEntry(path=b'ad', mode=stat.S_IFDIR,
+                          sha=b'0e2ce2cd7725ff4817791be31ccd6e627e801f4a'),
+                TreeEntry(path=b'c', mode=stat.S_IFREG | 0o100644,
+                          sha=self.blob_c.id),
+                TreeEntry(path=b'e', mode=stat.S_IFDIR,
+                          sha=b'6ab344e288724ac2fb38704728b8896e367ed108')
+                ])
+        e_tree = self.store[new_tree[b'e'][1]]
+        self.assertEqual(
+            e_tree.items(), [
+                TreeEntry(path=b'f', mode=stat.S_IFDIR,
+                          sha=b'24d2c94d8af232b15a0978c006bf61ef4479a0a5')
+                ])
+        f_tree = self.store[e_tree[b'f'][1]]
+        self.assertEqual(
+            f_tree.items(), [
+                TreeEntry(path=b'd', mode=stat.S_IFREG | 0o100644,
+                          sha=blob_d.id)
+                ])
+
+    def test_delete_blob(self):
+        new_tree = commit_tree_changes(
+                self.store, self.store[self.tree_id], [
+                    (b'ad/bd/c', None, None)])
+        self.assertEqual(set(new_tree), {b'a', b'ad', b'c'})
+        ad_tree = self.store[new_tree[b'ad'][1]]
+        self.assertEqual(set(ad_tree), {b'b', b'c'})

dulwich/tests/test_patch.py

@@ -281,7 +281,7 @@ class DiffTests(TestCase):
              b'index 0000000..a116b51 644',
              b'--- /dev/null',
              b'+++ b/bar.txt',
-             b'@@ -1,0 +1,2 @@',
+             b'@@ -0,0 +1,2 @@',
              b'+new',
              b'+same'
             ], f.getvalue().splitlines())
@@ -297,7 +297,7 @@ class DiffTests(TestCase):
             b'index a116b51..0000000',
             b'--- a/bar.txt',
             b'+++ /dev/null',
-            b'@@ -1,2 +1,0 @@',
+            b'@@ -1,2 +0,0 @@',
             b'-new',
             b'-same'
             ], f.getvalue().splitlines())
@@ -327,7 +327,7 @@ class DiffTests(TestCase):
             b'index 0000000..76d4bb8 644',
             b'--- /dev/null',
             b'+++ b/added.txt',
-            b'@@ -1,0 +1,1 @@',
+            b'@@ -0,0 +1 @@',
             b'+add',
             b'diff --git a/changed.txt b/changed.txt',
             b'index bf84e48..1be2436 644',
@@ -342,7 +342,7 @@ class DiffTests(TestCase):
             b'index 2c3f0b3..0000000',
             b'--- a/removed.txt',
             b'+++ /dev/null',
-            b'@@ -1,1 +1,0 @@',
+            b'@@ -1 +0,0 @@',
             b'-removed',
             ], f.getvalue().splitlines())
 
@@ -362,7 +362,7 @@ class DiffTests(TestCase):
             b'index 06d0bdd..cc97564 160000',
             b'--- a/asubmodule',
             b'+++ b/asubmodule',
-            b'@@ -1,1 +1,1 @@',
+            b'@@ -1 +1 @@',
             b'-Submodule commit 06d0bdd9e2e20377b3180e4986b14c8549b393e4',
             b'+Submodule commit cc975646af69f279396d4d5e1379ac6af80ee637',
             ], f.getvalue().splitlines())
@@ -399,7 +399,7 @@ class DiffTests(TestCase):
              b'index 0000000..a116b51 644',
              b'--- /dev/null',
              b'+++ b/bar.txt',
-             b'@@ -1,0 +1,2 @@',
+             b'@@ -0,0 +1,2 @@',
              b'+new',
              b'+same'
             ], f.getvalue().splitlines())
@@ -417,7 +417,7 @@ class DiffTests(TestCase):
             b'index a116b51..0000000',
             b'--- a/bar.txt',
             b'+++ /dev/null',
-            b'@@ -1,2 +1,0 @@',
+            b'@@ -1,2 +0,0 @@',
             b'-new',
             b'-same'
             ], f.getvalue().splitlines())
@@ -532,7 +532,7 @@ class DiffTests(TestCase):
             b'index a116b51..06d0bdd 160000',
             b'--- a/bar.txt',
             b'+++ b/bar.txt',
-            b'@@ -1,2 +1,1 @@',
+            b'@@ -1,2 +1 @@',
             b'-new',
             b'-same',
             b'+Submodule commit 06d0bdd9e2e20377b3180e4986b14c8549b393e4',

dulwich/tests/test_porcelain.py

@@ -399,7 +399,7 @@ new mode 100644
 index 0000000..ea5c7bf 100644
 --- /dev/null
 +++ b/somename
-@@ -1,0 +1,1 @@
+@@ -0,0 +1 @@
 +The Foo
 """)
 
@@ -430,7 +430,7 @@ diff --git a/somename b/somename
 index ea5c7bf..fd38bcb 100644
 --- a/somename
 +++ b/somename
-@@ -1,1 +1,1 @@
+@@ -1 +1 @@
 -The Foo
 +The Bar
 """)
@@ -950,8 +950,9 @@ class ReceivePackTests(PorcelainTestCase):
                 self.repo.path, BytesIO(b"0000"), outf)
         outlines = outf.getvalue().splitlines()
         self.assertEqual([
-            b'00739e65bdcf4a22cdd4f3700604a275cd2aaf146b23 HEAD\x00 report-status '  # noqa: E501
-            b'delete-refs quiet ofs-delta side-band-64k no-done',
+            b'00919e65bdcf4a22cdd4f3700604a275cd2aaf146b23 HEAD\x00 report-status '  # noqa: E501
+            b'delete-refs quiet ofs-delta side-band-64k '
+            b'no-done symref=HEAD:refs/heads/master',
             b'003f9e65bdcf4a22cdd4f3700604a275cd2aaf146b23 refs/heads/master',
             b'0000'], outlines)
         self.assertEqual(0, exitcode)
@@ -1134,3 +1135,36 @@ class CheckIgnoreTests(PorcelainTestCase):
         self.assertEqual(
             ['foo'],
             list(porcelain.check_ignore(self.repo, ['foo'], no_index=True)))
+
+
+class UpdateHeadTests(PorcelainTestCase):
+
+    def test_set_to_branch(self):
+        [c1] = build_commit_graph(self.repo.object_store, [[1]])
+        self.repo.refs[b"refs/heads/blah"] = c1.id
+        porcelain.update_head(self.repo, "blah")
+        self.assertEqual(c1.id, self.repo.head())
+        self.assertEqual(b'ref: refs/heads/blah',
+                         self.repo.refs.read_ref('HEAD'))
+
+    def test_set_to_branch_detached(self):
+        [c1] = build_commit_graph(self.repo.object_store, [[1]])
+        self.repo.refs[b"refs/heads/blah"] = c1.id
+        porcelain.update_head(self.repo, "blah", detached=True)
+        self.assertEqual(c1.id, self.repo.head())
+        self.assertEqual(c1.id, self.repo.refs.read_ref(b'HEAD'))
+
+    def test_set_to_commit_detached(self):
+        [c1] = build_commit_graph(self.repo.object_store, [[1]])
+        self.repo.refs[b"refs/heads/blah"] = c1.id
+        porcelain.update_head(self.repo, c1.id, detached=True)
+        self.assertEqual(c1.id, self.repo.head())
+        self.assertEqual(c1.id, self.repo.refs.read_ref(b'HEAD'))
+
+    def test_set_new_branch(self):
+        [c1] = build_commit_graph(self.repo.object_store, [[1]])
+        self.repo.refs[b"refs/heads/blah"] = c1.id
+        porcelain.update_head(self.repo, "blah", new_branch="bar")
+        self.assertEqual(c1.id, self.repo.head())
+        self.assertEqual(b'ref: refs/heads/bar',
+                         self.repo.refs.read_ref(b'HEAD'))

dulwich/tests/test_refs.py

@@ -36,6 +36,7 @@ from dulwich.refs import (
     InfoRefsContainer,
     check_ref_format,
     _split_ref_line,
+    parse_symref_value,
     read_packed_refs_with_peeled,
     read_packed_refs,
     write_packed_refs,
@@ -164,6 +165,7 @@ _TEST_REFS = {
     b'refs/heads/packed': b'42d06bd4b77fed026b154d16493e5deab78f02ec',
     b'refs/tags/refs-0.1': b'df6800012397fb85c56e7418dd4eb9405dee075c',
     b'refs/tags/refs-0.2': b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8',
+    b'refs/heads/loop': b'ref: refs/heads/loop',
     }
 
 
@@ -172,8 +174,6 @@ class RefsContainerTests(object):
     def test_keys(self):
         actual_keys = set(self._refs.keys())
         self.assertEqual(set(self._refs.allkeys()), actual_keys)
-        # ignore the symref loop if it exists
-        actual_keys.discard(b'refs/heads/loop')
         self.assertEqual(set(_TEST_REFS.keys()), actual_keys)
 
         actual_keys = self._refs.keys(b'refs/heads')
@@ -186,7 +186,18 @@ class RefsContainerTests(object):
 
     def test_as_dict(self):
         # refs/heads/loop does not show up even if it exists
-        self.assertEqual(_TEST_REFS, self._refs.as_dict())
+        expected_refs = dict(_TEST_REFS)
+        del expected_refs[b'refs/heads/loop']
+        self.assertEqual(expected_refs, self._refs.as_dict())
+
+    def test_get_symrefs(self):
+        self._refs.set_symbolic_ref(b'refs/heads/src', b'refs/heads/dst')
+        symrefs = self._refs.get_symrefs()
+        if b'HEAD' in symrefs:
+            symrefs.pop(b'HEAD')
+        self.assertEqual({b'refs/heads/src': b'refs/heads/dst',
+                          b'refs/heads/loop': b'refs/heads/loop'},
+                         symrefs)
 
     def test_setitem(self):
         self._refs[b'refs/some/ref'] = (
@@ -288,6 +299,7 @@ class DictRefsContainerTests(RefsContainerTests, TestCase):
         # some way of injecting invalid refs.
         self._refs._refs[b'refs/stash'] = b'00' * 20
         expected_refs = dict(_TEST_REFS)
+        del expected_refs[b'refs/heads/loop']
         expected_refs[b'refs/stash'] = b'00' * 20
         self.assertEqual(expected_refs, self._refs.as_dict())
 
@@ -468,6 +480,7 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
 
         expected_refs = dict(_TEST_REFS)
         expected_refs[encoded_ref] = b'00' * 20
+        del expected_refs[b'refs/heads/loop']
 
         self.assertEqual(expected_refs, self._repo.get_refs())
 
@@ -489,16 +502,16 @@ class InfoRefsContainerTests(TestCase):
         expected_refs = dict(_TEST_REFS)
         del expected_refs[b'HEAD']
         expected_refs[b'refs/stash'] = b'00' * 20
+        del expected_refs[b'refs/heads/loop']
         self.assertEqual(expected_refs, refs.as_dict())
 
     def test_keys(self):
         refs = InfoRefsContainer(BytesIO(_TEST_REFS_SERIALIZED))
         actual_keys = set(refs.keys())
         self.assertEqual(set(refs.allkeys()), actual_keys)
-        # ignore the symref loop if it exists
-        actual_keys.discard(b'refs/heads/loop')
         expected_refs = dict(_TEST_REFS)
         del expected_refs[b'HEAD']
+        del expected_refs[b'refs/heads/loop']
         self.assertEqual(set(expected_refs.keys()), actual_keys)
 
         actual_keys = refs.keys(b'refs/heads')
@@ -514,6 +527,7 @@ class InfoRefsContainerTests(TestCase):
         # refs/heads/loop does not show up even if it exists
         expected_refs = dict(_TEST_REFS)
         del expected_refs[b'HEAD']
+        del expected_refs[b'refs/heads/loop']
         self.assertEqual(expected_refs, refs.as_dict())
 
     def test_contains(self):
@@ -527,3 +541,14 @@ class InfoRefsContainerTests(TestCase):
         self.assertEqual(
             _TEST_REFS[b'refs/heads/master'],
             refs.get_peeled(b'refs/heads/master'))
+
+
+class ParseSymrefValueTests(TestCase):
+
+    def test_valid(self):
+        self.assertEqual(
+                b'refs/heads/foo',
+                parse_symref_value(b'ref: refs/heads/foo'))
+
+    def test_invalid(self):
+        self.assertRaises(ValueError, parse_symref_value, b'foobar')

dulwich/tests/test_server.py

@@ -50,7 +50,7 @@ from dulwich.server import (
     _split_proto_line,
     serve_command,
     _find_shallow,
-    ProtocolGraphWalker,
+    _ProtocolGraphWalker,
     ReceivePackHandler,
     SingleAckGraphWalkerImpl,
     UploadPackHandler,
@@ -111,11 +111,11 @@ class TestGenericPackHandler(PackHandler):
 
     @classmethod
     def capabilities(cls):
-        return (b'cap1', b'cap2', b'cap3')
+        return [b'cap1', b'cap2', b'cap3']
 
     @classmethod
     def required_capabilities(cls):
-        return (b'cap2',)
+        return [b'cap2']
 
 
 class HandlerTestCase(TestCase):
@@ -131,7 +131,9 @@ class HandlerTestCase(TestCase):
             self.fail(e)
 
     def test_capability_line(self):
-        self.assertEqual(b' cap1 cap2 cap3', self._handler.capability_line())
+        self.assertEqual(
+                b' cap1 cap2 cap3',
+                self._handler.capability_line([b'cap1', b'cap2', b'cap3']))
 
     def test_set_client_capabilities(self):
         set_caps = self._handler.set_client_capabilities
@@ -288,9 +290,10 @@ class FindShallowTests(TestCase):
 
 
 class TestUploadPackHandler(UploadPackHandler):
+
     @classmethod
     def required_capabilities(self):
-        return ()
+        return []
 
 
 class ReceivePackHandlerTestCase(TestCase):
@@ -308,6 +311,7 @@ class ReceivePackHandlerTestCase(TestCase):
             b'refs/heads/fake-branch': ONE}
         self._repo.refs._update(refs)
         update_refs = [[ONE, ZERO_SHA, b'refs/heads/fake-branch'], ]
+        self._handler.set_client_capabilities([b'delete-refs'])
         status = self._handler._apply_pack(update_refs)
         self.assertEqual(status[0][0], b'unpack')
         self.assertEqual(status[0][1], b'ok')
@@ -320,10 +324,11 @@ class ProtocolGraphWalkerEmptyTestCase(TestCase):
         super(ProtocolGraphWalkerEmptyTestCase, self).setUp()
         self._repo = MemoryRepo.init_bare([], {})
         backend = DictBackend({b'/': self._repo})
-        self._walker = ProtocolGraphWalker(
+        self._walker = _ProtocolGraphWalker(
                 TestUploadPackHandler(backend, [b'/', b'host=lolcats'],
                                       TestProto()),
-                self._repo.object_store, self._repo.get_peeled)
+                self._repo.object_store, self._repo.get_peeled,
+                self._repo.refs.get_symrefs)
 
     def test_empty_repository(self):
         # The server should wait for a flush packet.
@@ -353,10 +358,11 @@ class ProtocolGraphWalkerTestCase(TestCase):
           ]
         self._repo = MemoryRepo.init_bare(commits, {})
         backend = DictBackend({b'/': self._repo})
-        self._walker = ProtocolGraphWalker(
+        self._walker = _ProtocolGraphWalker(
                 TestUploadPackHandler(backend, [b'/', b'host=lolcats'],
                                       TestProto()),
-                self._repo.object_store, self._repo.get_peeled)
+                self._repo.object_store, self._repo.get_peeled,
+                self._repo.refs.get_symrefs)
 
     def test_all_wants_satisfied_no_haves(self):
         self._walker.set_wants([ONE])

setup.py

@@ -11,7 +11,7 @@ from distutils.core import Distribution
 import os
 import sys
 
-dulwich_version_string = '0.18.2'
+dulwich_version_string = '0.18.3'
 
 include_dirs = []
 # Windows MSVC support