Merged master

AUTHORS

@@ -2,6 +2,7 @@ Jelmer Vernooij <jelmer@samba.org>
 James Westby <jw+debian@jameswestby.net>
 John Carr <john.carr@unrouted.co.uk>
 Dave Borowitz <dborowitz@google.com>
+Chris Eberle <eberle1080@gmail.com>
 
 Hervé Cauwelier <herve@itaapy.com> wrote the original tutorial.
 

Makefile

@@ -25,12 +25,20 @@ install::
 check:: build
 	$(RUNTEST) dulwich.tests.test_suite
 
+check-tutorial:: build
+	$(RUNTEST) dulwich.tests.tutorial_test_suite 
+
 check-nocompat:: build
 	$(RUNTEST) dulwich.tests.nocompat_test_suite
 
+check-pypy:: clean
+	$(MAKE) check-noextensions PYTHON=pypy
+
 check-noextensions:: clean
 	$(RUNTEST) dulwich.tests.test_suite
 
+check-all: check check-pypy check-noextensions
+
 clean::
 	$(SETUP) clean --all
 	rm -f dulwich/*.so

NEWS

@@ -1,4 +1,62 @@
-0.8.1	UNRELEASED
+0.8.4	UNRELEASED
+
+ BUG FIXES
+
+  * Options on the same line as sections in config files are now supported.
+    (Jelmer Vernooij, #920553)
+
+ TESTS
+
+  * $HOME is now explicitly specified for tests that use it to read
+    ``~/.gitconfig``, to prevent test isolation issues.
+    (Jelmer Vernooij, #920330)
+
+0.8.3	2012-01-21
+
+ FEATURES
+
+  * The config parser now supports the git-config file format as
+    described in git-config(1) and can write git config files.
+    (Jelmer Vernooij, #531092, #768687)
+
+  * ``Repo.do_commit`` will now use the user identity from
+    .git/config or ~/.gitconfig if none was explicitly specified.
+   (Jelmer Vernooij)
+
+ BUG FIXES
+
+  * Allow ``determine_wants`` methods to include the zero sha in their
+    return value. (Jelmer Vernooij)
+
+0.8.2	2011-12-18
+
+ BUG FIXES
+
+  * Cope with different zlib buffer sizes in sha1 file parser.
+    (Jelmer Vernooij)
+
+  * Fix get_transport_and_path for HTTP/HTTPS URLs.
+    (Bruno Renié)
+
+  * Avoid calling free_objects() on NULL in error cases. (Chris Eberle)
+
+  * Fix use --bare argument to 'dulwich init'. (Chris Eberle)
+
+  * Properly abort connections when the determine_wants function
+    raises an exception. (Jelmer Vernooij, #856769)
+
+  * Tweak xcodebuild hack to deal with more error output.
+    (Jelmer Vernooij, #903840)
+
+ FEATURES
+
+  * Add support for retrieving tarballs from remote servers.
+    (Jelmer Vernooij, #379087)
+
+  * New method ``update_server_info`` which generates data
+    for dumb server access. (Jelmer Vernooij, #731235)
+
+0.8.1	2011-10-31
 
  FEATURES
 
@@ -6,13 +64,43 @@
 
   * Repo.do_commit has a new argument 'merge_heads'. (Jelmer Vernooij)
 
+  * New ``Repo.get_walker`` method. (Jelmer Vernooij)
+
   * New ``Repo.clone`` method. (Jelmer Vernooij, #725369)
 
+  * ``GitClient.send_pack`` now supports the 'side-band-64k' capability.
+    (Jelmer Vernooij)
+
+  * ``HttpGitClient`` which supports the smart server protocol over
+    HTTP. "dumb" access is not yet supported. (Jelmer Vernooij, #373688)
+
+  * Add basic support for alternates. (Jelmer Vernooij, #810429)
+
  CHANGES
 
   * unittest2 or python >= 2.7 is now required for the testsuite.
     testtools is no longer supported. (Jelmer Vernooij, #830713)
 
+ BUG FIXES
+
+  * Fix compilation with older versions of MSVC.  (Martin gz)
+
+  * Special case 'refs/stash' as a valid ref. (Jelmer Vernooij, #695577)
+
+  * Smart protocol clients can now change refs even if they are
+    not uploading new data. (Jelmer Vernooij, #855993)
+
+ * Don't compile C extensions when running in pypy.
+   (Ronny Pfannschmidt, #881546)
+
+ * Use different name for strnlen replacement function to avoid clashing
+   with system strnlen. (Jelmer Vernooij, #880362)
+
+ API CHANGES
+
+  * ``Repo.revision_history`` is now deprecated in favor of ``Repo.get_walker``.
+    (Jelmer Vernooij)
+
 0.8.0	2011-08-07
 
  FEATURES

README

@@ -11,3 +11,11 @@ maintained by Jelmer Vernooij et al.
 Please file bugs in the Dulwich project on Launchpad: 
 
 https://bugs.launchpad.net/dulwich/+filebug
+
+The dulwich documentation can be found in doc/ and on the web:
+
+http://www.samba.org/~jelmer/dulwich/docs/
+
+The API reference can be generated using pydoctor, by running "make pydoctor", or on the web:
+
+http://www.samba.org/~jelmer/dulwich/apidocs

bin/dulwich

@@ -1,6 +1,8 @@
-#!/usr/bin/python
+#!/usr/bin/python -u
+#
 # dulwich - Simple command-line interface to Dulwich
-# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2011 Jelmer Vernooij <jelmer@samba.org>
+# vim: expandtab
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -33,6 +35,14 @@ from dulwich.errors import ApplyDeltaError
 from dulwich.index import Index
 from dulwich.pack import Pack, sha_to_hex
 from dulwich.repo import Repo
+from dulwich.server import update_server_info
+
+
+def cmd_archive(args):
+    opts, args = getopt(args, "", [])
+    client, path = get_transport_and_path(args.pop(0))
+    committish = args.pop(0)
+    client.archive(path, committish, sys.stdout.write, sys.stderr.write)
 
 
 def cmd_fetch_pack(args):
@@ -44,8 +54,17 @@ def cmd_fetch_pack(args):
         determine_wants = r.object_store.determine_wants_all
     else:
         determine_wants = lambda x: [y for y in args if not y in r.object_store]
-    graphwalker = r.get_graph_walker()
-    client.fetch(path, r.object_store, determine_wants)
+    client.fetch(path, r, determine_wants)
+
+
+def cmd_fetch(args):
+    opts, args = getopt(args, "", [])
+    opts = dict(opts)
+    client, path = get_transport_and_path(args.pop(0))
+    r = Repo(".")
+    if "--all" in opts:
+        determine_wants = r.object_store.determine_wants_all
+    refs = client.fetch(path, r, progress=sys.stdout.write)
 
 
 def cmd_log(args):
@@ -114,7 +133,7 @@ def cmd_dump_index(args):
 
 
 def cmd_init(args):
-    opts, args = getopt(args, "", ["--bare"])
+    opts, args = getopt(args, "", ["bare"])
     opts = dict(opts)
 
     if args == []:
@@ -165,14 +184,22 @@ def cmd_commit(args):
     r.do_commit(committer=committer, author=author, message=opts["--message"])
 
 
+def cmd_update_server_info(args):
+    r = Repo(".")
+    update_server_info(r)
+
+
 commands = {
     "commit": cmd_commit,
     "fetch-pack": cmd_fetch_pack,
+    "fetch": cmd_fetch,
     "dump-pack": cmd_dump_pack,
     "dump-index": cmd_dump_index,
     "init": cmd_init,
     "log": cmd_log,
     "clone": cmd_clone,
+    "archive": cmd_archive,
+    "update-server-info": cmd_update_server_info,
     }
 
 if len(sys.argv) < 2:

docs/tutorial/index.txt

@@ -10,5 +10,6 @@ Tutorial
    introduction
    repo
    object-store
+   remote
    conclusion
 

docs/tutorial/remote.txt

@@ -0,0 +1,83 @@
+.. _tutorial-remote:
+
+Most of the tests in this file require a Dulwich server, so let's start one:
+
+    >>> from dulwich.repo import Repo
+    >>> from dulwich.server import DictBackend, TCPGitServer
+    >>> import threading
+    >>> repo = Repo.init("remote", mkdir=True)
+    >>> cid = repo.do_commit("message", committer="Jelmer <jelmer@samba.org>")
+    >>> backend = DictBackend({'/': repo})
+    >>> dul_server = TCPGitServer(backend, 'localhost', 0)
+    >>> threading.Thread(target=dul_server.serve).start()
+    >>> server_address, server_port = dul_server.socket.getsockname()
+
+Remote repositories
+===================
+
+The interface for remote Git repositories is different from that
+for local repositories.
+
+The Git smart server protocol provides three basic operations:
+
+ * upload-pack - provides a pack with objects requested by the client
+ * receive-pack - imports a pack with objects provided by the client
+ * upload-archive - provides a tarball with the contents of a specific revision
+
+The smart server protocol can be accessed over either plain TCP (git://),
+SSH (git+ssh://) or tunneled over HTTP (http://).
+
+Dulwich provides support for accessing remote repositories in
+``dulwich.client``. To create a new client, you can either construct
+one manually::
+
+   >>> from dulwich.client import TCPGitClient
+   >>> client = TCPGitClient(server_address, server_port)
+
+Retrieving raw pack files
+-------------------------
+
+The client object can then be used to retrieve a pack. The ``fetch_pack``
+method takes a ``determine_wants`` callback argument, which allows the
+client to determine which objects it wants to end up with::
+
+   >>> def determine_wants(refs):
+   ...    # retrieve all objects
+   ...    return refs.values()
+
+Another required object is a "graph walker", which is used to determine
+which objects that the client already has should not be sent again
+by the server. Here in the tutorial we'll just use a dummy graph walker
+which claims that the client doesn't have any objects::
+
+   >>> class DummyGraphWalker(object):
+   ...     def ack(self, sha): pass
+   ...     def next(self): pass
+
+With the determine_wants function in place, we can now fetch a pack,
+which we will write to a ``StringIO`` object::
+
+   >>> from cStringIO import StringIO
+   >>> f = StringIO()
+   >>> remote_refs = client.fetch_pack("/", determine_wants,
+   ...    DummyGraphWalker(), pack_data=f.write)
+
+``f`` will now contain a full pack file::
+
+   >>> f.getvalue()[:4]
+   'PACK'
+
+Fetching objects into a local repository
+----------------------------------------
+
+It also possible to fetch from a remote repository into a local repository,
+in which case dulwich takes care of providing the right graph walker, and
+importing the received pack file into the local repository::
+
+   >>> from dulwich.repo import Repo
+   >>> local = Repo.init("local", mkdir=True)
+   >>> remote_refs = client.fetch("/", local)
+
+Let's show down the server now that all tests have been run::
+
+   >>> dul_server.shutdown()

docs/tutorial/repo.txt

@@ -1,6 +1,6 @@
 .. _tutorial-repo:
 
-The Repository
+The repository
 ==============
 
 After this introduction, let's start directly with code::
@@ -18,6 +18,9 @@ repositories:
   contains itself the "branches", "hooks"... folders. These are used for
   published repositories (mirrors). They do not have a working tree.
 
+Creating a repository
+---------------------
+
 Let's create a folder and turn it into a repository, like ``git init`` would::
 
   >>> from os import mkdir
@@ -28,3 +31,70 @@ Let's create a folder and turn it into a repository, like ``git init`` would::
 
 You can already look a the structure of the "myrepo/.git" folder, though it
 is mostly empty for now.
+
+Opening an existing repository
+------------------------------
+
+To reopen an existing repository, simply pass its path to the constructor
+of ``Repo``::
+
+    >>> repo = Repo("myrepo")
+    >>> repo
+    <Repo at 'myrepo'>
+
+Opening the index
+-----------------
+
+The index is used as a staging area. Once you do a commit,
+the files tracked in the index will be recorded as the contents of the new
+commit. As mentioned earlier, only non-bare repositories have a working tree,
+so only non-bare repositories will have an index, too. To open the index, simply
+call::
+
+    >>> index = repo.open_index()
+    >>> repr(index).replace('\\\\', '/')
+    "Index('myrepo/.git/index')"
+
+Since the repository was just created, the index will be empty::
+
+    >>> list(index)
+    []
+
+Staging new files
+-----------------
+
+The repository allows "staging" files. Only files can be staged - directories
+aren't tracked explicitly by git. Let's create a simple text file and stage it::
+
+    >>> f = open('myrepo/foo', 'w')
+    >>> f.write("monty")
+    >>> f.close()
+
+    >>> repo.stage(["foo"])
+
+It will now show up in the index::
+
+    >>> list(repo.open_index())
+    ['foo']
+
+
+Creating new commits
+--------------------
+
+Now that we have staged a change, we can commit it. The easiest way to
+do this is by using ``Repo.do_commit``. It is also possible to manipulate
+the lower-level objects involved in this, but we'll leave that for a
+separate chapter of the tutorial.
+
+To create a simple commit on the current branch, it is only necessary
+to specify the message. The committer and author will be retrieved from the
+repository configuration or global configuration if they are not specified::
+
+    >>> commit_id = repo.do_commit(
+    ...     "The first commit", committer="Jelmer Vernooij <jelmer@samba.org>")
+
+``do_commit`` returns the SHA1 of the commit. Since the commit was to the 
+default branch, the repository's head will now be set to that commit::
+
+    >>> repo.head() == commit_id
+    True

dulwich/__init__.py

@@ -23,4 +23,4 @@
 
 from dulwich import (client, protocol, repo, server)
 
-__version__ = (0, 8, 1)
+__version__ = (0, 8, 4)

dulwich/_diff_tree.c

@@ -135,7 +135,8 @@ static PyObject **tree_entries(char *path, Py_ssize_t path_len, PyObject *tree,
 	return result;
 
 error:
-	free_objects(result, i);
+	if (result)
+		free_objects(result, i);
 	Py_DECREF(items);
 	return NULL;
 }
@@ -243,8 +244,10 @@ error:
 	result = NULL;
 
 done:
-	free_objects(entries1, n1);
-	free_objects(entries2, n2);
+	if (entries1)
+		free_objects(entries1, n1);
+	if (entries2)
+		free_objects(entries2, n2);
 	return result;
 }
 

dulwich/_objects.c

@@ -21,20 +21,18 @@
 #include <stdlib.h>
 #include <sys/stat.h>
 
-#if defined(__APPLE__)
-#include <Availability.h>
-#endif
-
 #if (PY_VERSION_HEX < 0x02050000)
 typedef int Py_ssize_t;
 #endif
 
-#if defined(__MINGW32_VERSION) || (defined(__APPLE__) && __MAC_OS_X_VERSION_MIN_REQUIRED < 1070)
-size_t strnlen(char *text, size_t maxlen)
+#if defined(__MINGW32_VERSION) || defined(__APPLE__)
+size_t rep_strnlen(char *text, size_t maxlen);
+size_t rep_strnlen(char *text, size_t maxlen)
 {
 	const char *last = memchr(text, '\0', maxlen);
 	return last ? (size_t) (last - text) : maxlen;
 }
+#define strnlen rep_strnlen
 #endif
 
 #define bytehex(x) (((x)<0xa)?('0'+(x)):('a'-0xa+(x)))

dulwich/client.py

@@ -17,21 +17,43 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-"""Client side support for the Git protocol."""
+"""Client side support for the Git protocol.
+
+The Dulwich client supports the following capabilities:
+
+ * thin-pack
+ * multi_ack_detailed
+ * multi_ack
+ * side-band-64k
+ * ofs-delta
+ * report-status
+ * delete-refs
+
+Known capabilities that are not supported:
+
+ * shallow
+ * no-progress
+ * include-tag
+"""
 
 __docformat__ = 'restructuredText'
 
+from cStringIO import StringIO
 import select
 import socket
 import subprocess
+import urllib2
 import urlparse
 
 from dulwich.errors import (
     GitProtocolError,
+    NotGitRepository,
     SendPackError,
     UpdateRefsError,
     )
 from dulwich.protocol import (
+    _RBUFSIZE,
+    PktLineParser,
     Protocol,
     TCP_GIT_PORT,
     ZERO_SHA,
@@ -52,13 +74,72 @@ def _fileno_can_read(fileno):
     """Check if a file descriptor is readable."""
     return len(select.select([fileno], [], [], 0)[0]) > 0
 
-COMMON_CAPABILITIES = ['ofs-delta']
-FETCH_CAPABILITIES = ['multi_ack', 'side-band-64k'] + COMMON_CAPABILITIES
+COMMON_CAPABILITIES = ['ofs-delta', 'side-band-64k']
+FETCH_CAPABILITIES = ['multi_ack', 'multi_ack_detailed'] + COMMON_CAPABILITIES
 SEND_CAPABILITIES = ['report-status'] + COMMON_CAPABILITIES
 
+
+class ReportStatusParser(object):
+    """Handle status as reported by servers with the 'report-status' capability.
+    """
+
+    def __init__(self):
+        self._done = False
+        self._pack_status = None
+        self._ref_status_ok = True
+        self._ref_statuses = []
+
+    def check(self):
+        """Check if there were any errors and, if so, raise exceptions.
+
+        :raise SendPackError: Raised when the server could not unpack
+        :raise UpdateRefsError: Raised when refs could not be updated
+        """
+        if self._pack_status not in ('unpack ok', None):
+            raise SendPackError(self._pack_status)
+        if not self._ref_status_ok:
+            ref_status = {}
+            ok = set()
+            for status in self._ref_statuses:
+                if ' ' not in status:
+                    # malformed response, move on to the next one
+                    continue
+                status, ref = status.split(' ', 1)
+
+                if status == 'ng':
+                    if ' ' in ref:
+                        ref, status = ref.split(' ', 1)
+                else:
+                    ok.add(ref)
+                ref_status[ref] = status
+            raise UpdateRefsError('%s failed to update' %
+                                  ', '.join([ref for ref in ref_status
+                                             if ref not in ok]),
+                                  ref_status=ref_status)
+
+    def handle_packet(self, pkt):
+        """Handle a packet.
+
+        :raise GitProtocolError: Raised when packets are received after a
+            flush packet.
+        """
+        if self._done:
+            raise GitProtocolError("received more data after status report")
+        if pkt is None:
+            self._done = True
+            return
+        if self._pack_status is None:
+            self._pack_status = pkt.strip()
+        else:
+            ref_status = pkt.strip()
+            self._ref_statuses.append(ref_status)
+            if not ref_status.startswith('ok '):
+                self._ref_status_ok = False
+
+
 # TODO(durin42): this doesn't correctly degrade if the server doesn't
 # support some capabilities. This should work properly with servers
-# that don't support side-band-64k and multi_ack.
+# that don't support multi_ack.
 class GitClient(object):
     """Git smart server client.
 
@@ -77,12 +158,27 @@ class GitClient(object):
         if thin_packs:
             self._fetch_capabilities.append('thin-pack')
 
-    def send_pack(self, path, determine_wants, generate_pack_contents):
+    def _read_refs(self, proto):
+        server_capabilities = None
+        refs = {}
+        # Receive refs from server
+        for pkt in proto.read_pkt_seq():
+            (sha, ref) = pkt.rstrip('\n').split(' ', 1)
+            if sha == 'ERR':
+                raise GitProtocolError(ref)
+            if server_capabilities is None:
+                (ref, server_capabilities) = extract_capabilities(ref)
+            refs[ref] = sha
+        return refs, server_capabilities
+
+    def send_pack(self, path, determine_wants, generate_pack_contents,
+                  progress=None):
         """Upload a pack to a remote repository.
 
         :param path: Repository path
         :param generate_pack_contents: Function that can return a sequence of the
             shas of the objects to upload.
+        :param progress: Optional progress function
 
         :raises SendPackError: if server rejects the pack data
         :raises UpdateRefsError: if the server supports report-status
@@ -98,7 +194,7 @@ class GitClient(object):
         :param determine_wants: Optional function to determine what refs
             to fetch
         :param progress: Optional progress function
-        :return: remote refs
+        :return: remote refs as dictionary
         """
         if determine_wants is None:
             determine_wants = target.object_store.determine_wants_all
@@ -110,7 +206,7 @@ class GitClient(object):
             commit()
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-                   progress):
+                   progress=None):
         """Retrieve a pack from a git smart server.
 
         :param determine_wants: Callback that returns list of commits to fetch
@@ -120,37 +216,6 @@ class GitClient(object):
         """
         raise NotImplementedError(self.fetch_pack)
 
-
-class TraditionalGitClient(GitClient):
-    """Traditional Git client."""
-
-    def _connect(self, cmd, path):
-        """Create a connection to the server.
-
-        This method is abstract - concrete implementations should
-        implement their own variant which connects to the server and
-        returns an initialized Protocol object with the service ready
-        for use and a can_read function which may be used to see if
-        reads would block.
-
-        :param cmd: The git service name to which we should connect.
-        :param path: The path we should pass to the service.
-        """
-        raise NotImplementedError()
-
-    def _read_refs(self, proto):
-        server_capabilities = None
-        refs = {}
-        # Receive refs from server
-        for pkt in proto.read_pkt_seq():
-            (sha, ref) = pkt.rstrip('\n').split(' ', 1)
-            if sha == 'ERR':
-                raise GitProtocolError(ref)
-            if server_capabilities is None:
-                (ref, server_capabilities) = extract_capabilities(ref)
-            refs[ref] = sha
-        return refs, server_capabilities
-
     def _parse_status_report(self, proto):
         unpack = proto.read_pkt_line().strip()
         if unpack != 'unpack ok':
@@ -189,26 +254,35 @@ class TraditionalGitClient(GitClient):
                                              if ref not in ok]),
                                   ref_status=ref_status)
 
-    # TODO(durin42): add side-band-64k capability support here and advertise it
-    def send_pack(self, path, determine_wants, generate_pack_contents):
-        """Upload a pack to a remote repository.
+    def _read_side_band64k_data(self, proto, channel_callbacks):
+        """Read per-channel data.
 
-        :param path: Repository path
-        :param generate_pack_contents: Function that can return a sequence of the
-            shas of the objects to upload.
+        This requires the side-band-64k capability.
 
-        :raises SendPackError: if server rejects the pack data
-        :raises UpdateRefsError: if the server supports report-status
-                                 and rejects ref updates
+        :param proto: Protocol object to read from
+        :param channel_callbacks: Dictionary mapping channels to packet
+            handlers to use. None for a callback discards channel data.
+        """
+        for pkt in proto.read_pkt_seq():
+            channel = ord(pkt[0])
+            pkt = pkt[1:]
+            try:
+                cb = channel_callbacks[channel]
+            except KeyError:
+                raise AssertionError('Invalid sideband channel %d' % channel)
+            else:
+                if cb is not None:
+                    cb(pkt)
+
+    def _handle_receive_pack_head(self, proto, capabilities, old_refs, new_refs):
+        """Handle the head of a 'git-receive-pack' request.
+
+        :param proto: Protocol object to read from
+        :param capabilities: List of negotiated capabilities
+        :param old_refs: Old refs, as received from the server
+        :param new_refs: New refs
+        :return: (have, want) tuple
         """
-        proto, unused_can_read = self._connect('receive-pack', path)
-        old_refs, server_capabilities = self._read_refs(proto)
-        if 'report-status' not in server_capabilities:
-            self._send_capabilities.remove('report-status')
-        new_refs = determine_wants(old_refs)
-        if not new_refs:
-            proto.write_pkt_line(None)
-            return {}
         want = []
         have = [x for x in old_refs.values() if not x == ZERO_SHA]
         sent_capabilities = False
@@ -222,42 +296,57 @@ class TraditionalGitClient(GitClient):
                 else:
                     proto.write_pkt_line(
                       '%s %s %s\0%s' % (old_sha1, new_sha1, refname,
-                                        ' '.join(self._send_capabilities)))
+                                        ' '.join(capabilities)))
                     sent_capabilities = True
             if new_sha1 not in have and new_sha1 != ZERO_SHA:
                 want.append(new_sha1)
         proto.write_pkt_line(None)
-        if not want:
-            return new_refs
-        objects = generate_pack_contents(have, want)
-        entries, sha = write_pack_objects(proto.write_file(), objects)
+        return (have, want)
 
-        if 'report-status' in self._send_capabilities:
-            self._parse_status_report(proto)
+    def _handle_receive_pack_tail(self, proto, capabilities, progress=None):
+        """Handle the tail of a 'git-receive-pack' request.
+
+        :param proto: Protocol object to read from
+        :param capabilities: List of negotiated capabilities
+        :param progress: Optional progress reporting function
+        """
+        if 'report-status' in capabilities:
+            report_status_parser = ReportStatusParser()
+        else:
+            report_status_parser = None
+        if "side-band-64k" in capabilities:
+            if progress is None:
+                progress = lambda x: None
+            channel_callbacks = { 2: progress }
+            if 'report-status' in capabilities:
+                channel_callbacks[1] = PktLineParser(
+                    report_status_parser.handle_packet).parse
+            self._read_side_band64k_data(proto, channel_callbacks)
+        else:
+            if 'report-status':
+                for pkt in proto.read_pkt_seq():
+                    report_status_parser.handle_packet(pkt)
+        if report_status_parser is not None:
+            report_status_parser.check()
         # wait for EOF before returning
         data = proto.read()
         if data:
             raise SendPackError('Unexpected response %r' % data)
-        return new_refs
 
-    def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-                   progress):
-        """Retrieve a pack from a git smart server.
+    def _handle_upload_pack_head(self, proto, capabilities, graph_walker,
+                                 wants, can_read):
+        """Handle the head of a 'git-upload-pack' request.
 
-        :param determine_wants: Callback that returns list of commits to fetch
-        :param graph_walker: Object with next() and ack().
-        :param pack_data: Callback called for each bit of data in the pack
-        :param progress: Callback for progress reports (strings)
+        :param proto: Protocol object to read from
+        :param capabilities: List of negotiated capabilities
+        :param graph_walker: GraphWalker instance to call .ack() on
+        :param wants: List of commits to fetch
+        :param can_read: function that returns a boolean that indicates
+            whether there is extra graph data to read on proto
         """
-        proto, can_read = self._connect('upload-pack', path)
-        (refs, server_capabilities) = self._read_refs(proto)
-        wants = determine_wants(refs)
-        if not wants:
-            proto.write_pkt_line(None)
-            return refs
         assert isinstance(wants, list) and type(wants[0]) == str
         proto.write_pkt_line('want %s %s\n' % (
-            wants[0], ' '.join(self._fetch_capabilities)))
+            wants[0], ' '.join(capabilities)))
         for want in wants[1:]:
             proto.write_pkt_line('want %s\n' % want)
         proto.write_pkt_line(None)
@@ -269,31 +358,154 @@ class TraditionalGitClient(GitClient):
                 parts = pkt.rstrip('\n').split(' ')
                 if parts[0] == 'ACK':
                     graph_walker.ack(parts[1])
-                    assert parts[2] == 'continue'
+                    if parts[2] in ('continue', 'common'):
+                        pass
+                    elif parts[2] == 'ready':
+                        break
+                    else:
+                        raise AssertionError(
+                            "%s not in ('continue', 'ready', 'common)" %
+                            parts[2])
             have = graph_walker.next()
         proto.write_pkt_line('done\n')
+
+    def _handle_upload_pack_tail(self, proto, capabilities, graph_walker,
+                                 pack_data, progress=None, rbufsize=_RBUFSIZE):
+        """Handle the tail of a 'git-upload-pack' request.
+
+        :param proto: Protocol object to read from
+        :param capabilities: List of negotiated capabilities
+        :param graph_walker: GraphWalker instance to call .ack() on
+        :param pack_data: Function to call with pack data
+        :param progress: Optional progress reporting function
+        :param rbufsize: Read buffer size
+        """
         pkt = proto.read_pkt_line()
         while pkt:
             parts = pkt.rstrip('\n').split(' ')
             if parts[0] == 'ACK':
                 graph_walker.ack(pkt.split(' ')[1])
-            if len(parts) < 3 or parts[2] != 'continue':
+            if len(parts) < 3 or parts[2] not in (
+                    'ready', 'continue', 'common'):
                 break
             pkt = proto.read_pkt_line()
-        # TODO(durin42): this is broken if the server didn't support the
-        # side-band-64k capability.
-        for pkt in proto.read_pkt_seq():
-            channel = ord(pkt[0])
-            pkt = pkt[1:]
-            if channel == 1:
-                pack_data(pkt)
-            elif channel == 2:
-                if progress is not None:
-                    progress(pkt)
-            else:
-                raise AssertionError('Invalid sideband channel %d' % channel)
+        if "side-band-64k" in capabilities:
+            if progress is None:
+                # Just ignore progress data
+                progress = lambda x: None
+            self._read_side_band64k_data(proto, {1: pack_data, 2: progress})
+            # wait for EOF before returning
+            data = proto.read()
+            if data:
+                raise Exception('Unexpected response %r' % data)
+        else:
+            while True:
+                data = self.read(rbufsize)
+                if data == "":
+                    break
+                pack_data(data)
+
+
+class TraditionalGitClient(GitClient):
+    """Traditional Git client."""
+
+    def _connect(self, cmd, path):
+        """Create a connection to the server.
+
+        This method is abstract - concrete implementations should
+        implement their own variant which connects to the server and
+        returns an initialized Protocol object with the service ready
+        for use and a can_read function which may be used to see if
+        reads would block.
+
+        :param cmd: The git service name to which we should connect.
+        :param path: The path we should pass to the service.
+        """
+        raise NotImplementedError()
+
+    def send_pack(self, path, determine_wants, generate_pack_contents,
+                  progress=None):
+        """Upload a pack to a remote repository.
+
+        :param path: Repository path
+        :param generate_pack_contents: Function that can return a sequence of the
+            shas of the objects to upload.
+        :param progress: Optional callback called with progress updates
+
+        :raises SendPackError: if server rejects the pack data
+        :raises UpdateRefsError: if the server supports report-status
+                                 and rejects ref updates
+        """
+        proto, unused_can_read = self._connect('receive-pack', path)
+        old_refs, server_capabilities = self._read_refs(proto)
+        negotiated_capabilities = list(self._send_capabilities)
+        if 'report-status' not in server_capabilities:
+            negotiated_capabilities.remove('report-status')
+        try:
+            new_refs = determine_wants(old_refs)
+        except:
+            proto.write_pkt_line(None)
+            raise
+        if new_refs is None:
+            proto.write_pkt_line(None)
+            return old_refs
+        (have, want) = self._handle_receive_pack_head(proto,
+            negotiated_capabilities, old_refs, new_refs)
+        if not want and old_refs == new_refs:
+            return new_refs
+        objects = generate_pack_contents(have, want)
+        if len(objects) > 0:
+            entries, sha = write_pack_objects(proto.write_file(), objects)
+        self._handle_receive_pack_tail(proto, negotiated_capabilities,
+            progress)
+        return new_refs
+
+    def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
+                   progress=None):
+        """Retrieve a pack from a git smart server.
+
+        :param determine_wants: Callback that returns list of commits to fetch
+        :param graph_walker: Object with next() and ack().
+        :param pack_data: Callback called for each bit of data in the pack
+        :param progress: Callback for progress reports (strings)
+        """
+        proto, can_read = self._connect('upload-pack', path)
+        (refs, server_capabilities) = self._read_refs(proto)
+        negotiated_capabilities = list(self._fetch_capabilities)
+        try:
+            wants = determine_wants(refs)
+        except:
+            proto.write_pkt_line(None)
+            raise
+        if wants is not None:
+            wants = [cid for cid in wants if cid != ZERO_SHA]
+        if not wants:
+            proto.write_pkt_line(None)
+            return refs
+        self._handle_upload_pack_head(proto, negotiated_capabilities,
+            graph_walker, wants, can_read)
+        self._handle_upload_pack_tail(proto, negotiated_capabilities,
+            graph_walker, pack_data, progress)
         return refs
 
+    def archive(self, path, committish, write_data, progress=None):
+        proto, can_read = self._connect('upload-archive', path)
+        proto.write_pkt_line("argument %s" % committish)
+        proto.write_pkt_line(None)
+        pkt = proto.read_pkt_line()
+        if pkt == "NACK\n":
+            return
+        elif pkt == "ACK\n":
+            pass
+        elif pkt.startswith("ERR "):
+            raise GitProtocolError(pkt[4:].rstrip("\n"))
+        else:
+            raise AssertionError("invalid response %r" % pkt)
+        ret = proto.read_pkt_line()
+        if ret is not None:
+            raise AssertionError("expected pkt tail")
+        self._read_side_band64k_data(proto, {1: write_data, 2: progress})
+
 
 class TCPGitClient(TraditionalGitClient):
     """A Git Client that works over TCP directly (i.e. git://)."""
@@ -362,6 +574,10 @@ class SubprocessGitClient(TraditionalGitClient):
 
     def __init__(self, *args, **kwargs):
         self._connection = None
+        self._stderr = None
+        self._stderr = kwargs.get('stderr')
+        if 'stderr' in kwargs:
+            del kwargs['stderr']
         TraditionalGitClient.__init__(self, *args, **kwargs)
 
     def _connect(self, service, path):
@@ -369,7 +585,8 @@ class SubprocessGitClient(TraditionalGitClient):
         argv = ['git', service, path]
         p = SubprocessWrapper(
             subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
-                             stdout=subprocess.PIPE))
+                             stdout=subprocess.PIPE,
+                             stderr=self._stderr))
         return Protocol(p.read, p.write,
                         report_activity=self._report_activity), p.can_read
 
@@ -416,43 +633,135 @@ class SSHGitClient(TraditionalGitClient):
 
 class HttpGitClient(GitClient):
 
-    def __init__(self, host, port=None, username=None, force_dumb=False, *args, **kwargs):
-        self.host = host
-        self.port = port
-        self.force_dumb = force_dumb
-        self.username = username
+    def __init__(self, base_url, dumb=None, *args, **kwargs):
+        self.base_url = base_url.rstrip("/") + "/"
+        self.dumb = dumb
         GitClient.__init__(self, *args, **kwargs)
 
-    @classmethod
-    def from_url(cls, url):
-        parsed = urlparse.urlparse(url)
-        assert parsed.scheme == 'http'
-        return cls(parsed.hostname, port=parsed.port, username=parsed.port,
-                   password=parsed.password)
+    def _get_url(self, path):
+        return urlparse.urljoin(self.base_url, path).rstrip("/") + "/"
+
+    def _perform(self, req):
+        """Perform a HTTP request.
+
+        This is provided so subclasses can provide their own version.
 
-    def send_pack(self, path, determine_wants, generate_pack_contents):
+        :param req: urllib2.Request instance
+        :return: matching response
+        """
+        return urllib2.urlopen(req)
+
+    def _discover_references(self, service, url):
+        assert url[-1] == "/"
+        url = urlparse.urljoin(url, "info/refs")
+        headers = {}
+        if self.dumb != False:
+            url += "?service=%s" % service
+            headers["Content-Type"] = "application/x-%s-request" % service
+        req = urllib2.Request(url, headers=headers)
+        resp = self._perform(req)
+        if resp.getcode() == 404:
+            raise NotGitRepository()
+        if resp.getcode() != 200:
+            raise GitProtocolError("unexpected http response %d" %
+                resp.getcode())
+        self.dumb = (not resp.info().gettype().startswith("application/x-git-"))
+        proto = Protocol(resp.read, None)
+        if not self.dumb:
+            # The first line should mention the service
+            pkts = list(proto.read_pkt_seq())
+            if pkts != [('# service=%s\n' % service)]:
+                raise GitProtocolError(
+                    "unexpected first line %r from smart server" % pkts)
+        return self._read_refs(proto)
+
+    def _smart_request(self, service, url, data):
+        assert url[-1] == "/"
+        url = urlparse.urljoin(url, service)
+        req = urllib2.Request(url,
+            headers={"Content-Type": "application/x-%s-request" % service},
+            data=data)
+        resp = self._perform(req)
+        if resp.getcode() == 404:
+            raise NotGitRepository()
+        if resp.getcode() != 200:
+            raise GitProtocolError("Invalid HTTP response from server: %d"
+                % resp.getcode())
+        if resp.info().gettype() != ("application/x-%s-result" % service):
+            raise GitProtocolError("Invalid content-type from server: %s"
+                % resp.info().gettype())
+        return resp
+
+    def send_pack(self, path, determine_wants, generate_pack_contents,
+                  progress=None):
         """Upload a pack to a remote repository.
 
         :param path: Repository path
         :param generate_pack_contents: Function that can return a sequence of the
             shas of the objects to upload.
+        :param progress: Optional progress function
 
         :raises SendPackError: if server rejects the pack data
         :raises UpdateRefsError: if the server supports report-status
                                  and rejects ref updates
         """
-        raise NotImplementedError(self.send_pack)
+        url = self._get_url(path)
+        old_refs, server_capabilities = self._discover_references(
+            "git-receive-pack", url)
+        negotiated_capabilities = list(self._send_capabilities)
+        new_refs = determine_wants(old_refs)
+        if new_refs is None:
+            return old_refs
+        if self.dumb:
+            raise NotImplementedError(self.fetch_pack)
+        req_data = StringIO()
+        req_proto = Protocol(None, req_data.write)
+        (have, want) = self._handle_receive_pack_head(
+            req_proto, negotiated_capabilities, old_refs, new_refs)
+        if not want and old_refs == new_refs:
+            return new_refs
+        objects = generate_pack_contents(have, want)
+        if len(objects) > 0:
+            entries, sha = write_pack_objects(req_proto.write_file(), objects)
+        resp = self._smart_request("git-receive-pack", url,
+            data=req_data.getvalue())
+        resp_proto = Protocol(resp.read, None)
+        self._handle_receive_pack_tail(resp_proto, negotiated_capabilities,
+            progress)
+        return new_refs
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-                   progress):
+                   progress=None):
         """Retrieve a pack from a git smart server.
 
         :param determine_wants: Callback that returns list of commits to fetch
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
+        :return: Dictionary with the refs of the remote repository
         """
-        raise NotImplementedError(self.fetch_pack)
+        url = self._get_url(path)
+        refs, server_capabilities = self._discover_references(
+            "git-upload-pack", url)
+        negotiated_capabilities = list(server_capabilities)
+        wants = determine_wants(refs)
+        if wants is not None:
+            wants = [cid for cid in wants if cid != ZERO_SHA]
+        if not wants:
+            return refs
+        if self.dumb:
+            raise NotImplementedError(self.send_pack)
+        req_data = StringIO()
+        req_proto = Protocol(None, req_data.write)
+        self._handle_upload_pack_head(req_proto,
+            negotiated_capabilities, graph_walker, wants,
+            lambda: False)
+        resp = self._smart_request("git-upload-pack", url,
+            data=req_data.getvalue())
+        resp_proto = Protocol(resp.read, None)
+        self._handle_upload_pack_tail(resp_proto, negotiated_capabilities,
+            graph_walker, pack_data, progress)
+        return refs
 
 
 def get_transport_and_path(uri):
@@ -467,6 +776,8 @@ def get_transport_and_path(uri):
     elif parsed.scheme == 'git+ssh':
         return SSHGitClient(parsed.hostname, port=parsed.port,
                             username=parsed.username), parsed.path
+    elif parsed.scheme in ('http', 'https'):
+        return HttpGitClient(urlparse.urlunparse(parsed)), parsed.path
 
     if parsed.scheme and not parsed.netloc:
         # SSH with no user@, zero or one leading slash.

dulwich/config.py

@@ -0,0 +1,343 @@
+# config.py - Reading and writing Git config files
+# Copyright (C) 2011 Jelmer Vernooij <jelmer@samba.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) a later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Reading and writing Git configuration files.
+
+TODO:
+ * preserve formatting when updating configuration files
+ * treat subsection names as case-insensitive for [branch.foo] style
+   subsections
+"""
+
+import errno
+import os
+import re
+
+from dulwich.file import GitFile
+
+
+class Config(object):
+    """A Git configuration."""
+
+    def get(self, section, name):
+        """Retrieve the contents of a configuration setting.
+        
+        :param section: Tuple with section name and optional subsection namee
+        :param subsection: Subsection name
+        :return: Contents of the setting
+        :raise KeyError: if the value is not set
+        """
+        raise NotImplementedError(self.get)
+
+    def get_boolean(self, section, name, default=None):
+        """Retrieve a configuration setting as boolean.
+
+        :param section: Tuple with section name and optional subsection namee
+        :param name: Name of the setting, including section and possible
+            subsection.
+        :return: Contents of the setting
+        :raise KeyError: if the value is not set
+        """
+        try:
+            value = self.get(section, name)
+        except KeyError:
+            return default
+        if value.lower() == "true":
+            return True
+        elif value.lower() == "false":
+            return False
+        raise ValueError("not a valid boolean string: %r" % value)
+
+    def set(self, section, name, value):
+        """Set a configuration value.
+        
+        :param name: Name of the configuration value, including section
+            and optional subsection
+        :param: Value of the setting
+        """
+        raise NotImplementedError(self.set)
+
+
+class ConfigDict(Config):
+    """Git configuration stored in a dictionary."""
+
+    def __init__(self, values=None):
+        """Create a new ConfigDict."""
+        if values is None:
+            values = {}
+        self._values = values
+
+    def __repr__(self):
+        return "%s(%r)" % (self.__class__.__name__, self._values)
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, self.__class__) and
+            other._values == self._values)
+
+    @classmethod
+    def _parse_setting(cls, name):
+        parts = name.split(".")
+        if len(parts) == 3:
+            return (parts[0], parts[1], parts[2])
+        else:
+            return (parts[0], None, parts[1])
+
+    def get(self, section, name):
+        if isinstance(section, basestring):
+            section = (section, )
+        if len(section) > 1:
+            try:
+                return self._values[section][name]
+            except KeyError:
+                pass
+        return self._values[(section[0],)][name]
+
+    def set(self, section, name, value):
+        if isinstance(section, basestring):
+            section = (section, )
+        self._values.setdefault(section, {})[name] = value
+
+
+def _format_string(value):
+    if (value.startswith(" ") or
+        value.startswith("\t") or
+        value.endswith(" ") or
+        value.endswith("\t")):
+        return '"%s"' % _escape_value(value)
+    return _escape_value(value)
+
+
+def _parse_string(value):
+    value = value.strip()
+    ret = []
+    block = []
+    in_quotes  = False
+    for c in value:
+        if c == "\"":
+            in_quotes = (not in_quotes)
+            ret.append(_unescape_value("".join(block)))
+            block = []
+        elif c in ("#", ";") and not in_quotes:
+            # the rest of the line is a comment
+            break
+        else:
+            block.append(c)
+
+    if in_quotes:
+        raise ValueError("value starts with quote but lacks end quote")
+
+    ret.append(_unescape_value("".join(block)).rstrip())
+
+    return "".join(ret)
+
+
+def _unescape_value(value):
+    """Unescape a value."""
+    def unescape(c):
+        return {
+            "\\\\": "\\",
+            "\\\"": "\"",
+            "\\n": "\n",
+            "\\t": "\t",
+            "\\b": "\b",
+            }[c.group(0)]
+    return re.sub(r"(\\.)", unescape, value)
+
+
+def _escape_value(value):
+    """Escape a value."""
+    return value.replace("\\", "\\\\").replace("\n", "\\n").replace("\t", "\\t").replace("\"", "\\\"")
+
+
+def _check_variable_name(name):
+    for c in name:
+        if not c.isalnum() and c != '-':
+            return False
+    return True
+
+
+def _check_section_name(name):
+    for c in name:
+        if not c.isalnum() and c not in ('-', '.'):
+            return False
+    return True
+
+
+def _strip_comments(line):
+    line = line.split("#")[0]
+    line = line.split(";")[0]
+    return line
+
+
+class ConfigFile(ConfigDict):
+    """A Git configuration file, like .git/config or ~/.gitconfig.
+    """
+
+    @classmethod
+    def from_file(cls, f):
+        """Read configuration from a file-like object."""
+        ret = cls()
+        section = None
+        setting = None
+        for lineno, line in enumerate(f.readlines()):
+            line = line.lstrip()
+            if setting is None:
+                if len(line) > 0 and line[0] == "[":
+                    line = _strip_comments(line).rstrip()
+                    last = line.index("]")
+                    if last == -1:
+                        raise ValueError("expected trailing ]")
+                    pts = line[1:last].split(" ", 1)
+                    line = line[last+1:]
+                    pts[0] = pts[0].lower()
+                    if len(pts) == 2:
+                        if pts[1][0] != "\"" or pts[1][-1] != "\"":
+                            raise ValueError(
+                                "Invalid subsection " + pts[1])
+                        else:
+                            pts[1] = pts[1][1:-1]
+                        if not _check_section_name(pts[0]):
+                            raise ValueError("invalid section name %s" %
+                                             pts[0])
+                        section = (pts[0], pts[1])
+                    else:
+                        if not _check_section_name(pts[0]):
+                            raise ValueError("invalid section name %s" %
+                                    pts[0])
+                        pts = pts[0].split(".", 1)
+                        if len(pts) == 2:
+                            section = (pts[0], pts[1])
+                        else:
+                            section = (pts[0], )
+                    ret._values[section] = {}
+                if _strip_comments(line).strip() == "":
+                    continue
+                if section is None:
+                    raise ValueError("setting %r without section" % line)
+                try:
+                    setting, value = line.split("=", 1)
+                except ValueError:
+                    setting = line
+                    value = "true"
+                setting = setting.strip().lower()
+                if not _check_variable_name(setting):
+                    raise ValueError("invalid variable name %s" % setting)
+                if value.endswith("\\\n"):
+                    value = value[:-2]
+                    continuation = True
+                else:
+                    continuation = False
+                value = _parse_string(value)
+                ret._values[section][setting] = value
+                if not continuation:
+                    setting = None
+            else: # continuation line
+                if line.endswith("\\\n"):
+                    line = line[:-2]
+                    continuation = True
+                else:
+                    continuation = False
+                value = _parse_string(line)
+                ret._values[section][setting] += value
+                if not continuation:
+                    setting = None
+        return ret
+
+    @classmethod
+    def from_path(cls, path):
+        """Read configuration from a file on disk."""
+        f = GitFile(path, 'rb')
+        try:
+            ret = cls.from_file(f)
+            ret.path = path
+            return ret
+        finally:
+            f.close()
+
+    def write_to_path(self, path=None):
+        """Write configuration to a file on disk."""
+        if path is None:
+            path = self.path
+        f = GitFile(path, 'wb')
+        try:
+            self.write_to_file(f)
+        finally:
+            f.close()
+
+    def write_to_file(self, f):
+        """Write configuration to a file-like object."""
+        for section, values in self._values.iteritems():
+            try:
+                section_name, subsection_name = section
+            except ValueError:
+                (section_name, ) = section
+                subsection_name = None
+            if subsection_name is None:
+                f.write("[%s]\n" % section_name)
+            else:
+                f.write("[%s \"%s\"]\n" % (section_name, subsection_name))
+            for key, value in values.iteritems():
+                f.write("%s = %s\n" % (key, _escape_value(value)))
+
+
+class StackedConfig(Config):
+    """Configuration which reads from multiple config files.."""
+
+    def __init__(self, backends, writable=None):
+        self.backends = backends
+        self.writable = writable
+
+    def __repr__(self):
+        return "<%s for %r>" % (self.__class__.__name__, self.backends)
+
+    @classmethod
+    def default_backends(cls):
+        """Retrieve the default configuration.
+
+        This will look in the repository configuration (if for_path is
+        specified), the users' home directory and the system
+        configuration.
+        """
+        paths = []
+        paths.append(os.path.expanduser("~/.gitconfig"))
+        paths.append("/etc/gitconfig")
+        backends = []
+        for path in paths:
+            try:
+                cf = ConfigFile.from_path(path)
+            except (IOError, OSError), e:
+                if e.errno != errno.ENOENT:
+                    raise
+                else:
+                    continue
+            backends.append(cf)
+        return backends
+
+    def get(self, section, name):
+        for backend in self.backends:
+            try:
+                return backend.get(section, name)
+            except KeyError:
+                pass
+        raise KeyError(name)
+
+    def set(self, section, name, value):
+        if self.writable is None:
+            raise NotImplementedError(self.set)
+        return self.writable.set(section, name, value)

dulwich/index.py

@@ -193,6 +193,9 @@ class Index(object):
         self.clear()
         self.read()
 
+    def __repr__(self):
+        return "%s(%r)" % (self.__class__.__name__, self._filename)
+
     def write(self):
         """Write current contents of index to disk."""
         f = GitFile(self._filename, 'wb')
@@ -275,19 +278,13 @@ class Index(object):
         :param want_unchanged: Whether unchanged files should be reported
         :return: Iterator over tuples with (oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
         """
-        mine = set(self._byname.keys())
-        for (name, mode, sha) in object_store.iter_tree_contents(tree):
-            if name in mine:
-                if (want_unchanged or self.get_sha1(name) != sha or 
-                    self.get_mode(name) != mode):
-                    yield ((name, name), (mode, self.get_mode(name)), (sha, self.get_sha1(name)))
-                mine.remove(name)
-            else:
-                # Was removed
-                yield ((name, None), (mode, None), (sha, None))
-        # Mention added files
-        for name in mine:
-            yield ((None, name), (None, self.get_mode(name)), (None, self.get_sha1(name)))
+        def lookup_entry(path):
+            entry = self[path]
+            return entry[-2], entry[-6]
+        for (name, mode, sha) in changes_from_tree(self._byname.keys(),
+                lookup_entry, object_store, tree,
+                want_unchanged=want_unchanged):
+            yield (name, mode, sha)
 
     def commit(self, object_store):
         """Create a new tree from an index.
@@ -345,3 +342,48 @@ def commit_index(object_store, index):
     :return: Root tree sha.
     """
     return commit_tree(object_store, index.iterblobs())
+
+
+def changes_from_tree(names, lookup_entry, object_store, tree,
+        want_unchanged=False):
+    """Find the differences between the contents of a tree and
+    a working copy.
+
+    :param names: Iterable of names in the working copy
+    :param lookup_entry: Function to lookup an entry in the working copy
+    :param object_store: Object store to use for retrieving tree contents
+    :param tree: SHA1 of the root tree
+    :param want_unchanged: Whether unchanged files should be reported
+    :return: Iterator over tuples with (oldpath, newpath), (oldmode, newmode),
+        (oldsha, newsha)
+    """
+    other_names = set(names)
+    for (name, mode, sha) in object_store.iter_tree_contents(tree):
+        try:
+            (other_sha, other_mode) = lookup_entry(name)
+        except KeyError:
+            # Was removed
+            yield ((name, None), (mode, None), (sha, None))
+        else:
+            other_names.remove(name)
+            if (want_unchanged or other_sha != sha or other_mode != mode):
+                yield ((name, name), (mode, other_mode), (sha, other_sha))
+
+    # Mention added files
+    for name in other_names:
+        (other_sha, other_mode) = lookup_entry(name)
+        yield ((None, name), (None, other_mode), (None, other_sha))
+
+
+def index_entry_from_stat(stat_val, hex_sha, flags, mode=None):
+    """Create a new index entry from a stat value.
+
+    :param stat_val: POSIX stat_result instance
+    :param hex_sha: Hex sha of the object
+    :param flags: Index flags
+    """
+    if mode is None:
+        mode = cleanup_mode(stat_val.st_mode)
+    return (stat_val.st_ctime, stat_val.st_mtime, stat_val.st_dev,
+            stat_val.st_ino, mode, stat_val.st_uid,
+            stat_val.st_gid, stat_val.st_size, hex_sha, flags)

dulwich/object_store.py

@@ -132,8 +132,8 @@ class BaseObjectStore(object):
     def tree_changes(self, source, target, want_unchanged=False):
         """Find the differences between the contents of two trees
 
-        :param object_store: Object store to use for retrieving tree contents
-        :param tree: SHA1 of the root tree
+        :param source: SHA1 of the source tree
+        :param target: SHA1 of the target tree
         :param want_unchanged: Whether unchanged files should be reported
         :return: Iterator over tuples with
             (oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
@@ -226,6 +226,10 @@ class PackBasedObjectStore(BaseObjectStore):
     def __init__(self):
         self._pack_cache = None
 
+    @property
+    def alternates(self):
+        return []
+
     def contains_packed(self, sha):
         """Check if a particular object is present by SHA1 and is packed."""
         for pack in self.packs:
@@ -310,6 +314,11 @@ class PackBasedObjectStore(BaseObjectStore):
         ret = self._get_loose_object(hexsha)
         if ret is not None:
             return ret.type_num, ret.as_raw_string()
+        for alternate in self.alternates:
+            try:
+                return alternate.get_raw(hexsha)
+            except KeyError:
+                pass
         raise KeyError(hexsha)
 
     def add_objects(self, objects):
@@ -338,6 +347,63 @@ class DiskObjectStore(PackBasedObjectStore):
         self.path = path
         self.pack_dir = os.path.join(self.path, PACKDIR)
         self._pack_cache_time = 0
+        self._alternates = None
+
+    @property
+    def alternates(self):
+        if self._alternates is not None:
+            return self._alternates
+        self._alternates = []
+        for path in self._read_alternate_paths():
+            self._alternates.append(DiskObjectStore(path))
+        return self._alternates
+
+    def _read_alternate_paths(self):
+        try:
+            f = GitFile(os.path.join(self.path, "info", "alternates"),
+                    'rb')
+        except (OSError, IOError), e:
+            if e.errno == errno.ENOENT:
+                return []
+            raise
+        ret = []
+        try:
+            for l in f.readlines():
+                l = l.rstrip("\n")
+                if l[0] == "#":
+                    continue
+                if not os.path.isabs(l):
+                    continue
+                ret.append(l)
+            return ret
+        finally:
+            f.close()
+
+    def add_alternate_path(self, path):
+        """Add an alternate path to this object store.
+        """
+        try:
+            os.mkdir(os.path.join(self.path, "info"))
+        except OSError, e:
+            if e.errno != errno.EEXIST:
+                raise
+        alternates_path = os.path.join(self.path, "info/alternates")
+        f = GitFile(alternates_path, 'wb')
+        try:
+            try:
+                orig_f = open(alternates_path, 'rb')
+            except (OSError, IOError), e:
+                if e.errno != errno.ENOENT:
+                    raise
+            else:
+                try:
+                    f.write(orig_f.read())
+                finally:
+                    orig_f.close()
+            f.write("%s\n" % path)
+        finally:
+            f.close()
+        self.alternates.append(DiskObjectStore(path))
 
     def _load_packs(self):
         pack_files = []
@@ -405,11 +471,18 @@ class DiskObjectStore(PackBasedObjectStore):
         f.seek(0)
         write_pack_header(f, len(entries) + len(indexer.ext_refs()))
 
+        # Must flush before reading (http://bugs.python.org/issue3207)
+        f.flush()
+
         # Rescan the rest of the pack, computing the SHA with the new header.
         new_sha = compute_file_sha(f, end_ofs=-20)
 
+        # Must reposition before writing (http://bugs.python.org/issue3207)
+        f.seek(0, os.SEEK_CUR)
+
         # Complete the pack.
         for ext_sha in indexer.ext_refs():
+            assert len(ext_sha) == 20
             type_num, data = self.get_raw(ext_sha)
             offset = f.tell()
             crc32 = write_pack_object(f, type_num, data, sha=new_sha)
@@ -544,9 +617,17 @@ class MemoryObjectStore(BaseObjectStore):
         super(MemoryObjectStore, self).__init__()
         self._data = {}
 
+    def _to_hexsha(self, sha):
+        if len(sha) == 40:
+            return sha
+        elif len(sha) == 20:
+            return sha_to_hex(sha)
+        else:
+            raise ValueError("Invalid sha %r" % sha)
+
     def contains_loose(self, sha):
         """Check if a particular object is present by SHA1 and is loose."""
-        return sha in self._data
+        return self._to_hexsha(sha) in self._data
 
     def contains_packed(self, sha):
         """Check if a particular object is present by SHA1 and is packed."""
@@ -567,15 +648,15 @@ class MemoryObjectStore(BaseObjectStore):
         :param name: sha for the object.
         :return: tuple with numeric type and object contents.
         """
-        obj = self[name]
+        obj = self[self._to_hexsha(name)]
         return obj.type_num, obj.as_raw_string()
 
     def __getitem__(self, name):
-        return self._data[name]
+        return self._data[self._to_hexsha(name)]
 
     def __delitem__(self, name):
         """Delete an object from this store, for testing only."""
-        del self._data[name]
+        del self._data[self._to_hexsha(name)]
 
     def add_object(self, obj):
         """Add a single object to this object store.

dulwich/objects.py

@@ -19,7 +19,6 @@
 
 """Access to base git objects."""
 
-
 import binascii
 from cStringIO import (
     StringIO,
@@ -64,6 +63,11 @@ _TAGGER_HEADER = "tagger"
 S_IFGITLINK = 0160000
 
 def S_ISGITLINK(m):
+    """Check if a mode indicates a submodule.
+
+    :param m: Mode to check
+    :return: a `boolean`
+    """
     return (stat.S_IFMT(m) == S_IFGITLINK)
 
 
@@ -114,6 +118,8 @@ def object_header(num_type, length):
 
 
 def serializable_property(name, docstring=None):
+    """A property that helps tracking whether serialization is necessary.
+    """
     def set(obj, value):
         obj._ensure_parsed()
         setattr(obj, "_"+name, value)
@@ -135,6 +141,12 @@ def object_class(type):
 
 
 def check_hexsha(hex, error_msg):
+    """Check if a string is a valid hex sha string.
+
+    :param hex: Hex string to check
+    :param error_msg: Error message to use in exception
+    :raise ObjectFormatException: Raised when the string is not valid
+    """
     try:
         hex_to_sha(hex)
     except (TypeError, AssertionError):
@@ -168,9 +180,11 @@ class FixedSha(object):
         self._sha = hex_to_sha(hexsha)
 
     def digest(self):
+        """Return the raw SHA digest."""
         return self._sha
 
     def hexdigest(self):
+        """Return the hex SHA digest."""
         return self._hexsha
 
 
@@ -213,6 +227,10 @@ class ShaFile(object):
         self.set_raw_string(text[header_end+1:])
 
     def as_legacy_object_chunks(self):
+        """Return chunks representing the object in the experimental format.
+
+        :return: List of strings
+        """
         compobj = zlib.compressobj()
         yield compobj.compress(self._header())
         for chunk in self.as_raw_chunks():
@@ -220,9 +238,15 @@ class ShaFile(object):
         yield compobj.flush()
 
     def as_legacy_object(self):
+        """Return string representing the object in the experimental format.
+        """
         return "".join(self.as_legacy_object_chunks())
 
     def as_raw_chunks(self):
+        """Return chunks with serialization of the object.
+
+        :return: List of strings, not necessarily one per line
+        """
         if self._needs_parsing:
             self._ensure_parsed()
         elif self._needs_serialization:
@@ -230,15 +254,22 @@ class ShaFile(object):
         return self._chunked_text
 
     def as_raw_string(self):
+        """Return raw string with serialization of the object.
+
+        :return: String object
+        """
         return "".join(self.as_raw_chunks())
 
     def __str__(self):
+        """Return raw string serialization of this object."""
         return self.as_raw_string()
 
     def __hash__(self):
+        """Return unique hash for this object."""
         return hash(self.id)
 
     def as_pretty_string(self):
+        """Return a string representing this object, fit for display."""
         return self.as_raw_string()
 
     def _ensure_parsed(self):
@@ -256,11 +287,13 @@ class ShaFile(object):
             self._needs_parsing = False
 
     def set_raw_string(self, text):
+        """Set the contents of this object from a serialized string."""
         if type(text) != str:
             raise TypeError(text)
         self.set_raw_chunks([text])
 
     def set_raw_chunks(self, chunks):
+        """Set the contents of this object from a list of chunks."""
         self._chunked_text = chunks
         self._deserialize(chunks)
         self._sha = None
@@ -294,7 +327,7 @@ class ShaFile(object):
     def _is_legacy_object(cls, magic):
         b0, b1 = map(ord, magic)
         word = (b0 << 8) + b1
-        return b0 == 0x78 and (word % 31) == 0
+        return (b0 & 0x8F) == 0x08 and (word % 31) == 0
 
     @classmethod
     def _parse_file_header(cls, f):
@@ -339,6 +372,7 @@ class ShaFile(object):
 
     @classmethod
     def from_path(cls, path):
+        """Open a SHA file from disk."""
         f = GitFile(path, 'rb')
         try:
             obj = cls.from_file(f)
@@ -454,12 +488,15 @@ class ShaFile(object):
 
     @property
     def id(self):
+        """The hex SHA of this object."""
         return self.sha().hexdigest()
 
     def get_type(self):
+        """Return the type number for this object class."""
         return self.type_num
 
     def set_type(self, type):
+        """Set the type number for this object class."""
         self.type_num = type
 
     # DEPRECATED: use type_num or type_name as needed.
@@ -557,6 +594,7 @@ def _parse_tag_or_commit(text):
 
 
 def parse_tag(text):
+    """Parse a tag object."""
     return _parse_tag_or_commit(text)
 
 
@@ -693,6 +731,8 @@ class TreeEntry(namedtuple('TreeEntry', ['path', 'mode', 'sha'])):
 
     def in_path(self, path):
         """Return a copy of this entry with the given path prepended."""
+        if type(self.path) != str:
+            raise TypeError
         return TreeEntry(posixpath.join(path, self.path), self.mode, self.sha)
 
 
@@ -747,6 +787,8 @@ def sorted_tree_items(entries, name_order):
     for name, entry in sorted(entries.iteritems(), cmp=cmp_func):
         mode, hexsha = entry
         # Stricter type checks than normal to mirror checks in the C version.
+        if not isinstance(mode, int) and not isinstance(mode, long):
+            raise TypeError('Expected integer/long for mode, got %r' % mode)
         mode = int(mode)
         if not isinstance(hexsha, str):
             raise TypeError('Expected a string for SHA, got %r' % hexsha)

dulwich/pack.py

@@ -605,9 +605,11 @@ class PackIndex2(FilePackIndex):
 
     def __init__(self, filename, file=None, contents=None, size=None):
         super(PackIndex2, self).__init__(filename, file, contents, size)
-        assert self._contents[:4] == '\377tOc', 'Not a v2 pack index file'
+        if self._contents[:4] != '\377tOc':
+            raise AssertionError('Not a v2 pack index file')
         (self.version, ) = unpack_from('>L', self._contents, 4)
-        assert self.version == 2, 'Version was %d' % self.version
+        if self.version != 2:
+            raise AssertionError('Version was %d' % self.version)
         self._fan_out_table = self._read_fan_out_table(8)
         self._name_table_offset = 8 + 0x100 * 4
         self._crc32_table_offset = self._name_table_offset + 20 * len(self)
@@ -641,9 +643,11 @@ def read_pack_header(read):
     header = read(12)
     if not header:
         return None, None
-    assert header[:4] == 'PACK'
+    if header[:4] != 'PACK':
+        raise AssertionError('Invalid pack header %r' % header)
     (version,) = unpack_from('>L', header, 4)
-    assert version in (2, 3), 'Version was %d' % version
+    if version not in (2, 3):
+        raise AssertionError('Version was %d' % version)
     (num_objects,) = unpack_from('>L', header, 8)
     return (version, num_objects)
 
@@ -693,7 +697,8 @@ def unpack_object(read_all, read_some=None, compute_crc32=False,
     if type_num == OFS_DELTA:
         bytes, crc32 = take_msb_bytes(read_all, crc32=crc32)
         raw_base += len(bytes)
-        assert not (bytes[-1] & 0x80)
+        if bytes[-1] & 0x80:
+            raise AssertionError
         delta_base_offset = bytes[0] & 0x7f
         for byte in bytes[1:]:
             delta_base_offset += 1

dulwich/protocol.py

@@ -406,3 +406,36 @@ class BufferedPktLineWriter(object):
             self._write(data)
         self._len = 0
         self._wbuf = StringIO()
+
+
+class PktLineParser(object):
+    """Packet line parser that hands completed packets off to a callback.
+    """
+
+    def __init__(self, handle_pkt):
+        self.handle_pkt = handle_pkt
+        self._readahead = StringIO()
+
+    def parse(self, data):
+        """Parse a fragment of data and call back for any completed packets.
+        """
+        self._readahead.write(data)
+        buf = self._readahead.getvalue()
+        if len(buf) < 4:
+            return
+        while len(buf) >= 4:
+            size = int(buf[:4], 16)
+            if size == 0:
+                self.handle_pkt(None)
+                buf = buf[4:]
+            elif size <= len(buf):
+                self.handle_pkt(buf[4:size])
+                buf = buf[size:]
+            else:
+                break
+        self._readahead = StringIO()
+        self._readahead.write(buf)
+
+    def get_tail(self):
+        """Read back any unused data."""
+        return self._readahead.getvalue()

dulwich/repo.py

@@ -19,7 +19,13 @@
 # MA  02110-1301, USA.
 
 
-"""Repository access."""
+"""Repository access.
+
+This module contains the base class for git repositories
+(BaseRepo) and an implementation which uses a repository on 
+local disk (Repo).
+
+"""
 
 from cStringIO import StringIO
 import errno
@@ -52,9 +58,6 @@ from dulwich.objects import (
     Tree,
     hex_to_sha,
     )
-from dulwich.walk import (
-    Walker,
-    )
 import warnings
 
 
@@ -213,7 +216,7 @@ class RefsContainer(object):
         :param name: The name of the reference.
         :raises KeyError: if a refname is not HEAD or is otherwise not valid.
         """
-        if name == 'HEAD':
+        if name in ('HEAD', 'refs/stash'):
             return
         if not name.startswith('refs/') or not check_ref_format(name[5:]):
             raise RefFormatError(name)
@@ -391,6 +394,40 @@ class DictRefsContainer(RefsContainer):
         self._peeled.update(peeled)
 
 
+class InfoRefsContainer(RefsContainer):
+    """Refs container that reads refs from a info/refs file."""
+
+    def __init__(self, f):
+        self._refs = {}
+        self._peeled = {}
+        for l in f.readlines():
+            sha, name = l.rstrip("\n").split("\t")
+            if name.endswith("^{}"):
+                name = name[:-3]
+                if not check_ref_format(name):
+                    raise ValueError("invalid ref name '%s'" % name)
+                self._peeled[name] = sha
+            else:
+                if not check_ref_format(name):
+                    raise ValueError("invalid ref name '%s'" % name)
+                self._refs[name] = sha
+
+    def allkeys(self):
+        return self._refs.keys()
+
+    def read_loose_ref(self, name):
+        return self._refs.get(name, None)
+
+    def get_packed_refs(self):
+        return {}
+
+    def get_peeled(self, name):
+        try:
+            return self._peeled[name]
+        except KeyError:
+            return self._refs[name]
+
+
 class DiskRefsContainer(RefsContainer):
     """Refs container that reads refs from disk."""
 
@@ -760,21 +797,34 @@ class BaseRepo(object):
 
     :ivar object_store: Dictionary-like object for accessing
         the objects
-    :ivar refs: Dictionary-like object with the refs in this repository
+    :ivar refs: Dictionary-like object with the refs in this
+        repository
     """
 
     def __init__(self, object_store, refs):
+        """Open a repository.
+
+        This shouldn't be called directly, but rather through one of the
+        base classes, such as MemoryRepo or Repo.
+
+        :param object_store: Object store to use
+        :param refs: Refs container to use
+        """
         self.object_store = object_store
         self.refs = refs
 
     def _init_files(self, bare):
         """Initialize a default set of named files."""
+        from dulwich.config import ConfigFile
         self._put_named_file('description', "Unnamed repository")
-        self._put_named_file('config', ('[core]\n'
-                                        'repositoryformatversion = 0\n'
-                                        'filemode = true\n'
-                                        'bare = ' + str(bare).lower() + '\n'
-                                        'logallrefupdates = true\n'))
+        f = StringIO()
+        cf = ConfigFile()
+        cf.set("core", "repositoryformatversion", "0")
+        cf.set("core", "filemode", "true")
+        cf.set("core", "bare", str(bare).lower())
+        cf.set("core", "logallrefupdates", "true")
+        cf.write_to_file(f)
+        self._put_named_file('config', f.getvalue())
         self._put_named_file(os.path.join('info', 'exclude'), '')
 
     def get_named_file(self, path):
@@ -846,6 +896,14 @@ class BaseRepo(object):
                                                  get_tagged))
 
     def get_graph_walker(self, heads=None):
+        """Retrieve a graph walker.
+
+        A graph walker is used by a remote repository (or proxy)
+        to find out which objects are present in this repository.
+
+        :param heads: Repository heads to use (optional)
+        :return: A graph walker object
+        """
         if heads is None:
             heads = self.refs.as_dict('refs/heads').values()
         return self.object_store.get_graph_walker(heads)
@@ -880,17 +938,50 @@ class BaseRepo(object):
         return ret
 
     def get_object(self, sha):
+        """Retrieve the object with the specified SHA.
+
+        :param sha: SHA to retrieve
+        :return: A ShaFile object
+        :raise KeyError: when the object can not be found
+        """
         return self.object_store[sha]
 
     def get_parents(self, sha):
+        """Retrieve the parents of a specific commit.
+
+        :param sha: SHA of the commit for which to retrieve the parents
+        :return: List of parents
+        """
         return self.commit(sha).parents
 
     def get_config(self):
-        import ConfigParser
-        p = ConfigParser.RawConfigParser()
-        p.read(os.path.join(self._controldir, 'config'))
-        return dict((section, dict(p.items(section)))
-                    for section in p.sections())
+        """Retrieve the config object.
+
+        :return: `ConfigFile` object for the ``.git/config`` file.
+        """
+        from dulwich.config import ConfigFile
+        path = os.path.join(self._controldir, 'config')
+        try:
+            return ConfigFile.from_path(path)
+        except (IOError, OSError), e:
+            if e.errno != errno.ENOENT:
+                raise
+            ret = ConfigFile()
+            ret.path = path
+            return ret
+
+    def get_config_stack(self):
+        """Return a config stack for this repository.
+
+        This stack accesses the configuration for both this repository
+        itself (.git/config) and the global configuration, which usually
+        lives in ~/.gitconfig.
+
+        :return: `Config` instance for this repository
+        """
+        from dulwich.config import StackedConfig
+        backends = [self.get_config()] + StackedConfig.default_backends()
+        return StackedConfig(backends, writable=backends[0])
 
     def commit(self, sha):
         """Retrieve the commit with a particular SHA.
@@ -953,6 +1044,35 @@ class BaseRepo(object):
             return cached
         return self.object_store.peel_sha(self.refs[ref]).id
 
+    def get_walker(self, include=None, *args, **kwargs):
+        """Obtain a walker for this repository.
+
+        :param include: Iterable of SHAs of commits to include along with their
+            ancestors. Defaults to [HEAD]
+        :param exclude: Iterable of SHAs of commits to exclude along with their
+            ancestors, overriding includes.
+        :param order: ORDER_* constant specifying the order of results. Anything
+            other than ORDER_DATE may result in O(n) memory usage.
+        :param reverse: If True, reverse the order of output, requiring O(n)
+            memory.
+        :param max_entries: The maximum number of entries to yield, or None for
+            no limit.
+        :param paths: Iterable of file or subtree paths to show entries for.
+        :param rename_detector: diff.RenameDetector object for detecting
+            renames.
+        :param follow: If True, follow path across renames/copies. Forces a
+            default rename_detector.
+        :param since: Timestamp to list commits after.
+        :param until: Timestamp to list commits before.
+        :param queue_cls: A class to use for a queue of commits, supporting the
+            iterator protocol. The constructor takes a single argument, the
+            Walker.
+        """
+        from dulwich.walk import Walker
+        if include is None:
+            include = [self.head()]
+        return Walker(self.object_store, include, *args, **kwargs)
+
     def revision_history(self, head):
         """Returns a list of the commits reachable from head.
 
@@ -962,11 +1082,18 @@ class BaseRepo(object):
         :raise MissingCommitError: if any missing commits are referenced,
             including if the head parameter isn't the SHA of a commit.
         """
-        # TODO(dborowitz): Expose more of the Walker functionality here or in a
-        # separate Repo/BaseObjectStore method.
-        return [e.commit for e in Walker(self.object_store, [head])]
+        warnings.warn("Repo.revision_history() is deprecated."
+            "Use dulwich.walker.Walker(repo) instead.",
+            category=DeprecationWarning, stacklevel=2)
+        return [e.commit for e in self.get_walker(include=[head])]
 
     def __getitem__(self, name):
+        """Retrieve a Git object by SHA1 or ref.
+
+        :param name: A Git object SHA1 or a ref name
+        :return: A `ShaFile` object, such as a Commit or Blob
+        :raise KeyError: when the specified ref or object does not exist
+        """
         if len(name) in (20, 40):
             try:
                 return self.object_store[name]
@@ -978,12 +1105,21 @@ class BaseRepo(object):
             raise KeyError(name)
 
     def __contains__(self, name):
+        """Check if a specific Git object or ref is present.
+
+        :param name: Git object SHA1 or ref name
+        """
         if len(name) in (20, 40):
             return name in self.object_store or name in self.refs
         else:
             return name in self.refs
 
     def __setitem__(self, name, value):
+        """Set a ref.
+
+        :param name: ref name
+        :param value: Ref value - either a ShaFile object, or a hex sha
+        """
         if name.startswith("refs/") or name == "HEAD":
             if isinstance(value, ShaFile):
                 self.refs[name] = value.id
@@ -995,11 +1131,21 @@ class BaseRepo(object):
             raise ValueError(name)
 
     def __delitem__(self, name):
-        if name.startswith("refs") or name == "HEAD":
+        """Remove a ref.
+
+        :param name: Name of the ref to remove
+        """
+        if name.startswith("refs/") or name == "HEAD":
             del self.refs[name]
         else:
             raise ValueError(name)
 
+    def _get_user_identity(self):
+        config = self.get_config_stack()
+        return "%s <%s>" % (
+            config.get(("user", ), "name"),
+            config.get(("user", ), "email"))
+
     def do_commit(self, message=None, committer=None,
                   author=None, commit_timestamp=None,
                   commit_timezone=None, author_timestamp=None,
@@ -1034,9 +1180,8 @@ class BaseRepo(object):
         if merge_heads is None:
             # FIXME: Read merge heads from .git/MERGE_HEADS
             merge_heads = []
-        # TODO: Allow username to be missing, and get it from .git/config
         if committer is None:
-            raise ValueError("committer not set")
+            committer = self._get_user_identity()
         c.committer = committer
         if commit_timestamp is None:
             commit_timestamp = time.time()
@@ -1078,7 +1223,13 @@ class BaseRepo(object):
 
 
 class Repo(BaseRepo):
-    """A git repository backed by local disk."""
+    """A git repository backed by local disk.
+
+    To open an existing repository, call the contructor with
+    the path of the repository.
+
+    To create a new repository, use the Repo.init class method.
+    """
 
     def __init__(self, root):
         if os.path.isdir(os.path.join(root, ".git", OBJECTDIR)):
@@ -1155,11 +1306,12 @@ class Repo(BaseRepo):
 
         :param paths: List of paths, relative to the repository path
         """
-        from dulwich.index import cleanup_mode
+        if isinstance(paths, basestring):
+            paths = [paths]
+        from dulwich.index import index_entry_from_stat
         index = self.open_index()
         for path in paths:
             full_path = os.path.join(self.path, path)
-            blob = Blob()
             try:
                 st = os.stat(full_path)
             except OSError:
@@ -1167,21 +1319,20 @@ class Repo(BaseRepo):
                 try:
                     del index[path]
                 except KeyError:
-                    pass  # Doesn't exist in the index either
+                    pass # already removed
             else:
+                blob = Blob()
                 f = open(full_path, 'rb')
                 try:
                     blob.data = f.read()
                 finally:
                     f.close()
                 self.object_store.add_object(blob)
-                # XXX: Cleanup some of the other file properties as well?
-                index[path] = (st.st_ctime, st.st_mtime, st.st_dev, st.st_ino,
-                    cleanup_mode(st.st_mode), st.st_uid, st.st_gid, st.st_size,
-                    blob.id, 0)
+                index[path] = index_entry_from_stat(st, blob.id, 0)
         index.write()
 
-    def clone(self, target_path, mkdir=True, bare=False, origin="origin"):
+    def clone(self, target_path, mkdir=True, bare=False,
+            origin="origin"):
         """Clone this repository.
 
         :param target_path: Target path
@@ -1221,6 +1372,12 @@ class Repo(BaseRepo):
 
     @classmethod
     def init(cls, path, mkdir=False):
+        """Create a new repository.
+
+        :param path: Path in which to create the repository
+        :param mkdir: Whether to create the directory
+        :return: `Repo` instance
+        """
         if mkdir:
             os.mkdir(path)
         controldir = os.path.join(path, ".git")
@@ -1230,6 +1387,13 @@ class Repo(BaseRepo):
 
     @classmethod
     def init_bare(cls, path):
+        """Create a new bare repository.
+
+        ``path`` should already exist and be an emty directory.
+
+        :param path: Path to create bare repository in
+        :return: a `Repo` instance
+        """
         return cls._init_maybe_bare(path, True)
 
     create = init_bare
@@ -1276,6 +1440,13 @@ class MemoryRepo(BaseRepo):
 
     @classmethod
     def init_bare(cls, objects, refs):
+        """Create a new bare repository in memory.
+
+        :param objects: Objects for the new repository,
+            as iterable
+        :param refs: Refs as dictionary, mapping names
+            to object SHA1s
+        """
         ret = cls()
         for obj in objects:
             ret.object_store.add_object(obj)

dulwich/server.py

@@ -23,10 +23,25 @@ Documentation/technical directory in the cgit distribution, and in particular:
 
 * Documentation/technical/protocol-capabilities.txt
 * Documentation/technical/pack-protocol.txt
-"""
 
+Currently supported capabilities:
+
+ * include-tag
+ * thin-pack
+ * multi_ack_detailed
+ * multi_ack
+ * side-band-64k
+ * ofs-delta
+ * no-progress
+ * report-status
+ * delete-refs
+
+Known capabilities that are not supported:
+ * shallow (http://pad.lv/909524)
+"""
 
 import collections
+import os
 import socket
 import SocketServer
 import sys
@@ -36,6 +51,7 @@ from dulwich.errors import (
     ApplyDeltaError,
     ChecksumMismatch,
     GitProtocolError,
+    NotGitRepository,
     UnexpectedCommandError,
     ObjectFormatException,
     )
@@ -72,14 +88,19 @@ class Backend(object):
     """A backend for the Git smart server implementation."""
 
     def open_repository(self, path):
-        """Open the repository at a path."""
+        """Open the repository at a path.
+
+        :param path: Path to the repository
+        :raise NotGitRepository: no git repository was found at path
+        :return: Instance of BackendRepo
+        """
         raise NotImplementedError(self.open_repository)
 
 
 class BackendRepo(object):
     """Repository abstraction used by the Git server.
-    
-    Please note that the methods required here are a 
+
+    Please note that the methods required here are a
     subset of those provided by dulwich.repo.Repo.
     """
 
@@ -125,8 +146,11 @@ class DictBackend(Backend):
 
     def open_repository(self, path):
         logger.debug('Opening repository at %s', path)
-        # FIXME: What to do in case there is no repo ?
-        return self.repos[path]
+        try:
+            return self.repos[path]
+        except KeyError:
+            raise NotGitRepository("No git repository was found at %(path)s",
+                path=path)
 
 
 class FileSystemBackend(Backend):
@@ -778,3 +802,40 @@ def serve_command(handler_cls, argv=sys.argv, backend=None, inf=sys.stdin,
     # FIXME: Catch exceptions and write a single-line summary to outf.
     handler.handle()
     return 0
+
+
+def generate_info_refs(repo):
+    """Generate an info refs file."""
+    refs = repo.get_refs()
+    for name in sorted(refs.iterkeys()):
+        # get_refs() includes HEAD as a special case, but we don't want to
+        # advertise it
+        if name == 'HEAD':
+            continue
+        sha = refs[name]
+        o = repo.object_store[sha]
+        if not o:
+            continue
+        yield '%s\t%s\n' % (sha, name)
+        peeled_sha = repo.get_peeled(name)
+        if peeled_sha != sha:
+            yield '%s\t%s^{}\n' % (peeled_sha, name)
+
+
+def generate_objects_info_packs(repo):
+    """Generate an index for for packs."""
+    for pack in repo.object_store.packs:
+        yield 'P pack-%s.pack\n' % pack.name()
+
+
+def update_server_info(repo):
+    """Generate server info for dumb file access.
+
+    This generates info/refs and objects/info/packs,
+    similar to "git update-server-info".
+    """
+    repo._put_named_file(os.path.join('info', 'refs'),
+        "".join(generate_info_refs(repo)))
+
+    repo._put_named_file(os.path.join('objects', 'info', 'packs'),
+        "".join(generate_objects_info_packs(repo)))

dulwich/tests/__init__.py

@@ -68,6 +68,7 @@ class BlackboxTestCase(TestCase):
         return subprocess.Popen(argv,
             stdout=subprocess.PIPE,
             stdin=subprocess.PIPE, stderr=subprocess.PIPE,
+            universal_newlines=True,
             env=env)
 
 
@@ -75,6 +76,7 @@ def self_test_suite():
     names = [
         'blackbox',
         'client',
+        'config',
         'diff_tree',
         'fastexport',
         'file',
@@ -100,13 +102,16 @@ def tutorial_test_suite():
         'introduction',
         'repo',
         'object-store',
+        'remote',
         'conclusion',
         ]
     tutorial_files = ["../../docs/tutorial/%s.txt" % name for name in tutorial]
     def setup(test):
+        test.__old_cwd = os.getcwd()
         test.__dulwich_tempdir = tempfile.mkdtemp()
         os.chdir(test.__dulwich_tempdir)
     def teardown(test):
+        os.chdir(test.__old_cwd)
         shutil.rmtree(test.__dulwich_tempdir)
     return doctest.DocFileSuite(setUp=setup, tearDown=teardown,
         *tutorial_files)

dulwich/tests/compat/test_client.py

@@ -19,6 +19,7 @@
 
 """Compatibilty tests between the Dulwich client and the cgit server."""
 
+from cStringIO import StringIO
 import BaseHTTPServer
 import SimpleHTTPServer
 import copy
@@ -27,6 +28,7 @@ import select
 import shutil
 import signal
 import subprocess
+import tarfile
 import tempfile
 import threading
 import urllib
@@ -60,9 +62,9 @@ class DulwichClientTestBase(object):
 
     def setUp(self):
         self.gitroot = os.path.dirname(import_repo_to_dir('server_new.export'))
-        dest = os.path.join(self.gitroot, 'dest')
-        file.ensure_dir_exists(dest)
-        run_git_or_fail(['init', '--quiet', '--bare'], cwd=dest)
+        self.dest = os.path.join(self.gitroot, 'dest')
+        file.ensure_dir_exists(self.dest)
+        run_git_or_fail(['init', '--quiet', '--bare'], cwd=self.dest)
 
     def tearDown(self):
         shutil.rmtree(self.gitroot)
@@ -108,11 +110,7 @@ class DulwichClientTestBase(object):
                     src.object_store.generate_pack_contents)
         self.assertDestEqualsSrc()
 
-    def disable_ff_and_make_dummy_commit(self):
-        # disable non-fast-forward pushes to the server
-        dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
-        run_git_or_fail(['config', 'receive.denyNonFastForwards', 'true'],
-                        cwd=dest.path)
+    def make_dummy_commit(self, dest):
         b = objects.Blob.from_string('hi')
         dest.object_store.add_object(b)
         t = index.commit_tree(dest.object_store, [('hi', b.id, 0100644)])
@@ -123,7 +121,15 @@ class DulwichClientTestBase(object):
         c.message = 'hi'
         c.tree = t
         dest.object_store.add_object(c)
-        return dest, c.id
+        return c.id
+
+    def disable_ff_and_make_dummy_commit(self):
+        # disable non-fast-forward pushes to the server
+        dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
+        run_git_or_fail(['config', 'receive.denyNonFastForwards', 'true'],
+                        cwd=dest.path)
+        commit_id = self.make_dummy_commit(dest)
+        return dest, commit_id
 
     def compute_send(self):
         srcpath = os.path.join(self.gitroot, 'server_new.export')
@@ -160,6 +166,14 @@ class DulwichClientTestBase(object):
                               'refs/heads/master': 'non-fast-forward'},
                              e.ref_status)
 
+    def test_archive(self):
+        c = self._client()
+        f = StringIO()
+        c.archive(self._build_path('/server_new.export'), 'HEAD', f.write)
+        f.seek(0)
+        tf = tarfile.open(fileobj=f)
+        self.assertEquals(['baz', 'foo'], tf.getnames())
+
     def test_fetch_pack(self):
         c = self._client()
         dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
@@ -177,6 +191,29 @@ class DulwichClientTestBase(object):
         map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
         self.assertDestEqualsSrc()
 
+    def test_fetch_pack_zero_sha(self):
+        # zero sha1s are already present on the client, and should
+        # be ignored
+        c = self._client()
+        dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
+        refs = c.fetch(self._build_path('/server_new.export'), dest,
+            lambda refs: [protocol.ZERO_SHA])
+        map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
+
+    def test_send_remove_branch(self):
+        dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
+        dummy_commit = self.make_dummy_commit(dest)
+        dest.refs['refs/heads/master'] = dummy_commit
+        dest.refs['refs/heads/abranch'] = dummy_commit
+        sendrefs = dict(dest.refs)
+        sendrefs['refs/heads/abranch'] = "00" * 20
+        del sendrefs['HEAD']
+        gen_pack = lambda have, want: []
+        c = self._client()
+        self.assertEquals(dest.refs["refs/heads/abranch"], dummy_commit)
+        c.send_pack(self._build_path('/dest'), lambda _: sendrefs, gen_pack)
+        self.assertFalse("refs/heads/abranch" in dest.refs)
+
 
 class DulwichTCPClientTest(CompatTestCase, DulwichClientTestBase):
 
@@ -193,7 +230,7 @@ class DulwichTCPClientTest(CompatTestCase, DulwichClientTestBase):
             ['daemon', '--verbose', '--export-all',
              '--pid-file=%s' % self.pidfile, '--base-path=%s' % self.gitroot,
              '--detach', '--reuseaddr', '--enable=receive-pack',
-             '--listen=localhost', self.gitroot], cwd=self.gitroot)
+             '--enable=upload-archive', '--listen=localhost', self.gitroot], cwd=self.gitroot)
         if not check_for_daemon():
             raise SkipTest('git-daemon failed to start')
 
@@ -254,7 +291,7 @@ class DulwichSubprocessClientTest(CompatTestCase, DulwichClientTestBase):
         CompatTestCase.tearDown(self)
 
     def _client(self):
-        return client.SubprocessGitClient()
+        return client.SubprocessGitClient(stderr=subprocess.PIPE)
 
     def _build_path(self, path):
         return self.gitroot + path
@@ -276,6 +313,10 @@ class GitHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
     def send_head(self):
         return self.run_backend()
 
+    def log_request(self, code='-', size='-'):
+        # Let's be quiet, the test suite is noisy enough already
+        pass
+
     def run_backend(self):
         """Call out to git http-backend."""
         # Based on CGIHTTPServer.CGIHTTPRequestHandler.run_cgi:
@@ -296,10 +337,10 @@ class GitHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
         env['SERVER_PROTOCOL'] = self.protocol_version
         env['SERVER_PORT'] = str(self.server.server_port)
         env['GIT_PROJECT_ROOT'] = self.server.root_path
+        env["GIT_HTTP_EXPORT_ALL"] = "1"
         env['REQUEST_METHOD'] = self.command
         uqrest = urllib.unquote(rest)
         env['PATH_INFO'] = uqrest
-        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
         env['SCRIPT_NAME'] = "/"
         if query:
             env['QUERY_STRING'] = query
@@ -402,12 +443,18 @@ if not getattr(HTTPGitServer, 'shutdown', None):
 
 class DulwichHttpClientTest(CompatTestCase, DulwichClientTestBase):
 
+    min_git_version = (1, 7, 0, 2)
+
     def setUp(self):
         CompatTestCase.setUp(self)
         DulwichClientTestBase.setUp(self)
-        self._httpd = HTTPGitServer(("localhost", 8080), self.gitroot)
+        self._httpd = HTTPGitServer(("localhost", 0), self.gitroot)
         self.addCleanup(self._httpd.shutdown)
         threading.Thread(target=self._httpd.serve_forever).start()
+        run_git_or_fail(['config', 'http.uploadpack', 'true'],
+                        cwd=self.dest)
+        run_git_or_fail(['config', 'http.receivepack', 'true'],
+                        cwd=self.dest)
 
     def tearDown(self):
         DulwichClientTestBase.tearDown(self)
@@ -418,3 +465,6 @@ class DulwichHttpClientTest(CompatTestCase, DulwichClientTestBase):
 
     def _build_path(self, path):
         return path
+
+    def test_archive(self):
+        raise SkipTest("exporting archives not supported over http")

dulwich/tests/compat/test_pack.py

@@ -43,12 +43,9 @@ class TestPack(PackTests):
 
     def setUp(self):
         require_git_version((1, 5, 0))
-        PackTests.setUp(self)
+        super(TestPack, self).setUp()
         self._tempdir = tempfile.mkdtemp()
-
-    def tearDown(self):
-        shutil.rmtree(self._tempdir)
-        PackTests.tearDown(self)
+        self.addCleanup(shutil.rmtree, self._tempdir)
 
     def test_copy(self):
         origpack = self.get_pack(pack1_sha)

dulwich/tests/compat/test_repository.py

@@ -45,12 +45,9 @@ class ObjectStoreTestCase(CompatTestCase):
     """Tests for git repository compatibility."""
 
     def setUp(self):
-        CompatTestCase.setUp(self)
+        super(ObjectStoreTestCase, self).setUp()
         self._repo = import_repo('server_new.export')
-
-    def tearDown(self):
-        CompatTestCase.tearDown(self)
-        tear_down_repo(self._repo)
+        self.addCleanup(tear_down_repo, self._repo)
 
     def _run_git(self, args):
         return run_git_or_fail(args, cwd=self._repo.path)

dulwich/tests/compat/utils.py

@@ -133,8 +133,9 @@ def run_git_or_fail(args, git_path=_DEFAULT_GIT, input=None, **popen_kwargs):
     popen_kwargs['stderr'] = subprocess.STDOUT
     returncode, stdout = run_git(args, git_path=git_path, input=input,
                                  capture_stdout=True, **popen_kwargs)
-    assert returncode == 0, "git with args %r failed with %d" % (
-        args, returncode)
+    if returncode != 0:
+        raise AssertionError("git with args %r failed with %d" % (
+            args, returncode))
     return stdout
 
 

dulwich/tests/test_client.py

@@ -23,6 +23,10 @@ from dulwich.client import (
     TCPGitClient,
     SubprocessGitClient,
     SSHGitClient,
+    HttpGitClient,
+    ReportStatusParser,
+    SendPackError,
+    UpdateRefsError,
     get_transport_and_path,
     )
 from dulwich.tests import (
@@ -58,11 +62,19 @@ class GitClientTests(TestCase):
 
     def test_caps(self):
         self.assertEquals(set(['multi_ack', 'side-band-64k', 'ofs-delta',
-                               'thin-pack']),
+                               'thin-pack', 'multi_ack_detailed']),
                           set(self.client._fetch_capabilities))
-        self.assertEquals(set(['ofs-delta', 'report-status']),
+        self.assertEquals(set(['ofs-delta', 'report-status', 'side-band-64k']),
                           set(self.client._send_capabilities))
 
+    def test_archive_ack(self):
+        self.rin.write(
+            '0009NACK\n'
+            '0000')
+        self.rin.seek(0)
+        self.client.archive('bla', 'HEAD', None, None)
+        self.assertEquals(self.rout.getvalue(), '0011argument HEAD0000')
+
     def test_fetch_pack_none(self):
         self.rin.write(
             '008855dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7 HEAD.multi_ack '
@@ -134,6 +146,12 @@ class GitClientTests(TestCase):
         self.assertRaises(ValueError, get_transport_and_path,
         'prospero://bar/baz')
 
+    def test_get_transport_and_path_http(self):
+        url = 'https://github.com/jelmer/dulwich'
+        client, path = get_transport_and_path(url)
+        self.assertTrue(isinstance(client, HttpGitClient))
+        self.assertEquals('/jelmer/dulwich', path)
+
 
 class SSHGitClientTests(TestCase):
 
@@ -151,3 +169,26 @@ class SSHGitClientTests(TestCase):
         self.assertEquals('/usr/lib/git/git-upload-pack',
             self.client._get_cmd_path('upload-pack'))
 
+
+class ReportStatusParserTests(TestCase):
+
+    def test_invalid_pack(self):
+        parser = ReportStatusParser()
+        parser.handle_packet("unpack error - foo bar")
+        parser.handle_packet("ok refs/foo/bar")
+        parser.handle_packet(None)
+        self.assertRaises(SendPackError, parser.check)
+
+    def test_update_refs_error(self):
+        parser = ReportStatusParser()
+        parser.handle_packet("unpack ok")
+        parser.handle_packet("ng refs/foo/bar need to pull")
+        parser.handle_packet(None)
+        self.assertRaises(UpdateRefsError, parser.check)
+
+    def test_ok(self):
+        parser = ReportStatusParser()
+        parser.handle_packet("unpack ok")
+        parser.handle_packet("ok refs/foo/bar")
+        parser.handle_packet(None)
+        parser.check()

dulwich/tests/test_config.py

@@ -0,0 +1,255 @@
+# test_config.py -- Tests for reading and writing configuration files
+# Copyright (C) 2011 Jelmer Vernooij <jelmer@samba.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# or (at your option) a later version of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Tests for reading and writing configuraiton files."""
+
+from cStringIO import StringIO
+from dulwich.config import (
+    ConfigDict,
+    ConfigFile,
+    StackedConfig,
+    _check_section_name,
+    _check_variable_name,
+    _format_string,
+    _escape_value,
+    _parse_string,
+    _unescape_value,
+    )
+from dulwich.tests import TestCase
+import os
+
+
+class ConfigFileTests(TestCase):
+
+    def from_file(self, text):
+        return ConfigFile.from_file(StringIO(text))
+
+    def test_empty(self):
+        ConfigFile()
+
+    def test_eq(self):
+        self.assertEquals(ConfigFile(), ConfigFile())
+
+    def test_default_config(self):
+        cf = self.from_file("""[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+""")
+        self.assertEquals(ConfigFile({("core", ): {
+            "repositoryformatversion": "0",
+            "filemode": "true",
+            "bare": "false",
+            "logallrefupdates": "true"}}), cf)
+
+    def test_from_file_empty(self):
+        cf = self.from_file("")
+        self.assertEquals(ConfigFile(), cf)
+
+    def test_empty_line_before_section(self):
+        cf = self.from_file("\n[section]\n")
+        self.assertEquals(ConfigFile({("section", ): {}}), cf)
+
+    def test_comment_before_section(self):
+        cf = self.from_file("# foo\n[section]\n")
+        self.assertEquals(ConfigFile({("section", ): {}}), cf)
+
+    def test_comment_after_section(self):
+        cf = self.from_file("[section] # foo\n")
+        self.assertEquals(ConfigFile({("section", ): {}}), cf)
+
+    def test_comment_after_variable(self):
+        cf = self.from_file("[section]\nbar= foo # a comment\n")
+        self.assertEquals(ConfigFile({("section", ): {"bar": "foo"}}), cf)
+
+    def test_from_file_section(self):
+        cf = self.from_file("[core]\nfoo = bar\n")
+        self.assertEquals("bar", cf.get(("core", ), "foo"))
+        self.assertEquals("bar", cf.get(("core", "foo"), "foo"))
+
+    def test_from_file_section_case_insensitive(self):
+        cf = self.from_file("[cOre]\nfOo = bar\n")
+        self.assertEquals("bar", cf.get(("core", ), "foo"))
+        self.assertEquals("bar", cf.get(("core", "foo"), "foo"))
+
+    def test_from_file_with_mixed_quoted(self):
+        cf = self.from_file("[core]\nfoo = \"bar\"la\n")
+        self.assertEquals("barla", cf.get(("core", ), "foo"))
+
+    def test_from_file_with_open_quoted(self):
+        self.assertRaises(ValueError,
+            self.from_file, "[core]\nfoo = \"bar\n")
+
+    def test_from_file_with_quotes(self):
+        cf = self.from_file(
+            "[core]\n"
+            'foo = " bar"\n')
+        self.assertEquals(" bar", cf.get(("core", ), "foo"))
+
+    def test_from_file_with_interrupted_line(self):
+        cf = self.from_file(
+            "[core]\n"
+            'foo = bar\\\n'
+            ' la\n')
+        self.assertEquals("barla", cf.get(("core", ), "foo"))
+
+    def test_from_file_with_boolean_setting(self):
+        cf = self.from_file(
+            "[core]\n"
+            'foo\n')
+        self.assertEquals("true", cf.get(("core", ), "foo"))
+
+    def test_from_file_subsection(self):
+        cf = self.from_file("[branch \"foo\"]\nfoo = bar\n")
+        self.assertEquals("bar", cf.get(("branch", "foo"), "foo"))
+
+    def test_from_file_subsection_invalid(self):
+        self.assertRaises(ValueError,
+            self.from_file, "[branch \"foo]\nfoo = bar\n")
+
+    def test_from_file_subsection_not_quoted(self):
+        cf = self.from_file("[branch.foo]\nfoo = bar\n")
+        self.assertEquals("bar", cf.get(("branch", "foo"), "foo"))
+
+    def test_write_to_file_empty(self):
+        c = ConfigFile()
+        f = StringIO()
+        c.write_to_file(f)
+        self.assertEquals("", f.getvalue())
+
+    def test_write_to_file_section(self):
+        c = ConfigFile()
+        c.set(("core", ), "foo", "bar")
+        f = StringIO()
+        c.write_to_file(f)
+        self.assertEquals("[core]\nfoo = bar\n", f.getvalue())
+
+    def test_write_to_file_subsection(self):
+        c = ConfigFile()
+        c.set(("branch", "blie"), "foo", "bar")
+        f = StringIO()
+        c.write_to_file(f)
+        self.assertEquals("[branch \"blie\"]\nfoo = bar\n", f.getvalue())
+
+    def test_same_line(self):
+        cf = self.from_file("[branch.foo] foo = bar\n")
+        self.assertEquals("bar", cf.get(("branch", "foo"), "foo"))
+
+
+class ConfigDictTests(TestCase):
+
+    def test_get_set(self):
+        cd = ConfigDict()
+        self.assertRaises(KeyError, cd.get, "foo", "core")
+        cd.set(("core", ), "foo", "bla")
+        self.assertEquals("bla", cd.get(("core", ), "foo"))
+        cd.set(("core", ), "foo", "bloe")
+        self.assertEquals("bloe", cd.get(("core", ), "foo"))
+
+    def test_get_boolean(self):
+        cd = ConfigDict()
+        cd.set(("core", ), "foo", "true")
+        self.assertTrue(cd.get_boolean(("core", ), "foo"))
+        cd.set(("core", ), "foo", "false")
+        self.assertFalse(cd.get_boolean(("core", ), "foo"))
+        cd.set(("core", ), "foo", "invalid")
+        self.assertRaises(ValueError, cd.get_boolean, ("core", ), "foo")
+
+
+class StackedConfigTests(TestCase):
+
+    def test_default_backends(self):
+        self.addCleanup(os.environ.__setitem__, "HOME", os.environ["HOME"])
+        os.environ["HOME"] = "/nonexistant"
+        StackedConfig.default_backends()
+
+
+class UnescapeTests(TestCase):
+
+    def test_nothing(self):
+        self.assertEquals("", _unescape_value(""))
+
+    def test_tab(self):
+        self.assertEquals("\tbar\t", _unescape_value("\\tbar\\t"))
+
+    def test_newline(self):
+        self.assertEquals("\nbar\t", _unescape_value("\\nbar\\t"))
+
+    def test_quote(self):
+        self.assertEquals("\"foo\"", _unescape_value("\\\"foo\\\""))
+
+
+class EscapeValueTests(TestCase):
+
+    def test_nothing(self):
+        self.assertEquals("foo", _escape_value("foo"))
+
+    def test_backslash(self):
+        self.assertEquals("foo\\\\", _escape_value("foo\\"))
+
+    def test_newline(self):
+        self.assertEquals("foo\\n", _escape_value("foo\n"))
+
+
+class FormatStringTests(TestCase):
+
+    def test_quoted(self):
+        self.assertEquals('" foo"', _format_string(" foo"))
+        self.assertEquals('"\\tfoo"', _format_string("\tfoo"))
+
+    def test_not_quoted(self):
+        self.assertEquals('foo', _format_string("foo"))
+        self.assertEquals('foo bar', _format_string("foo bar"))
+
+
+class ParseStringTests(TestCase):
+
+    def test_quoted(self):
+        self.assertEquals(' foo', _parse_string('" foo"'))
+        self.assertEquals('\tfoo', _parse_string('"\\tfoo"'))
+
+    def test_not_quoted(self):
+        self.assertEquals('foo', _parse_string("foo"))
+        self.assertEquals('foo bar', _parse_string("foo bar"))
+
+
+class CheckVariableNameTests(TestCase):
+
+    def test_invalid(self):
+        self.assertFalse(_check_variable_name("foo "))
+        self.assertFalse(_check_variable_name("bar,bar"))
+        self.assertFalse(_check_variable_name("bar.bar"))
+
+    def test_valid(self):
+        self.assertTrue(_check_variable_name("FOO"))
+        self.assertTrue(_check_variable_name("foo"))
+        self.assertTrue(_check_variable_name("foo-bar"))
+
+
+class CheckSectionNameTests(TestCase):
+
+    def test_invalid(self):
+        self.assertFalse(_check_section_name("foo "))
+        self.assertFalse(_check_section_name("bar,bar"))
+
+    def test_valid(self):
+        self.assertTrue(_check_section_name("FOO"))
+        self.assertTrue(_check_section_name("foo"))
+        self.assertTrue(_check_section_name("foo-bar"))
+        self.assertTrue(_check_section_name("bar.bar"))

dulwich/tests/test_index.py

@@ -32,6 +32,7 @@ from dulwich.index import (
     Index,
     cleanup_mode,
     commit_tree,
+    index_entry_from_stat,
     read_index,
     write_cache_time,
     write_index,
@@ -169,3 +170,41 @@ class WriteCacheTimeTests(TestCase):
         f = StringIO()
         write_cache_time(f, 434343.000000021)
         self.assertEquals(struct.pack(">LL", 434343, 21), f.getvalue())
+
+
+class IndexEntryFromStatTests(TestCase):
+
+    def test_simple(self):
+        st = os.stat_result((16877, 131078, 64769L,
+                154, 1000, 1000, 12288,
+                1323629595, 1324180496, 1324180496))
+        entry = index_entry_from_stat(st, "22" * 20, 0)
+        self.assertEquals(entry, (
+            1324180496,
+            1324180496,
+            64769L,
+            131078,
+            16384,
+            1000,
+            1000,
+            12288,
+            '2222222222222222222222222222222222222222',
+            0))
+
+    def test_override_mode(self):
+        st = os.stat_result((stat.S_IFREG + 0644, 131078, 64769L,
+                154, 1000, 1000, 12288,
+                1323629595, 1324180496, 1324180496))
+        entry = index_entry_from_stat(st, "22" * 20, 0,
+                mode=stat.S_IFREG + 0755)
+        self.assertEquals(entry, (
+            1324180496,
+            1324180496,
+            64769L,
+            131078,
+            33261,
+            1000,
+            1000,
+            12288,
+            '2222222222222222222222222222222222222222',
+            0))

dulwich/tests/test_object_store.py

@@ -221,12 +221,33 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
     def setUp(self):
         TestCase.setUp(self)
         self.store_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.store_dir)
         self.store = DiskObjectStore.init(self.store_dir)
 
     def tearDown(self):
         TestCase.tearDown(self)
         PackBasedObjectStoreTests.tearDown(self)
-        shutil.rmtree(self.store_dir)
+
+    def test_alternates(self):
+        alternate_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, alternate_dir)
+        alternate_store = DiskObjectStore(alternate_dir)
+        b2 = make_object(Blob, data="yummy data")
+        alternate_store.add_object(b2)
+        store = DiskObjectStore(self.store_dir)
+        self.assertRaises(KeyError, store.__getitem__, b2.id)
+        store.add_alternate_path(alternate_dir)
+        self.assertEquals(b2, store[b2.id])
+
+    def test_add_alternate_path(self):
+        store = DiskObjectStore(self.store_dir)
+        self.assertEquals([], store._read_alternate_paths())
+        store.add_alternate_path("/foo/path")
+        self.assertEquals(["/foo/path"], store._read_alternate_paths())
+        store.add_alternate_path("/bar/path")
+        self.assertEquals(
+            ["/foo/path", "/bar/path"],
+            store._read_alternate_paths())
 
     def test_pack_dir(self):
         o = DiskObjectStore(self.store_dir)
@@ -249,15 +270,20 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
           (REF_DELTA, (blob.id, 'more yummy data')),
           ], store=o)
         pack = o.add_thin_pack(f.read, None)
+        try:
+            packed_blob_sha = sha_to_hex(entries[0][3])
+            pack.check_length_and_checksum()
+            self.assertEqual(sorted([blob.id, packed_blob_sha]), list(pack))
+            self.assertTrue(o.contains_packed(packed_blob_sha))
+            self.assertTrue(o.contains_packed(blob.id))
+            self.assertEqual((Blob.type_num, 'more yummy data'),
+                             o.get_raw(packed_blob_sha))
+        finally:
+            # FIXME: DiskObjectStore should have close() which do the following:
+            for p in o._pack_cache or []:
+                p.close()
 
-        packed_blob_sha = sha_to_hex(entries[0][3])
-        pack.check_length_and_checksum()
-        self.assertEqual(sorted([blob.id, packed_blob_sha]), list(pack))
-        self.assertTrue(o.contains_packed(packed_blob_sha))
-        self.assertTrue(o.contains_packed(blob.id))
-        self.assertEqual((Blob.type_num, 'more yummy data'),
-                         o.get_raw(packed_blob_sha))
-
+            pack.close()
 
 class TreeLookupPathTests(TestCase):
 

dulwich/tests/test_objects.py

@@ -38,6 +38,7 @@ from dulwich.objects import (
     Blob,
     Tree,
     Commit,
+    ShaFile,
     Tag,
     format_timezone,
     hex_to_sha,
@@ -229,6 +230,31 @@ class ShaFileCheckTests(TestCase):
         self.assertEqual(None, obj.check())
 
 
+small_buffer_zlib_object = (
+ "\x48\x89\x15\xcc\x31\x0e\xc2\x30\x0c\x40\x51\xe6"
+ "\x9c\xc2\x3b\xaa\x64\x37\xc4\xc1\x12\x42\x5c\xc5"
+ "\x49\xac\x52\xd4\x92\xaa\x78\xe1\xf6\x94\xed\xeb"
+ "\x0d\xdf\x75\x02\xa2\x7c\xea\xe5\x65\xd5\x81\x8b"
+ "\x9a\x61\xba\xa0\xa9\x08\x36\xc9\x4c\x1a\xad\x88"
+ "\x16\xba\x46\xc4\xa8\x99\x6a\x64\xe1\xe0\xdf\xcd"
+ "\xa0\xf6\x75\x9d\x3d\xf8\xf1\xd0\x77\xdb\xfb\xdc"
+ "\x86\xa3\x87\xf1\x2f\x93\xed\x00\xb7\xc7\xd2\xab"
+ "\x2e\xcf\xfe\xf1\x3b\x50\xa4\x91\x53\x12\x24\x38"
+ "\x23\x21\x86\xf0\x03\x2f\x91\x24\x52"
+ )
+
+
+class ShaFileTests(TestCase):
+
+    def test_deflated_smaller_window_buffer(self):
+        # zlib on some systems uses smaller buffers,
+        # resulting in a different header.
+        # See https://github.com/libgit2/libgit2/pull/464
+        sf = ShaFile.from_file(StringIO(small_buffer_zlib_object))
+        self.assertEquals(sf.type_name, "tag")
+        self.assertEquals(sf.tagger, " <@localhost>")
+
+
 class CommitSerializationTests(TestCase):
 
     def make_commit(self, **kwargs):
@@ -582,6 +608,7 @@ OK2XeQOiEeXtT76rV4t2WR4=
 
 
 class TagParseTests(ShaFileCheckTests):
+
     def make_tag_lines(self,
                        object_sha="a38d6181ff27824c79fc7df825164a212eff6a3f",
                        object_type_name="commit",

dulwich/tests/test_pack.py

@@ -733,6 +733,7 @@ class TestPackIterator(DeltaChainIterator):
 class DeltaChainIteratorTests(TestCase):
 
     def setUp(self):
+        super(DeltaChainIteratorTests, self).setUp()
         self.store = MemoryObjectStore()
         self.fetched = set()
 

dulwich/tests/test_protocol.py

@@ -25,6 +25,7 @@ from dulwich.errors import (
     HangupException,
     )
 from dulwich.protocol import (
+    PktLineParser,
     Protocol,
     ReceivableProtocol,
     extract_capabilities,
@@ -280,3 +281,29 @@ class BufferedPktLineWriterTests(TestCase):
         self._writer.write('z')
         self._writer.flush()
         self.assertOutputEquals('0005z')
+
+
+class PktLineParserTests(TestCase):
+
+    def test_none(self):
+        pktlines = []
+        parser = PktLineParser(pktlines.append)
+        parser.parse("0000")
+        self.assertEquals(pktlines, [None])
+        self.assertEquals("", parser.get_tail())
+
+    def test_small_fragments(self):
+        pktlines = []
+        parser = PktLineParser(pktlines.append)
+        parser.parse("00")
+        parser.parse("05")
+        parser.parse("z0000")
+        self.assertEquals(pktlines, ["z", None])
+        self.assertEquals("", parser.get_tail())
+
+    def test_multiple_packets(self):
+        pktlines = []
+        parser = PktLineParser(pktlines.append)
+        parser.parse("0005z0006aba")
+        self.assertEquals(pktlines, ["z", "ab"])
+        self.assertEquals("a", parser.get_tail())

dulwich/tests/test_repository.py

@@ -33,9 +33,11 @@ from dulwich.object_store import (
     tree_lookup_path,
     )
 from dulwich import objects
+from dulwich.config import Config
 from dulwich.repo import (
     check_ref_format,
     DictRefsContainer,
+    InfoRefsContainer,
     Repo,
     MemoryRepo,
     read_packed_refs,
@@ -72,7 +74,8 @@ class CreateRepositoryTests(TestCase):
         self.assertFileContentsEqual('', repo, os.path.join('info', 'exclude'))
         self.assertFileContentsEqual(None, repo, 'nonexistent file')
         barestr = 'bare = %s' % str(expect_bare).lower()
-        self.assertTrue(barestr in repo.get_named_file('config').read())
+        config_text = repo.get_named_file('config').read()
+        self.assertTrue(barestr in config_text, "%r" % config_text)
 
     def test_create_disk_bare(self):
         tmp_dir = tempfile.mkdtemp()
@@ -247,8 +250,19 @@ class RepositoryTests(TestCase):
         self.addCleanup(warnings.resetwarnings)
         self.assertRaises(errors.NotBlobError, r.get_blob, r.head())
 
+    def test_get_walker(self):
+        r = self._repo = open_repo('a.git')
+        # include defaults to [r.head()]
+        self.assertEqual([e.commit.id for e in r.get_walker()],
+                         [r.head(), '2a72d929692c41d8554c07f6301757ba18a65d91'])
+        self.assertEqual(
+            [e.commit.id for e in r.get_walker(['2a72d929692c41d8554c07f6301757ba18a65d91'])],
+            ['2a72d929692c41d8554c07f6301757ba18a65d91'])
+
     def test_linear_history(self):
         r = self._repo = open_repo('a.git')
+        warnings.simplefilter("ignore", DeprecationWarning)
+        self.addCleanup(warnings.resetwarnings)
         history = r.revision_history(r.head())
         shas = [c.sha().hexdigest() for c in history]
         self.assertEqual(shas, [r.head(),
@@ -268,15 +282,13 @@ class RepositoryTests(TestCase):
             'refs/tags/mytag-packed':
                 'b0931cadc54336e78a1d980420e3268903b57a50',
             }, t.refs.as_dict())
-        history = t.revision_history(t.head())
-        shas = [c.sha().hexdigest() for c in history]
+        shas = [e.commit.id for e in r.get_walker()]
         self.assertEqual(shas, [t.head(),
                          '2a72d929692c41d8554c07f6301757ba18a65d91'])
 
     def test_merge_history(self):
         r = self._repo = open_repo('simple_merge.git')
-        history = r.revision_history(r.head())
-        shas = [c.sha().hexdigest() for c in history]
+        shas = [e.commit.id for e in r.get_walker()]
         self.assertEqual(shas, ['5dac377bdded4c9aeb8dff595f0faeebcc8498cc',
                                 'ab64bbdcc51b170d21588e5c5d391ee5c0c96dfd',
                                 '4cffe90e0a41ad3f5190079d7c8f036bde29cbe6',
@@ -285,14 +297,15 @@ class RepositoryTests(TestCase):
 
     def test_revision_history_missing_commit(self):
         r = self._repo = open_repo('simple_merge.git')
+        warnings.simplefilter("ignore", DeprecationWarning)
+        self.addCleanup(warnings.resetwarnings)
         self.assertRaises(errors.MissingCommitError, r.revision_history,
                           missing_sha)
 
     def test_out_of_order_merge(self):
         """Test that revision history is ordered by date, not parent order."""
         r = self._repo = open_repo('ooo_merge.git')
-        history = r.revision_history(r.head())
-        shas = [c.sha().hexdigest() for c in history]
+        shas = [e.commit.id for e in r.get_walker()]
         self.assertEqual(shas, ['7601d7f6231db6a57f7bbb79ee52e4d462fd44d1',
                                 'f507291b64138b875c28e03469025b1ea20bc614',
                                 'fb5b0425c7ce46959bec94d54b9a157645e114f5',
@@ -304,7 +317,13 @@ class RepositoryTests(TestCase):
 
     def test_get_config(self):
         r = self._repo = open_repo('ooo_merge.git')
-        self.assertEquals({}, r.get_config())
+        self.assertIsInstance(r.get_config(), Config)
+
+    def test_get_config_stack(self):
+        self.addCleanup(os.environ.__setitem__, "HOME", os.environ["HOME"])
+        os.environ["HOME"] = "/nonexistant"
+        r = self._repo = open_repo('ooo_merge.git')
+        self.assertIsInstance(r.get_config_stack(), Config)
 
     def test_common_revisions(self):
         """
@@ -440,6 +459,23 @@ class BuildRepoTests(TestCase):
              encoding="iso8859-1")
         self.assertEquals("iso8859-1", r[commit_sha].encoding)
 
+    def test_commit_config_identity(self):
+        self.addCleanup(os.environ.__setitem__, "HOME", os.environ["HOME"])
+        os.environ["HOME"] = "/nonexistant"
+        # commit falls back to the users' identity if it wasn't specified
+        r = self._repo
+        c = r.get_config()
+        c.set(("user", ), "name", "Jelmer")
+        c.set(("user", ), "email", "jelmer@apache.org")
+        c.write_to_path()
+        commit_sha = r.do_commit('message')
+        self.assertEquals(
+            "Jelmer <jelmer@apache.org>",
+            r[commit_sha].author)
+        self.assertEquals(
+            "Jelmer <jelmer@apache.org>",
+            r[commit_sha].committer)
+
     def test_commit_fail_ref(self):
         r = self._repo
 
@@ -674,6 +710,7 @@ class RefsContainerTests(object):
 
     def test_check_refname(self):
         self._refs._check_refname('HEAD')
+        self._refs._check_refname('refs/stash')
         self._refs._check_refname('refs/heads/foo')
 
         self.assertRaises(errors.RefFormatError, self._refs._check_refname,
@@ -876,3 +913,56 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
             self._refs.read_ref("refs/heads/packed"))
         self.assertEqual(None,
             self._refs.read_ref("nonexistant"))
+
+
+_TEST_REFS_SERIALIZED = (
+'42d06bd4b77fed026b154d16493e5deab78f02ec\trefs/heads/master\n'
+'42d06bd4b77fed026b154d16493e5deab78f02ec\trefs/heads/packed\n'
+'df6800012397fb85c56e7418dd4eb9405dee075c\trefs/tags/refs-0.1\n'
+'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8\trefs/tags/refs-0.2\n')
+
+
+class InfoRefsContainerTests(TestCase):
+
+    def test_invalid_refname(self):
+        text = _TEST_REFS_SERIALIZED + '00' * 20 + '\trefs/stash\n'
+        refs = InfoRefsContainer(StringIO(text))
+        expected_refs = dict(_TEST_REFS)
+        del expected_refs['HEAD']
+        expected_refs["refs/stash"] = "00" * 20
+        self.assertEquals(expected_refs, refs.as_dict())
+
+    def test_keys(self):
+        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        actual_keys = set(refs.keys())
+        self.assertEqual(set(refs.allkeys()), actual_keys)
+        # ignore the symref loop if it exists
+        actual_keys.discard('refs/heads/loop')
+        expected_refs = dict(_TEST_REFS)
+        del expected_refs['HEAD']
+        self.assertEqual(set(expected_refs.iterkeys()), actual_keys)
+
+        actual_keys = refs.keys('refs/heads')
+        actual_keys.discard('loop')
+        self.assertEqual(['master', 'packed'], sorted(actual_keys))
+        self.assertEqual(['refs-0.1', 'refs-0.2'],
+                         sorted(refs.keys('refs/tags')))
+
+    def test_as_dict(self):
+        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        # refs/heads/loop does not show up even if it exists
+        expected_refs = dict(_TEST_REFS)
+        del expected_refs['HEAD']
+        self.assertEqual(expected_refs, refs.as_dict())
+
+    def test_contains(self):
+        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        self.assertTrue('refs/heads/master' in refs)
+        self.assertFalse('refs/heads/bar' in refs)
+
+    def test_get_peeled(self):
+        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        # refs/heads/loop does not show up even if it exists
+        self.assertEqual(
+            _TEST_REFS['refs/heads/master'],
+            refs.get_peeled('refs/heads/master'))

dulwich/tests/test_server.py

@@ -44,6 +44,7 @@ from dulwich.server import (
     ReceivePackHandler,
     SingleAckGraphWalkerImpl,
     UploadPackHandler,
+    update_server_info,
     )
 from dulwich.tests import TestCase
 from dulwich.tests.utils import (
@@ -689,3 +690,30 @@ class ServeCommandTests(TestCase):
             outlines[0][4:].split("\x00")[0])
         self.assertEquals("0000", outlines[-1])
         self.assertEquals(0, exitcode)
+
+
+class UpdateServerInfoTests(TestCase):
+    """Tests for update_server_info."""
+
+    def setUp(self):
+        super(UpdateServerInfoTests, self).setUp()
+        self.path = tempfile.mkdtemp()
+        self.repo = Repo.init(self.path)
+
+    def test_empty(self):
+        update_server_info(self.repo)
+        self.assertEquals("",
+            open(os.path.join(self.path, ".git", "info", "refs"), 'r').read())
+        self.assertEquals("",
+            open(os.path.join(self.path, ".git", "objects", "info", "packs"), 'r').read())
+
+    def test_simple(self):
+        commit_id = self.repo.do_commit(
+            message="foo",
+            committer="Joe Example <joe@example.com>",
+            ref="refs/heads/foo")
+        update_server_info(self.repo)
+        ref_text = open(os.path.join(self.path, ".git", "info", "refs"), 'r').read()
+        self.assertEquals(ref_text, "%s\trefs/heads/foo\n" % commit_id)
+        packs_text = open(os.path.join(self.path, ".git", "objects", "info", "packs"), 'r').read()
+        self.assertEquals(packs_text, "")

dulwich/tests/test_utils.py

@@ -37,6 +37,7 @@ from utils import (
 class BuildCommitGraphTest(TestCase):
 
     def setUp(self):
+        super(BuildCommitGraphTest, self).setUp()
         self.store = MemoryObjectStore()
 
     def test_linear(self):

dulwich/tests/test_walk.py

@@ -74,6 +74,7 @@ class TestWalkEntry(object):
 class WalkerTest(TestCase):
 
     def setUp(self):
+        super(WalkerTest, self).setUp()
         self.store = MemoryObjectStore()
 
     def make_commits(self, commit_spec, **kwargs):
@@ -122,7 +123,7 @@ class WalkerTest(TestCase):
         del self.store[cs[-1].id]
         for i in xrange(1, 11):
             self.assertWalkYields(cs[:i], [cs[0].id], max_entries=i)
-        self.assertRaises(MissingCommitError, Walker, self.store, cs[0].id)
+        self.assertRaises(MissingCommitError, Walker, self.store, [cs[-1].id])
 
     def test_branch(self):
         c1, x2, x3, y4 = self.make_commits([[1], [2, 1], [3, 2], [4, 1]])

dulwich/tests/test_web.py

@@ -20,6 +20,7 @@
 
 from cStringIO import StringIO
 import re
+import os
 
 from dulwich.object_store import (
     MemoryObjectStore,
@@ -198,7 +199,7 @@ class DumbHandlersTestCase(WebTestCase):
         self.assertEquals(HTTP_ERROR, self._status)
 
     def test_get_pack_file(self):
-        pack_name = 'objects/pack/pack-%s.pack' % ('1' * 40)
+        pack_name = os.path.join('objects', 'pack', 'pack-%s.pack' % ('1' * 40))
         backend = _test_backend([], named_files={pack_name: 'pack contents'})
         mat = re.search('.*', pack_name)
         output = ''.join(get_pack_file(self._req, backend, mat))
@@ -208,7 +209,7 @@ class DumbHandlersTestCase(WebTestCase):
         self.assertTrue(self._req.cached)
 
     def test_get_idx_file(self):
-        idx_name = 'objects/pack/pack-%s.idx' % ('1' * 40)
+        idx_name = os.path.join('objects', 'pack', 'pack-%s.idx' % ('1' * 40))
         backend = _test_backend([], named_files={idx_name: 'idx contents'})
         mat = re.search('.*', idx_name)
         output = ''.join(get_idx_file(self._req, backend, mat))

dulwich/tests/utils.py

@@ -230,6 +230,7 @@ def build_pack(f, objects_spec, store=None):
     expected = []
     for i in xrange(num_objects):
         type_num, data, sha = full_objects[i]
+        assert len(sha) == 20
         expected.append((offsets[i], type_num, data, sha, crc32s[i]))
 
     sf.write_sha()

dulwich/walk.py

@@ -221,6 +221,8 @@ class Walker(object):
             iterator protocol. The constructor takes a single argument, the
             Walker.
         """
+        # Note: when adding arguments to this method, please also update
+        # dulwich.repo.BaseRepo.get_walker
         if order not in ALL_ORDERS:
             raise ValueError('Unknown walk order %s' % order)
         self.store = store

dulwich/web.py

@@ -38,6 +38,8 @@ from dulwich.repo import (
 from dulwich.server import (
     DictBackend,
     DEFAULT_HANDLERS,
+    generate_info_refs,
+    generate_objects_info_packs,
     )
 
 
@@ -180,28 +182,15 @@ def get_info_refs(req, backend, mat):
         req.respond(HTTP_OK, 'text/plain')
         logger.info('Emulating dumb info/refs')
         repo = get_repo(backend, mat)
-        refs = repo.get_refs()
-        for name in sorted(refs.iterkeys()):
-            # get_refs() includes HEAD as a special case, but we don't want to
-            # advertise it
-            if name == 'HEAD':
-                continue
-            sha = refs[name]
-            o = repo[sha]
-            if not o:
-                continue
-            yield '%s\t%s\n' % (sha, name)
-            peeled_sha = repo.get_peeled(name)
-            if peeled_sha != sha:
-                yield '%s\t%s^{}\n' % (peeled_sha, name)
+        for text in generate_info_refs(repo):
+            yield text
 
 
 def get_info_packs(req, backend, mat):
     req.nocache()
     req.respond(HTTP_OK, 'text/plain')
     logger.info('Emulating dumb info/packs')
-    for pack in get_repo(backend, mat).object_store.packs:
-        yield 'P pack-%s.pack\n' % pack.name()
+    return generate_objects_info_packs(get_repo(backend, mat))
 
 
 class _LengthLimitedFile(object):

setup.py

@@ -10,7 +10,7 @@ except ImportError:
     has_setuptools = False
 from distutils.core import Distribution
 
-dulwich_version_string = '0.8.1'
+dulwich_version_string = '0.8.4'
 
 include_dirs = []
 # Windows MSVC support
@@ -27,35 +27,27 @@ class DulwichDistribution(Distribution):
             return True
 
     def has_ext_modules(self):
-        return not self.pure
+        return not self.pure and not '__pypy__' in sys.modules
 
     global_options = Distribution.global_options + [
         ('pure', None, 
-            "use pure (slower) Python code instead of C extensions")]
+            "use pure Python code instead of C extensions (slower on CPython)")]
 
     pure = False
 
-def runcmd(cmd, env):
-    import subprocess
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE, env=env)
-    out, err = p.communicate()
-    err = [e for e in err.splitlines()
-           if not e.startswith('Not trusting file') \
-              and not e.startswith('warning: Not importing')]
-    if err:
-        return ''
-    return out
-
-
 if sys.platform == 'darwin' and os.path.exists('/usr/bin/xcodebuild'):
     # XCode 4.0 dropped support for ppc architecture, which is hardcoded in
     # distutils.sysconfig
-    version = runcmd(['/usr/bin/xcodebuild', '-version'], {}).splitlines()[0]
-    # Also parse only first digit, because 3.2.1 can't be parsed nicely
-    if (version.startswith('Xcode') and
-        int(version.split()[1].split('.')[0]) >= 4):
-        os.environ['ARCHFLAGS'] = ''
+    import subprocess
+    p = subprocess.Popen(
+        ['/usr/bin/xcodebuild', '-version'], stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE, env={})
+    out, err = p.communicate()
+    for l in out.splitlines():
+        # Also parse only first digit, because 3.2.1 can't be parsed nicely
+        if (l.startswith('Xcode') and
+            int(l.split()[1].split('.')[0]) >= 4):
+            os.environ['ARCHFLAGS'] = ''
 
 setup_kwargs = {}