Merge Dave.

Makefile

@@ -3,7 +3,7 @@ SETUP = $(PYTHON) setup.py
 PYDOCTOR ?= pydoctor
 TESTRUNNER = $(shell which nosetests)
 
-all: build 
+all: build
 
 doc:: pydoctor
 
@@ -23,6 +23,9 @@ check:: build
 check-noextensions:: clean
 	PYTHONPATH=. $(PYTHON) $(TESTRUNNER) dulwich
 
+check-compat:: build
+	PYTHONPATH=. $(PYTHON) $(TESTRUNNER) -i compat
+
 clean::
 	$(SETUP) clean --all
 	rm -f dulwich/*.so

NEWS

@@ -1,3 +1,21 @@
+0.5.1	UNRELEASED
+
+ BUG FIXES
+ 
+  * Fix ReceivePackHandler to disallow removing refs without delete-refs.
+    (Dave Borowitz)
+
+  * Deal with capabilities required by the client, even if they 
+    can not be disabled in the server. (Dave Borowitz)
+
+ FEATURES
+
+  * Add include-tag capability to server. (Dave Borowitz)
+
+ TESTS
+
+  * Add framework for testing compatibility with C Git. (Dave Borowitz)
+
 0.5.0	2010-03-03
 
  BUG FIXES

dulwich/client.py

@@ -32,6 +32,7 @@ from dulwich.errors import (
 from dulwich.protocol import (
     Protocol,
     TCP_GIT_PORT,
+    ZERO_SHA,
     extract_capabilities,
     )
 from dulwich.pack import (
@@ -97,18 +98,18 @@ class GitClient(object):
             self.proto.write_pkt_line(None)
             return {}
         want = []
-        have = [x for x in old_refs.values() if not x == "0" * 40]
+        have = [x for x in old_refs.values() if not x == ZERO_SHA]
         sent_capabilities = False
         for refname in set(new_refs.keys() + old_refs.keys()):
-            old_sha1 = old_refs.get(refname, "0" * 40)
-            new_sha1 = new_refs.get(refname, "0" * 40)
+            old_sha1 = old_refs.get(refname, ZERO_SHA)
+            new_sha1 = new_refs.get(refname, ZERO_SHA)
             if old_sha1 != new_sha1:
                 if sent_capabilities:
                     self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1, refname))
                 else:
                     self.proto.write_pkt_line("%s %s %s\0%s" % (old_sha1, new_sha1, refname, self.capabilities()))
                     sent_capabilities = True
-            if not new_sha1 in (have, "0" * 40):
+            if not new_sha1 in (have, ZERO_SHA):
                 want.append(new_sha1)
         self.proto.write_pkt_line(None)
         if not want:
@@ -333,4 +334,3 @@ class SSHGitClient(GitClient):
         client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
         return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
                                  progress)
-

dulwich/errors.py

@@ -61,6 +61,12 @@ class NotTreeError(WrongObjectException):
     _type = 'tree'
 
 
+class NotTagError(WrongObjectException):
+    """Indicates that the sha requested does not point to a tag."""
+
+    _type = 'tag'
+
+
 class NotBlobError(WrongObjectException):
     """Indicates that the sha requested does not point to a blob."""
   

dulwich/object_store.py

@@ -195,16 +195,20 @@ class BaseObjectStore(object):
                 else:
                     yield path, mode, hexsha
 
-    def find_missing_objects(self, haves, wants, progress=None):
+    def find_missing_objects(self, haves, wants, progress=None,
+                             get_tagged=None):
         """Find the missing objects required for a set of revisions.
 
         :param haves: Iterable over SHAs already in common.
         :param wants: Iterable over SHAs of objects to fetch.
         :param progress: Simple progress function that will be called with 
             updated progress strings.
+        :param get_tagged: Function that returns a dict of pointed-to sha -> tag
+            sha for including tags.
         :return: Iterator over (sha, path) pairs.
         """
-        return iter(MissingObjectFinder(self, haves, wants, progress).next, None)
+        finder = MissingObjectFinder(self, haves, wants, progress, get_tagged)
+        return iter(finder.next, None)
 
     def find_common_revisions(self, graphwalker):
         """Find which revisions this store has in common using graphwalker.
@@ -253,6 +257,10 @@ class PackBasedObjectStore(BaseObjectStore):
     def _load_packs(self):
         raise NotImplementedError(self._load_packs)
 
+    def _pack_cache_stale(self):
+        """Check whether the pack cache is stale."""
+        raise NotImplementedError(self._pack_cache_stale)
+
     def _add_known_pack(self, pack):
         """Add a newly appeared pack to the cache by path.
 
@@ -263,7 +271,7 @@ class PackBasedObjectStore(BaseObjectStore):
     @property
     def packs(self):
         """List with pack objects."""
-        if self._pack_cache is None:
+        if self._pack_cache is None or self._pack_cache_stale():
             self._pack_cache = self._load_packs()
         return self._pack_cache
 
@@ -332,11 +340,14 @@ class DiskObjectStore(PackBasedObjectStore):
         super(DiskObjectStore, self).__init__()
         self.path = path
         self.pack_dir = os.path.join(self.path, PACKDIR)
+        self._pack_cache_time = 0
 
     def _load_packs(self):
         pack_files = []
         try:
-            for name in os.listdir(self.pack_dir):
+            self._pack_cache_time = os.stat(self.pack_dir).st_mtime
+            pack_dir_contents = os.listdir(self.pack_dir)
+            for name in pack_dir_contents:
                 # TODO: verify that idx exists first
                 if name.startswith("pack-") and name.endswith(".pack"):
                     filename = os.path.join(self.pack_dir, name)
@@ -349,6 +360,14 @@ class DiskObjectStore(PackBasedObjectStore):
         suffix_len = len(".pack")
         return [Pack(f[:-suffix_len]) for _, f in pack_files]
 
+    def _pack_cache_stale(self):
+        try:
+            return os.stat(self.pack_dir).st_mtime > self._pack_cache_time
+        except OSError, e:
+            if e.errno == errno.ENOENT:
+                return True
+            raise
+
     def _get_shafile_path(self, sha):
         dir = sha[:2]
         file = sha[2:]
@@ -622,9 +641,13 @@ class MissingObjectFinder(object):
     :param haves: SHA1s of commits not to send (already present in target)
     :param wants: SHA1s of commits to send
     :param progress: Optional function to report progress to.
+    :param get_tagged: Function that returns a dict of pointed-to sha -> tag
+        sha for including tags.
+    :param tagged: dict of pointed-to sha -> tag sha for including tags
     """
 
-    def __init__(self, object_store, haves, wants, progress=None):
+    def __init__(self, object_store, haves, wants, progress=None,
+                 get_tagged=None):
         self.sha_done = set(haves)
         self.objects_to_send = set([(w, None, False) for w in wants if w not in haves])
         self.object_store = object_store
@@ -632,6 +655,7 @@ class MissingObjectFinder(object):
             self.progress = lambda x: None
         else:
             self.progress = progress
+        self._tagged = get_tagged and get_tagged() or {}
 
     def add_todo(self, entries):
         self.objects_to_send.update([e for e in entries if not e[0] in self.sha_done])
@@ -658,6 +682,8 @@ class MissingObjectFinder(object):
                 self.parse_tree(o)
             elif isinstance(o, Tag):
                 self.parse_tag(o)
+        if sha in self._tagged:
+            self.add_todo([(self._tagged[sha], None, True)])
         self.sha_done.add(sha)
         self.progress("counting objects: %d\r" % len(self.sha_done))
         return (sha, name)

dulwich/protocol.py

@@ -28,6 +28,8 @@ from dulwich.errors import (
 
 TCP_GIT_PORT = 9418
 
+ZERO_SHA = "0" * 40
+
 SINGLE_ACK = 0
 MULTI_ACK = 1
 MULTI_ACK_DETAILED = 2

dulwich/repo.py

@@ -32,6 +32,7 @@ from dulwich.errors import (
     NotCommitError, 
     NotGitRepository,
     NotTreeError, 
+    NotTagError,
     PackedRefsException,
     )
 from dulwich.file import (
@@ -48,6 +49,7 @@ from dulwich.objects import (
     Tag,
     Tree,
     hex_to_sha,
+    num_type_map,
     )
 
 OBJECTDIR = 'objects'
@@ -131,6 +133,16 @@ class RefsContainer(object):
         """
         raise NotImplementedError(self.get_packed_refs)
 
+    def get_peeled(self, name):
+        """Return the cached peeled value of a ref, if available.
+
+        :param name: Name of the ref to peel
+        :return: The peeled value of the ref. If the ref is known not point to a
+            tag, this will be the SHA the ref refers to. If the ref may point to
+            a tag, but no cached information is available, None is returned.
+        """
+        return None
+
     def import_refs(self, base, other):
         for name, value in other.iteritems():
             self["%s/%s" % (base, name)] = value
@@ -245,7 +257,7 @@ class DiskRefsContainer(RefsContainer):
     def __init__(self, path):
         self.path = path
         self._packed_refs = None
-        self._peeled_refs = {}
+        self._peeled_refs = None
 
     def __repr__(self):
         return "%s(%r)" % (self.__class__.__name__, self.path)
@@ -310,6 +322,7 @@ class DiskRefsContainer(RefsContainer):
                 first_line = iter(f).next().rstrip()
                 if (first_line.startswith("# pack-refs") and " peeled" in
                         first_line):
+                    self._peeled_refs = {}
                     for sha, name, peeled in read_packed_refs_with_peeled(f):
                         self._packed_refs[name] = sha
                         if peeled:
@@ -322,6 +335,24 @@ class DiskRefsContainer(RefsContainer):
                 f.close()
         return self._packed_refs
 
+    def get_peeled(self, name):
+        """Return the cached peeled value of a ref, if available.
+
+        :param name: Name of the ref to peel
+        :return: The peeled value of the ref. If the ref is known not point to a
+            tag, this will be the SHA the ref refers to. If the ref may point to
+            a tag, but no cached information is available, None is returned.
+        """
+        self.get_packed_refs()
+        if self._peeled_refs is None or name not in self._packed_refs:
+            # No cache: no peeled refs were read, or this ref is loose
+            return None
+        if name in self._peeled_refs:
+            return self._peeled_refs[name]
+        else:
+            # Known not peelable
+            return self[name]
+
     def read_loose_ref(self, name):
         """Read a reference file and return its contents.
 
@@ -558,6 +589,7 @@ def write_packed_refs(f, packed_refs, peeled_refs=None):
 
     :param f: empty file-like object to write to
     :param packed_refs: dict of refname to sha of packed refs to write
+    :param peeled_refs: dict of refname to peeled value of sha
     """
     if peeled_refs is None:
         peeled_refs = {}
@@ -616,7 +648,8 @@ class BaseRepo(object):
                 progress))
         return self.get_refs()
 
-    def fetch_objects(self, determine_wants, graph_walker, progress):
+    def fetch_objects(self, determine_wants, graph_walker, progress,
+                      get_tagged=None):
         """Fetch the missing objects required for a set of revisions.
 
         :param determine_wants: Function that takes a dictionary with heads 
@@ -626,12 +659,15 @@ class BaseRepo(object):
             that a revision is present.
         :param progress: Simple progress function that will be called with 
             updated progress strings.
+        :param get_tagged: Function that returns a dict of pointed-to sha -> tag
+            sha for including tags.
         :return: iterator over objects, with __len__ implemented
         """
         wants = determine_wants(self.get_refs())
         haves = self.object_store.find_common_revisions(graph_walker)
         return self.object_store.iter_shas(
-            self.object_store.find_missing_objects(haves, wants, progress))
+            self.object_store.find_missing_objects(haves, wants, progress,
+                                                   get_tagged))
 
     def get_graph_walker(self, heads=None):
         if heads is None:
@@ -660,6 +696,8 @@ class BaseRepo(object):
                 raise NotBlobError(ret)
             elif cls is Tree:
                 raise NotTreeError(ret)
+            elif cls is Tag:
+                raise NotTagError(ret)
             else:
                 raise Exception("Type invalid: %r != %r" % (ret._type, cls._type))
         return ret
@@ -686,6 +724,24 @@ class BaseRepo(object):
     def tag(self, sha):
         return self._get_object(sha, Tag)
 
+    def get_peeled(self, ref):
+        """Get the peeled value of a ref.
+
+        :param ref: the refname to peel
+        :return: the fully-peeled SHA1 of a tag object, after peeling all
+            intermediate tags; if the original ref does not point to a tag, this
+            will equal the original SHA1.
+        """
+        cached = self.refs.get_peeled(ref)
+        if cached is not None:
+            return cached
+        obj = self[ref]
+        obj_type = num_type_map[obj.type]
+        while obj_type == Tag:
+            obj_type, sha = obj.object
+            obj = self.get_object(sha)
+        return obj.id
+
     def get_blob(self, sha):
         return self._get_object(sha, Blob)
 

dulwich/server.py

@@ -41,6 +41,7 @@ from dulwich.protocol import (
     Protocol,
     ProtocolFile,
     TCP_GIT_PORT,
+    ZERO_SHA,
     extract_capabilities,
     extract_want_line_capabilities,
     SINGLE_ACK,
@@ -65,19 +66,23 @@ class Backend(object):
         """
         raise NotImplementedError
 
-    def apply_pack(self, refs, read):
+    def apply_pack(self, refs, read, delete_refs=True):
         """ Import a set of changes into a repository and update the refs
 
         :param refs: list of tuple(name, sha)
         :param read: callback to read from the incoming pack
+        :param delete_refs: whether to allow deleting refs
         """
         raise NotImplementedError
 
-    def fetch_objects(self, determine_wants, graph_walker, progress):
+    def fetch_objects(self, determine_wants, graph_walker, progress,
+                      get_tagged=None):
         """
         Yield the objects required for a list of commits.
 
         :param progress: is a callback to send progress messages to the client
+        :param get_tagged: Function that returns a dict of pointed-to sha -> tag
+            sha for including tags.
         """
         raise NotImplementedError
 
@@ -88,11 +93,12 @@ class GitBackend(Backend):
         if repo is None:
             repo = Repo(tmpfile.mkdtemp())
         self.repo = repo
+        self.refs = self.repo.refs
         self.object_store = self.repo.object_store
         self.fetch_objects = self.repo.fetch_objects
         self.get_refs = self.repo.get_refs
 
-    def apply_pack(self, refs, read):
+    def apply_pack(self, refs, read, delete_refs=True):
         f, commit = self.repo.object_store.add_thin_pack()
         all_exceptions = (IOError, OSError, ChecksumMismatch, ApplyDeltaError)
         status = []
@@ -120,10 +126,13 @@ class GitBackend(Backend):
             status.append(('unpack', 'ok'))
 
         for oldsha, sha, ref in refs:
-            # TODO: check refname
             ref_error = None
             try:
-                if ref == "0" * 40:
+                if sha == ZERO_SHA:
+                    if not delete_refs:
+                        raise GitProtocolError(
+                          'Attempted to delete refs without delete-refs '
+                          'capability.')
                     try:
                         del self.repo.refs[ref]
                     except all_exceptions:
@@ -159,12 +168,24 @@ class Handler(object):
     def capabilities(self):
         raise NotImplementedError(self.capabilities)
 
+    def innocuous_capabilities(self):
+        return ("include-tag", "thin-pack", "no-progress", "ofs-delta")
+
+    def required_capabilities(self):
+        """Return a list of capabilities that we require the client to have."""
+        return []
+
     def set_client_capabilities(self, caps):
-        my_caps = self.capabilities()
+        allowable_caps = set(self.innocuous_capabilities())
+        allowable_caps.update(self.capabilities())
         for cap in caps:
-            if cap not in my_caps:
+            if cap not in allowable_caps:
                 raise GitProtocolError('Client asked for capability %s that '
                                        'was not advertised.' % cap)
+        for cap in self.required_capabilities():
+            if cap not in caps:
+                raise GitProtocolError('Client does not support required '
+                                       'capability %s.' % cap)
         self._client_capabilities = set(caps)
 
     def has_capability(self, cap):
@@ -186,19 +207,52 @@ class UploadPackHandler(Handler):
 
     def capabilities(self):
         return ("multi_ack_detailed", "multi_ack", "side-band-64k", "thin-pack",
-                "ofs-delta", "no-progress")
+                "ofs-delta", "no-progress", "include-tag")
+
+    def required_capabilities(self):
+        return ("side-band-64k", "thin-pack", "ofs-delta")
 
     def progress(self, message):
         if self.has_capability("no-progress"):
             return
         self.proto.write_sideband(2, message)
 
+    def get_tagged(self, refs=None, repo=None):
+        """Get a dict of peeled values of tags to their original tag shas.
+
+        :param refs: dict of refname -> sha of possible tags; defaults to all of
+            the backend's refs.
+        :param repo: optional Repo instance for getting peeled refs; defaults to
+            the backend's repo, if available
+        :return: dict of peeled_sha -> tag_sha, where tag_sha is the sha of a
+            tag whose peeled value is peeled_sha.
+        """
+        if not self.has_capability("include-tag"):
+            return {}
+        if refs is None:
+            refs = self.backend.get_refs()
+        if repo is None:
+            repo = getattr(self.backend, "repo", None)
+            if repo is None:
+                # Bail if we don't have a Repo available; this is ok since
+                # clients must be able to handle if the server doesn't include
+                # all relevant tags.
+                # TODO: either guarantee a Repo, or fix behavior when missing
+                return {}
+        tagged = {}
+        for name, sha in refs.iteritems():
+            peeled_sha = repo.get_peeled(name)
+            if peeled_sha != sha:
+                tagged[peeled_sha] = sha
+        return tagged
+
     def handle(self):
         write = lambda x: self.proto.write_sideband(1, x)
 
         graph_walker = ProtocolGraphWalker(self)
         objects_iter = self.backend.fetch_objects(
-          graph_walker.determine_wants, graph_walker, self.progress)
+          graph_walker.determine_wants, graph_walker, self.progress,
+          get_tagged=self.get_tagged)
 
         # Do they want any objects?
         if len(objects_iter) == 0:
@@ -258,7 +312,10 @@ class ProtocolGraphWalker(object):
                 if not i:
                     line = "%s\x00%s" % (line, self.handler.capability_line())
                 self.proto.write_pkt_line("%s\n" % line)
-                # TODO: include peeled value of any tags
+                peeled_sha = self.handler.backend.repo.get_peeled(ref)
+                if peeled_sha != sha:
+                    self.proto.write_pkt_line('%s %s^{}\n' %
+                                              (peeled_sha, ref))
 
             # i'm done..
             self.proto.write_pkt_line(None)
@@ -508,12 +565,6 @@ class ReceivePackHandler(Handler):
         self.stateless_rpc = stateless_rpc
         self.advertise_refs = advertise_refs
 
-    def __init__(self, backend, read, write,
-                 stateless_rpc=False, advertise_refs=False):
-        Handler.__init__(self, backend, read, write)
-        self._stateless_rpc = stateless_rpc
-        self._advertise_refs = advertise_refs
-
     def capabilities(self):
         return ("report-status", "delete-refs")
 
@@ -523,13 +574,14 @@ class ReceivePackHandler(Handler):
         if self.advertise_refs or not self.stateless_rpc:
             if refs:
                 self.proto.write_pkt_line(
-                    "%s %s\x00%s\n" % (refs[0][1], refs[0][0],
-                                       self.capability_line()))
+                  "%s %s\x00%s\n" % (refs[0][1], refs[0][0],
+                                     self.capability_line()))
                 for i in range(1, len(refs)):
                     ref = refs[i]
                     self.proto.write_pkt_line("%s %s\n" % (ref[1], ref[0]))
             else:
-                self.proto.write_pkt_line("0000000000000000000000000000000000000000 capabilities^{} %s" % self.capability_line())
+                self.proto.write_pkt_line("%s capabilities^{} %s" % (
+                  ZERO_SHA, self.capability_line()))
 
             self.proto.write("0000")
             if self.advertise_refs:
@@ -551,7 +603,8 @@ class ReceivePackHandler(Handler):
             ref = self.proto.read_pkt_line()
 
         # backend can now deal with this refs and read a pack using self.read
-        status = self.backend.apply_pack(client_refs, self.proto.read)
+        status = self.backend.apply_pack(client_refs, self.proto.read,
+                                         self.has_capability('delete-refs'))
 
         # when we have read all the pack from the client, send a status report
         # if the client asked for it

dulwich/tests/compat/server_utils.py

@@ -0,0 +1,169 @@
+# server_utils.py -- Git server compatibility utilities
+# Copyright (C) 2010 Google, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) any later version of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Utilities for testing git server compatibility."""
+
+
+import select
+import socket
+import threading
+
+from dulwich.tests.utils import (
+    tear_down_repo,
+    )
+from utils import (
+    import_repo,
+    run_git,
+    )
+
+
+class ServerTests(object):
+    """Base tests for testing servers.
+
+    Does not inherit from TestCase so tests are not automatically run.
+    """
+
+    def setUp(self):
+        self._old_repo = import_repo('server_old.export')
+        self._new_repo = import_repo('server_new.export')
+        self._server = None
+
+    def tearDown(self):
+        if self._server is not None:
+            self._server.shutdown()
+            self._server = None
+        tear_down_repo(self._old_repo)
+        tear_down_repo(self._new_repo)
+
+    def assertReposEqual(self, repo1, repo2):
+        self.assertEqual(repo1.get_refs(), repo2.get_refs())
+        self.assertEqual(set(repo1.object_store), set(repo2.object_store))
+
+    def assertReposNotEqual(self, repo1, repo2):
+        refs1 = repo1.get_refs()
+        objs1 = set(repo1.object_store)
+        refs2 = repo2.get_refs()
+        objs2 = set(repo2.object_store)
+
+        self.assertFalse(refs1 == refs2 and objs1 == objs2)
+
+    def test_push_to_dulwich(self):
+        self.assertReposNotEqual(self._old_repo, self._new_repo)
+        port = self._start_server(self._old_repo)
+
+        all_branches = ['master', 'branch']
+        branch_args = ['%s:%s' % (b, b) for b in all_branches]
+        url = '%s://localhost:%s/' % (self.protocol, port)
+        returncode, _ = run_git(['push', url] + branch_args,
+                                cwd=self._new_repo.path)
+        self.assertEqual(0, returncode)
+        self.assertReposEqual(self._old_repo, self._new_repo)
+
+    def test_fetch_from_dulwich(self):
+        self.assertReposNotEqual(self._old_repo, self._new_repo)
+        port = self._start_server(self._new_repo)
+
+        all_branches = ['master', 'branch']
+        branch_args = ['%s:%s' % (b, b) for b in all_branches]
+        url = '%s://localhost:%s/' % (self.protocol, port)
+        returncode, _ = run_git(['fetch', url] + branch_args,
+                                cwd=self._old_repo.path)
+        # flush the pack cache so any new packs are picked up
+        self._old_repo.object_store._pack_cache = None
+        self.assertEqual(0, returncode)
+        self.assertReposEqual(self._old_repo, self._new_repo)
+
+
+class ShutdownServerMixIn:
+    """Mixin that allows serve_forever to be shut down.
+
+    The methods in this mixin are backported from SocketServer.py in the Python
+    2.6.4 standard library. The mixin is unnecessary in 2.6 and later, when
+    BaseServer supports the shutdown method directly.
+    """
+
+    def __init__(self):
+        self.__is_shut_down = threading.Event()
+        self.__serving = False
+
+    def serve_forever(self, poll_interval=0.5):
+        """Handle one request at a time until shutdown.
+
+        Polls for shutdown every poll_interval seconds. Ignores
+        self.timeout. If you need to do periodic tasks, do them in
+        another thread.
+        """
+        self.__serving = True
+        self.__is_shut_down.clear()
+        while self.__serving:
+            # XXX: Consider using another file descriptor or
+            # connecting to the socket to wake this up instead of
+            # polling. Polling reduces our responsiveness to a
+            # shutdown request and wastes cpu at all other times.
+            r, w, e = select.select([self], [], [], poll_interval)
+            if r:
+                self._handle_request_noblock()
+        self.__is_shut_down.set()
+
+    serve = serve_forever  # override alias from TCPGitServer
+
+    def shutdown(self):
+        """Stops the serve_forever loop.
+
+        Blocks until the loop has finished. This must be called while
+        serve_forever() is running in another thread, or it will deadlock.
+        """
+        self.__serving = False
+        self.__is_shut_down.wait()
+
+    def handle_request(self):
+        """Handle one request, possibly blocking.
+
+        Respects self.timeout.
+        """
+        # Support people who used socket.settimeout() to escape
+        # handle_request before self.timeout was available.
+        timeout = self.socket.gettimeout()
+        if timeout is None:
+            timeout = self.timeout
+        elif self.timeout is not None:
+            timeout = min(timeout, self.timeout)
+        fd_sets = select.select([self], [], [], timeout)
+        if not fd_sets[0]:
+            self.handle_timeout()
+            return
+        self._handle_request_noblock()
+
+    def _handle_request_noblock(self):
+        """Handle one request, without blocking.
+
+        I assume that select.select has returned that the socket is
+        readable before this function was called, so there should be
+        no risk of blocking in get_request().
+        """
+        try:
+            request, client_address = self.get_request()
+        except socket.error:
+            return
+        if self.verify_request(request, client_address):
+            try:
+                self.process_request(request, client_address)
+            except:
+                self.handle_error(request, client_address)
+                self.close_request(request)

dulwich/tests/compat/test_pack.py

@@ -0,0 +1,74 @@
+# test_pack.py -- Compatibilty tests for git packs.
+# Copyright (C) 2010 Google, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) any later version of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Compatibilty tests for git packs."""
+
+
+import binascii
+import os
+import shutil
+import tempfile
+
+from dulwich.pack import (
+    Pack,
+    write_pack,
+    )
+from dulwich.tests.test_pack import (
+    pack1_sha,
+    PackTests,
+    )
+from utils import (
+    require_git_version,
+    run_git,
+    )
+
+
+class TestPack(PackTests):
+    """Compatibility tests for reading and writing pack files."""
+
+    def setUp(self):
+        require_git_version((1, 5, 0))
+        PackTests.setUp(self)
+        self._tempdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self._tempdir)
+        PackTests.tearDown(self)
+
+    def test_copy(self):
+        origpack = self.get_pack(pack1_sha)
+        self.assertEquals(True, origpack.index.check())
+        pack_path = os.path.join(self._tempdir, "Elch")
+        write_pack(pack_path, [(x, "") for x in origpack.iterobjects()],
+                   len(origpack))
+
+        returncode, output = run_git(['verify-pack', '-v', pack_path],
+                                     capture_stdout=True)
+        self.assertEquals(0, returncode)
+
+        pack_shas = set()
+        for line in output.splitlines():
+            sha = line[:40]
+            try:
+                binascii.unhexlify(sha)
+            except TypeError:
+                continue  # non-sha line
+            pack_shas.add(sha)
+        orig_shas = set(o.id for o in origpack.iterobjects())
+        self.assertEquals(orig_shas, pack_shas)

dulwich/tests/compat/test_server.py

@@ -0,0 +1,77 @@
+# test_server.py -- Compatibilty tests for git server.
+# Copyright (C) 2010 Google, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) any later version of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Compatibilty tests between Dulwich and the cgit server.
+
+Warning: these tests should be fairly stable, but when writing/debugging new
+tests, deadlocks may freeze the test process such that it cannot be Ctrl-C'ed.
+On *nix, you can kill the tests with Ctrl-Z, "kill %".
+"""
+
+import threading
+import unittest
+
+import nose
+
+from dulwich import server
+from server_utils import (
+    ServerTests,
+    ShutdownServerMixIn,
+    )
+from utils import (
+    CompatTestCase,
+    )
+
+
+if getattr(server.TCPGitServer, 'shutdown', None):
+    TCPGitServer = server.TCPGitServer
+else:
+    class TCPGitServer(ShutdownServerMixIn, server.TCPGitServer):
+        """Subclass of TCPGitServer that can be shut down."""
+
+        def __init__(self, *args, **kwargs):
+            # BaseServer is old-style so we have to call both __init__s
+            ShutdownServerMixIn.__init__(self)
+            server.TCPGitServer.__init__(self, *args, **kwargs)
+
+        serve = ShutdownServerMixIn.serve_forever
+
+
+class GitServerTestCase(ServerTests, CompatTestCase):
+    """Tests for client/server compatibility."""
+
+    protocol = 'git'
+
+    def setUp(self):
+        ServerTests.setUp(self)
+        CompatTestCase.setUp(self)
+
+    def tearDown(self):
+        ServerTests.tearDown(self)
+        CompatTestCase.tearDown(self)
+
+    def _start_server(self, repo):
+        dul_server = TCPGitServer(server.GitBackend(repo), 'localhost', 0)
+        threading.Thread(target=dul_server.serve).start()
+        self._server = dul_server
+        _, port = self._server.socket.getsockname()
+        return port
+
+    def test_push_to_dulwich(self):
+        raise nose.SkipTest('Skipping push test due to known deadlock bug.')

dulwich/tests/compat/test_web.py

@@ -0,0 +1,127 @@
+# test_web.py -- Compatibilty tests for the git web server.
+# Copyright (C) 2010 Google, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) any later version of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Compatibilty tests between Dulwich and the cgit HTTP server.
+
+Warning: these tests should be fairly stable, but when writing/debugging new
+tests, deadlocks may freeze the test process such that it cannot be Ctrl-C'ed.
+On *nix, you can kill the tests with Ctrl-Z, "kill %".
+"""
+
+import sys
+import threading
+import unittest
+from wsgiref import simple_server
+
+import nose
+
+from dulwich.repo import (
+    Repo,
+    )
+from dulwich.server import (
+    GitBackend,
+    )
+from dulwich.web import (
+    HTTPGitApplication,
+    )
+
+from dulwich.tests.utils import (
+    open_repo,
+    tear_down_repo,
+    )
+from server_utils import (
+    ServerTests,
+    ShutdownServerMixIn,
+    )
+from utils import (
+    CompatTestCase,
+    )
+
+
+if getattr(simple_server.WSGIServer, 'shutdown', None):
+    WSGIServer = simple_server.WSGIServer
+else:
+    class WSGIServer(ShutdownServerMixIn, simple_server.WSGIServer):
+        """Subclass of WSGIServer that can be shut down."""
+
+        def __init__(self, *args, **kwargs):
+            # BaseServer is old-style so we have to call both __init__s
+            ShutdownServerMixIn.__init__(self)
+            simple_server.WSGIServer.__init__(self, *args, **kwargs)
+
+        serve = ShutdownServerMixIn.serve_forever
+
+
+class WebTests(ServerTests):
+    """Base tests for web server tests.
+
+    Contains utility and setUp/tearDown methods, but does non inherit from
+    TestCase so tests are not automatically run.
+    """
+
+    protocol = 'http'
+
+    def _start_server(self, repo):
+        app = self._make_app(GitBackend(repo))
+        dul_server = simple_server.make_server('localhost', 0, app,
+                                               server_class=WSGIServer)
+        threading.Thread(target=dul_server.serve_forever).start()
+        self._server = dul_server
+        _, port = dul_server.socket.getsockname()
+        return port
+
+
+class SmartWebTestCase(WebTests, CompatTestCase):
+    """Test cases for smart HTTP server."""
+
+    min_git_version = (1, 6, 6)
+
+    def setUp(self):
+        WebTests.setUp(self)
+        CompatTestCase.setUp(self)
+
+    def tearDown(self):
+        WebTests.tearDown(self)
+        CompatTestCase.tearDown(self)
+
+    def _make_app(self, backend):
+        return HTTPGitApplication(backend)
+
+    def test_push_to_dulwich(self):
+        # TODO(dborowitz): enable after merging thin pack fixes.
+        raise nose.SkipTest('Skipping push test due to known pack bug.')
+
+
+class DumbWebTestCase(WebTests, CompatTestCase):
+    """Test cases for dumb HTTP server."""
+
+    def setUp(self):
+        WebTests.setUp(self)
+        CompatTestCase.setUp(self)
+
+    def tearDown(self):
+        WebTests.tearDown(self)
+        CompatTestCase.tearDown(self)
+
+    def _make_app(self, backend):
+        return HTTPGitApplication(backend, dumb=True)
+
+    def test_push_to_dulwich(self):
+        # Note: remove this if dumb pushing is supported
+        raise nose.SkipTest('Dumb web pushing not supported.')

dulwich/tests/compat/utils.py

@@ -0,0 +1,143 @@
+# utils.py -- Git compatibility utilities
+# Copyright (C) 2010 Google, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) any later version of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Utilities for interacting with cgit."""
+
+import os
+import subprocess
+import tempfile
+import unittest
+
+import nose
+
+from dulwich.repo import Repo
+from dulwich.tests.utils import open_repo
+
+
+_DEFAULT_GIT = 'git'
+
+
+def git_version(git_path=_DEFAULT_GIT):
+    """Attempt to determine the version of git currently installed.
+
+    :param git_path: Path to the git executable; defaults to the version in
+        the system path.
+    :return: A tuple of ints of the form (major, minor, point), or None if no
+        git installation was found.
+    """
+    try:
+        _, output = run_git(['--version'], git_path=git_path,
+                            capture_stdout=True)
+    except OSError:
+        return None
+    version_prefix = 'git version '
+    if not output.startswith(version_prefix):
+        return None
+    output = output[len(version_prefix):]
+    nums = output.split('.')
+    if len(nums) == 2:
+        nums.add('0')
+    else:
+        nums = nums[:3]
+    try:
+        return tuple(int(x) for x in nums)
+    except ValueError:
+        return None
+
+
+def require_git_version(required_version, git_path=_DEFAULT_GIT):
+    """Require git version >= version, or skip the calling test."""
+    found_version = git_version(git_path=git_path)
+    if found_version < required_version:
+        required_version = '.'.join(map(str, required_version))
+        found_version = '.'.join(map(str, found_version))
+        raise nose.SkipTest('Test requires git >= %s, found %s' %
+                            (required_version, found_version))
+
+
+def run_git(args, git_path=_DEFAULT_GIT, input=None, capture_stdout=False,
+            **popen_kwargs):
+    """Run a git command.
+
+    Input is piped from the input parameter and output is sent to the standard
+    streams, unless capture_stdout is set.
+
+    :param args: A list of args to the git command.
+    :param git_path: Path to to the git executable.
+    :param input: Input data to be sent to stdin.
+    :param capture_stdout: Whether to capture and return stdout.
+    :param popen_kwargs: Additional kwargs for subprocess.Popen;
+        stdin/stdout args are ignored.
+    :return: A tuple of (returncode, stdout contents). If capture_stdout is
+        False, None will be returned as stdout contents.
+    :raise OSError: if the git executable was not found.
+    """
+    args = [git_path] + args
+    popen_kwargs['stdin'] = subprocess.PIPE
+    if capture_stdout:
+        popen_kwargs['stdout'] = subprocess.PIPE
+    else:
+        popen_kwargs.pop('stdout', None)
+    p = subprocess.Popen(args, **popen_kwargs)
+    stdout, stderr = p.communicate(input=input)
+    return (p.returncode, stdout)
+
+
+def run_git_or_fail(args, git_path=_DEFAULT_GIT, input=None, **popen_kwargs):
+    """Run a git command, capture stdout/stderr, and fail if git fails."""
+    popen_kwargs['stderr'] = subprocess.STDOUT
+    returncode, stdout = run_git(args, git_path=git_path, input=input,
+                                 capture_stdout=True, **popen_kwargs)
+    assert returncode == 0
+    return stdout
+
+
+def import_repo(name):
+    """Import a repo from a fast-export file in a temporary directory.
+
+    These are used rather than binary repos for compat tests because they are
+    more compact an human-editable, and we already depend on git.
+
+    :param name: The name of the repository export file, relative to
+        dulwich/tests/data/repos
+    :returns: An initialized Repo object that lives in a temporary directory.
+    """
+    temp_dir = tempfile.mkdtemp()
+    export_path = os.path.join(os.path.dirname(__file__), os.pardir, 'data',
+                               'repos', name)
+    temp_repo_dir = os.path.join(temp_dir, name)
+    export_file = open(export_path, 'rb')
+    run_git_or_fail(['init', '--bare', temp_repo_dir])
+    run_git_or_fail(['fast-import'], input=export_file.read(),
+                    cwd=temp_repo_dir)
+    export_file.close()
+    return Repo(temp_repo_dir)
+
+
+class CompatTestCase(unittest.TestCase):
+    """Test case that requires git for compatibility checks.
+
+    Subclasses can change the git version required by overriding
+    min_git_version.
+    """
+
+    min_git_version = (1, 5, 0)
+
+    def setUp(self):
+        require_git_version(self.min_git_version)

dulwich/tests/data/repos/a.git/objects/28/237f4dc30d0d462658d6b937b08a0f0b6ef55a

@@ -0,0 +1,2 @@
+x
5ÌA
+Â0…a×9Å\@™¦i›�""ÁLÚ1T"uPêéMA7�oó~å•ó»î2(0á�íHˆ\uB\]ÛMÞN‚c+ÄH�Ñõ!0ä”&5Zi-»)Ê~	œó’ß“~ Ã�§˜sœåP~G¨lÛÖ®Á†`�јkéÌüÔ÷ÀN0—

dulwich/tests/data/repos/a.git/objects/b0/931cadc54336e78a1d980420e3268903b57a50

@@ -0,0 +1,3 @@
+x
-�[
+Β0ύΞ*ξ*IΜ��Έ7�Η5T[o©΅RWo†Γΐ™
+­wο�*Θ`eφ�/“Ωi­·7sΰΒjƒpΑθμ«Ϋ��h�†ΚjkL[c7‡τΐόΈ„αL½‡ϊ�>Η�<Ά2βΎέ� ¤1JrηtάqΞΨµεhΜ°βςθΙΎΦ¥2v

dulwich/tests/data/repos/a.git/packed-refs

@@ -0,0 +1,3 @@
+# pack-refs with: peeled 
+b0931cadc54336e78a1d980420e3268903b57a50 refs/tags/mytag-packed
+^2a72d929692c41d8554c07f6301757ba18a65d91

dulwich/tests/data/repos/a.git/refs/tags/mytag

@@ -0,0 +1 @@
+28237f4dc30d0d462658d6b937b08a0f0b6ef55a

dulwich/tests/data/repos/refs.git/objects/3e/c9c43c84ff242e3ef4a9fc5bc111fd780a76a8

@@ -0,0 +1,3 @@
+x
-�Q
+Â0DýÎ)ö-›mšVñ^ i6±Ò6’.ŠžÞ~ÍÌÇ{#Cm›]rwy´Î×u�=’uº5^[³o¸õ<¸®H<*y?Æ´,“()ŽÌa«°¦ßˆœá2<Î)§×$8x÷¯§˜Rœ¹.è4Ykˆt�Pa�¨Ôµ¨
+q?…À™W)'«”ÜÔǧ6

dulwich/tests/data/repos/refs.git/objects/cd/a609072918d7b70057b6bef9f4c2537843fcfe

@@ -0,0 +1,5 @@
+x
-ŤQ
+Â0DýÎ)öm7i’Vń^ i6±bIEOo
+~Íc`ŢđAví.ą;ŤZy´Îk×u�<*ë¤Ń^Z˝oÉx\×T4
+ţ<	Ć4Ď.ŽLam
+¬ÍFÖj«#e¸/‚sĘé=ńŢýńSŠŞ‹äRY«•Bc ÂQ�k‘–Ĺ
üeZ¸Ü-\r?)Y9Ţ

dulwich/tests/data/repos/refs.git/packed-refs

@@ -1,3 +1,4 @@
 # pack-refs with: peeled 
 df6800012397fb85c56e7418dd4eb9405dee075c refs/tags/refs-0.1
 ^42d06bd4b77fed026b154d16493e5deab78f02ec
+42d06bd4b77fed026b154d16493e5deab78f02ec refs/heads/packed

dulwich/tests/data/repos/refs.git/refs/tags/refs-0.2

@@ -0,0 +1 @@
+3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8

dulwich/tests/data/repos/server_new.export

@@ -0,0 +1,99 @@
+blob
+mark :1
+data 13
+foo contents
+
+reset refs/heads/master
+commit refs/heads/master
+mark :2
+author Dave Borowitz <dborowitz@google.com> 1265755064 -0800
+committer Dave Borowitz <dborowitz@google.com> 1265755064 -0800
+data 16
+initial checkin
+M 100644 :1 foo
+
+blob
+mark :3
+data 13
+baz contents
+
+blob
+mark :4
+data 21
+updated foo contents
+
+commit refs/heads/master
+mark :5
+author Dave Borowitz <dborowitz@google.com> 1265755140 -0800
+committer Dave Borowitz <dborowitz@google.com> 1265755140 -0800
+data 15
+master checkin
+from :2
+M 100644 :3 baz
+M 100644 :4 foo
+
+blob
+mark :6
+data 24
+updated foo contents v2
+
+commit refs/heads/master
+mark :7
+author Dave Borowitz <dborowitz@google.com> 1265755287 -0800
+committer Dave Borowitz <dborowitz@google.com> 1265755287 -0800
+data 17
+master checkin 2
+from :5
+M 100644 :6 foo
+
+blob
+mark :8
+data 24
+updated foo contents v3
+
+commit refs/heads/master
+mark :9
+author Dave Borowitz <dborowitz@google.com> 1265755295 -0800
+committer Dave Borowitz <dborowitz@google.com> 1265755295 -0800
+data 17
+master checkin 3
+from :7
+M 100644 :8 foo
+
+blob
+mark :10
+data 22
+branched bar contents
+
+blob
+mark :11
+data 22
+branched foo contents
+
+commit refs/heads/branch
+mark :12
+author Dave Borowitz <dborowitz@google.com> 1265755111 -0800
+committer Dave Borowitz <dborowitz@google.com> 1265755111 -0800
+data 15
+branch checkin
+from :2
+M 100644 :10 bar
+M 100644 :11 foo
+
+blob
+mark :13
+data 25
+branched bar contents v2
+
+commit refs/heads/branch
+mark :14
+author Dave Borowitz <dborowitz@google.com> 1265755319 -0800
+committer Dave Borowitz <dborowitz@google.com> 1265755319 -0800
+data 17
+branch checkin 2
+from :12
+M 100644 :13 bar
+
+reset refs/heads/master
+from :9
+

dulwich/tests/data/repos/server_old.export

@@ -0,0 +1,57 @@
+blob
+mark :1
+data 13
+foo contents
+
+reset refs/heads/master
+commit refs/heads/master
+mark :2
+author Dave Borowitz <dborowitz@google.com> 1265755064 -0800
+committer Dave Borowitz <dborowitz@google.com> 1265755064 -0800
+data 16
+initial checkin
+M 100644 :1 foo
+
+blob
+mark :3
+data 22
+branched bar contents
+
+blob
+mark :4
+data 22
+branched foo contents
+
+commit refs/heads/branch
+mark :5
+author Dave Borowitz <dborowitz@google.com> 1265755111 -0800
+committer Dave Borowitz <dborowitz@google.com> 1265755111 -0800
+data 15
+branch checkin
+from :2
+M 100644 :3 bar
+M 100644 :4 foo
+
+blob
+mark :6
+data 13
+baz contents
+
+blob
+mark :7
+data 21
+updated foo contents
+
+commit refs/heads/master
+mark :8
+author Dave Borowitz <dborowitz@google.com> 1265755140 -0800
+committer Dave Borowitz <dborowitz@google.com> 1265755140 -0800
+data 15
+master checkin
+from :2
+M 100644 :6 baz
+M 100644 :7 foo
+
+reset refs/heads/master
+from :8
+

dulwich/tests/test_object_store.py

@@ -99,3 +99,6 @@ class DiskObjectStoreTests(ObjectStoreTests,TestCase):
             shutil.rmtree("foo")
         os.makedirs(os.path.join("foo", "pack"))
         self.store = DiskObjectStore("foo")
+
+
+# TODO: MissingObjectFinderTests

dulwich/tests/test_repository.py

@@ -27,6 +27,7 @@ import tempfile
 import unittest
 
 from dulwich import errors
+from dulwich import objects
 from dulwich.repo import (
     check_ref_format,
     Repo,
@@ -35,34 +36,14 @@ from dulwich.repo import (
     write_packed_refs,
     _split_ref_line,
     )
+from dulwich.tests.utils import (
+    open_repo,
+    tear_down_repo,
+    )
 
 missing_sha = 'b91fa4d900e17e99b433218e988c4eb4a3e9a097'
 
 
-def open_repo(name):
-    """Open a copy of a repo in a temporary directory.
-
-    Use this function for accessing repos in dulwich/tests/data/repos to avoid
-    accidentally or intentionally modifying those repos in place. Use
-    tear_down_repo to delete any temp files created.
-
-    :param name: The name of the repository, relative to
-        dulwich/tests/data/repos
-    :returns: An initialized Repo object that lives in a temporary directory.
-    """
-    temp_dir = tempfile.mkdtemp()
-    repo_dir = os.path.join(os.path.dirname(__file__), 'data', 'repos', name)
-    temp_repo_dir = os.path.join(temp_dir, name)
-    shutil.copytree(repo_dir, temp_repo_dir, symlinks=True)
-    return Repo(temp_repo_dir)
-
-def tear_down_repo(repo):
-    """Tear down a test repository."""
-    temp_dir = os.path.dirname(repo.path.rstrip(os.sep))
-    shutil.rmtree(temp_dir)
-
-
-
 class CreateRepositoryTests(unittest.TestCase):
 
     def test_create(self):
@@ -82,7 +63,7 @@ class RepositoryTests(unittest.TestCase):
     def tearDown(self):
         if self._repo is not None:
             tear_down_repo(self._repo)
-
+  
     def test_simple_props(self):
         r = self._repo = open_repo('a.git')
         self.assertEqual(r.controldir(), r.path)
@@ -95,8 +76,10 @@ class RepositoryTests(unittest.TestCase):
     def test_get_refs(self):
         r = self._repo = open_repo('a.git')
         self.assertEqual({
-            'HEAD': 'a90fa2d900a17e99b433217e988c4eb4a2e9a097', 
-            'refs/heads/master': 'a90fa2d900a17e99b433217e988c4eb4a2e9a097'
+            'HEAD': 'a90fa2d900a17e99b433217e988c4eb4a2e9a097',
+            'refs/heads/master': 'a90fa2d900a17e99b433217e988c4eb4a2e9a097',
+            'refs/tags/mytag': '28237f4dc30d0d462658d6b937b08a0f0b6ef55a',
+            'refs/tags/mytag-packed': 'b0931cadc54336e78a1d980420e3268903b57a50',
             }, r.get_refs())
   
     def test_head(self):
@@ -132,7 +115,40 @@ class RepositoryTests(unittest.TestCase):
     def test_tree_not_tree(self):
         r = self._repo = open_repo('a.git')
         self.assertRaises(errors.NotTreeError, r.tree, r.head())
-  
+
+    def test_tag(self):
+        r = self._repo = open_repo('a.git')
+        tag_sha = '28237f4dc30d0d462658d6b937b08a0f0b6ef55a'
+        tag = r.tag(tag_sha)
+        self.assertEqual(tag._type, 'tag')
+        self.assertEqual(tag.sha().hexdigest(), tag_sha)
+        obj_type, obj_sha = tag.object
+        self.assertEqual(obj_type, objects.Commit)
+        self.assertEqual(obj_sha, r.head())
+
+    def test_tag_not_tag(self):
+        r = self._repo = open_repo('a.git')
+        self.assertRaises(errors.NotTagError, r.tag, r.head())
+
+    def test_get_peeled(self):
+        # unpacked ref
+        r = self._repo = open_repo('a.git')
+        tag_sha = '28237f4dc30d0d462658d6b937b08a0f0b6ef55a'
+        self.assertNotEqual(r[tag_sha].sha().hexdigest(), r.head())
+        self.assertEqual(r.get_peeled('refs/tags/mytag'), r.head())
+
+        # packed ref with cached peeled value
+        packed_tag_sha = 'b0931cadc54336e78a1d980420e3268903b57a50'
+        parent_sha = r[r.head()].parents[0]
+        self.assertNotEqual(r[packed_tag_sha].sha().hexdigest(), parent_sha)
+        self.assertEqual(r.get_peeled('refs/tags/mytag-packed'), parent_sha)
+
+        # TODO: add more corner cases to test repo
+
+    def test_get_peeled_not_tag(self):
+        r = self._repo = open_repo('a.git')
+        self.assertEqual(r.get_peeled('HEAD'), r.head())
+
     def test_get_blob(self):
         r = self._repo = open_repo('a.git')
         commit = r.commit(r.head())
@@ -219,7 +235,6 @@ THREES = "3" * 40
 FOURS = "4" * 40
 
 class PackedRefsFileTests(unittest.TestCase):
-
     def test_split_ref_line_errors(self):
         self.assertRaises(errors.PackedRefsException, _split_ref_line,
                           'singlefield')
@@ -268,7 +283,6 @@ class PackedRefsFileTests(unittest.TestCase):
 
 
 class RefsContainerTests(unittest.TestCase):
-
     def setUp(self):
         self._repo = open_repo('refs.git')
         self._refs = self._repo.refs
@@ -277,27 +291,47 @@ class RefsContainerTests(unittest.TestCase):
         tear_down_repo(self._repo)
 
     def test_get_packed_refs(self):
-        self.assertEqual(
-            {'refs/tags/refs-0.1': 'df6800012397fb85c56e7418dd4eb9405dee075c'},
-            self._refs.get_packed_refs())
+        self.assertEqual({
+            'refs/heads/packed': '42d06bd4b77fed026b154d16493e5deab78f02ec',
+            'refs/tags/refs-0.1': 'df6800012397fb85c56e7418dd4eb9405dee075c',
+            }, self._refs.get_packed_refs())
+
+    def test_get_peeled_not_packed(self):
+        # not packed
+        self.assertEqual(None, self._refs.get_peeled('refs/tags/refs-0.2'))
+        self.assertEqual('3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8',
+                         self._refs['refs/tags/refs-0.2'])
+
+        # packed, known not peelable
+        self.assertEqual(self._refs['refs/heads/packed'],
+                         self._refs.get_peeled('refs/heads/packed'))
+
+        # packed, peeled
+        self.assertEqual('42d06bd4b77fed026b154d16493e5deab78f02ec',
+                         self._refs.get_peeled('refs/tags/refs-0.1'))
 
     def test_keys(self):
         self.assertEqual([
             'HEAD',
             'refs/heads/loop',
             'refs/heads/master',
+            'refs/heads/packed',
             'refs/tags/refs-0.1',
+            'refs/tags/refs-0.2',
             ], sorted(list(self._refs.keys())))
-        self.assertEqual(['loop', 'master'],
+        self.assertEqual(['loop', 'master', 'packed'],
                          sorted(self._refs.keys('refs/heads')))
-        self.assertEqual(['refs-0.1'], list(self._refs.keys('refs/tags')))
+        self.assertEqual(['refs-0.1', 'refs-0.2'],
+                         sorted(self._refs.keys('refs/tags')))
 
     def test_as_dict(self):
         # refs/heads/loop does not show up
         self.assertEqual({
             'HEAD': '42d06bd4b77fed026b154d16493e5deab78f02ec',
             'refs/heads/master': '42d06bd4b77fed026b154d16493e5deab78f02ec',
+            'refs/heads/packed': '42d06bd4b77fed026b154d16493e5deab78f02ec',
             'refs/tags/refs-0.1': 'df6800012397fb85c56e7418dd4eb9405dee075c',
+            'refs/tags/refs-0.2': '3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8',
             }, self._refs.as_dict())
 
     def test_setitem(self):

dulwich/tests/test_server.py

@@ -40,6 +40,7 @@ TWO = '2' * 40
 THREE = '3' * 40
 FOUR = '4' * 40
 FIVE = '5' * 40
+SIX = '6' * 40
 
 class TestProto(object):
 
@@ -77,24 +78,34 @@ class HandlerTestCase(TestCase):
     def setUp(self):
         self._handler = Handler(None, None, None)
         self._handler.capabilities = lambda: ('cap1', 'cap2', 'cap3')
+        self._handler.required_capabilities = lambda: ('cap2',)
 
     def assertSucceeds(self, func, *args, **kwargs):
         try:
             func(*args, **kwargs)
-        except GitProtocolError:
-            self.fail()
+        except GitProtocolError, e:
+            self.fail(e)
 
     def test_capability_line(self):
         self.assertEquals('cap1 cap2 cap3', self._handler.capability_line())
 
     def test_set_client_capabilities(self):
         set_caps = self._handler.set_client_capabilities
-        self.assertSucceeds(set_caps, [])
         self.assertSucceeds(set_caps, ['cap2'])
         self.assertSucceeds(set_caps, ['cap1', 'cap2'])
+
         # different order
         self.assertSucceeds(set_caps, ['cap3', 'cap1', 'cap2'])
-        self.assertRaises(GitProtocolError, set_caps, ['capxxx', 'cap1'])
+
+        # error cases
+        self.assertRaises(GitProtocolError, set_caps, ['capxxx', 'cap2'])
+        self.assertRaises(GitProtocolError, set_caps, ['cap1', 'cap3'])
+
+        # ignore innocuous but unknown capabilities
+        self.assertRaises(GitProtocolError, set_caps, ['cap2', 'ignoreme'])
+        self.assertFalse('ignoreme' in self._handler.capabilities())
+        self._handler.innocuous_capabilities = lambda: ('ignoreme',)
+        self.assertSucceeds(set_caps, ['cap2', 'ignoreme'])
 
     def test_has_capability(self):
         self.assertRaises(GitProtocolError, self._handler.has_capability, 'cap')
@@ -112,7 +123,8 @@ class UploadPackHandlerTestCase(TestCase):
         self._handler.proto = TestProto()
 
     def test_progress(self):
-        self._handler.set_client_capabilities([])
+        caps = self._handler.required_capabilities()
+        self._handler.set_client_capabilities(caps)
         self._handler.progress('first message')
         self._handler.progress('second message')
         self.assertEqual('first message',
@@ -122,11 +134,37 @@ class UploadPackHandlerTestCase(TestCase):
         self.assertEqual(None, self._handler.proto.get_received_line(2))
 
     def test_no_progress(self):
-        self._handler.set_client_capabilities(['no-progress'])
+        caps = list(self._handler.required_capabilities()) + ['no-progress']
+        self._handler.set_client_capabilities(caps)
         self._handler.progress('first message')
         self._handler.progress('second message')
         self.assertEqual(None, self._handler.proto.get_received_line(2))
 
+    def test_get_tagged(self):
+        refs = {
+            'refs/tags/tag1': ONE,
+            'refs/tags/tag2': TWO,
+            'refs/heads/master': FOUR,  # not a tag, no peeled value
+            }
+        peeled = {
+            'refs/tags/tag1': '1234',
+            'refs/tags/tag2': '5678',
+            }
+
+        class TestRepo(object):
+            def get_peeled(self, ref):
+                return peeled.get(ref, refs[ref])
+
+        caps = list(self._handler.required_capabilities()) + ['include-tag']
+        self._handler.set_client_capabilities(caps)
+        self.assertEquals({'1234': ONE, '5678': TWO},
+                          self._handler.get_tagged(refs, repo=TestRepo()))
+
+        # non-include-tag case
+        caps = self._handler.required_capabilities()
+        self._handler.set_client_capabilities(caps)
+        self.assertEquals({}, self._handler.get_tagged(refs, repo=TestRepo()))
+
 
 class TestCommit(object):
 
@@ -142,16 +180,25 @@ class TestCommit(object):
         return '%s(%s)' % (self.__class__.__name__, self._sha)
 
 
+class TestRepo(object):
+    def __init__(self):
+        self.peeled = {}
+
+    def get_peeled(self, name):
+        return self.peeled[name]
+
+
 class TestBackend(object):
 
-    def __init__(self, objects):
+    def __init__(self, repo, objects):
+        self.repo = repo
         self.object_store = objects
 
 
 class TestUploadPackHandler(Handler):
 
     def __init__(self, objects, proto):
-        self.backend = TestBackend(objects)
+        self.backend = TestBackend(TestRepo(), objects)
         self.proto = proto
         self.stateless_rpc = False
         self.advertise_refs = False
@@ -174,6 +221,7 @@ class ProtocolGraphWalkerTestCase(TestCase):
             FOUR: TestCommit(FOUR, [TWO], 444),
             FIVE: TestCommit(FIVE, [THREE], 555),
             }
+
         self._walker = ProtocolGraphWalker(
             TestUploadPackHandler(self._objects, TestProto()))
 
@@ -227,6 +275,7 @@ class ProtocolGraphWalkerTestCase(TestCase):
             'want %s' % TWO,
             ])
         heads = {'ref1': ONE, 'ref2': TWO, 'ref3': THREE}
+        self._walker.handler.backend.repo.peeled = heads
         self.assertEquals([ONE, TWO], self._walker.determine_wants(heads))
 
         self._walker.proto.set_output(['want %s multi_ack' % FOUR])
@@ -241,6 +290,35 @@ class ProtocolGraphWalkerTestCase(TestCase):
         self._walker.proto.set_output(['want %s multi_ack' % FOUR])
         self.assertRaises(GitProtocolError, self._walker.determine_wants, heads)
 
+    def test_determine_wants_advertisement(self):
+        self._walker.proto.set_output([])
+        # advertise branch tips plus tag
+        heads = {'ref4': FOUR, 'ref5': FIVE, 'tag6': SIX}
+        peeled = {'ref4': FOUR, 'ref5': FIVE, 'tag6': FIVE}
+        self._walker.handler.backend.repo.peeled = peeled
+        self._walker.determine_wants(heads)
+        lines = []
+        while True:
+            line = self._walker.proto.get_received_line()
+            if line == 'None':
+                break
+            # strip capabilities list if present
+            if '\x00' in line:
+                line = line[:line.index('\x00')]
+            lines.append(line.rstrip())
+
+        self.assertEquals([
+            '%s ref4' % FOUR,
+            '%s ref5' % FIVE,
+            '%s tag6^{}' % FIVE,
+            '%s tag6' % SIX,
+            ], sorted(lines))
+
+        # ensure peeled tag was advertised immediately following tag
+        for i, line in enumerate(lines):
+            if line.endswith(' tag6'):
+                self.assertEquals('%s tag6^{}' % FIVE, lines[i+1])
+
     # TODO: test commit time cutoff
 
 

dulwich/tests/test_web.py

@@ -112,13 +112,28 @@ class DumbHandlersTestCase(WebTestCase):
         blob2 = TestBlob('222')
         blob3 = TestBlob('333')
 
-        tag1 = TestTag('aaa', TestTag.type, 'bbb')
-        tag2 = TestTag('bbb', TestBlob.type, '222')
+        tag1 = TestTag('aaa', TestBlob.type, '222')
+
+        class TestRepo(object):
+            def __init__(self, objects, peeled):
+                self._objects = dict((o.sha(), o) for o in objects)
+                self._peeled = peeled
+
+            def get_peeled(self, sha):
+                return self._peeled[sha]
+
+            def __getitem__(self, sha):
+                return self._objects[sha]
 
         class TestBackend(object):
             def __init__(self):
-                objects = [blob1, blob2, blob3, tag1, tag2]
-                self.repo = dict((o.sha(), o) for o in objects)
+                objects = [blob1, blob2, blob3, tag1]
+                self.repo = TestRepo(objects, {
+                    'HEAD': '000',
+                    'refs/heads/master': blob1.sha(),
+                    'refs/tags/tag-tag': blob2.sha(),
+                    'refs/tags/blob-tag': blob3.sha(),
+                    })
 
             def get_refs(self):
                 return {

dulwich/tests/utils.py

@@ -0,0 +1,51 @@
+# utils.py -- Test utilities for Dulwich.
+# Copyright (C) 2010 Google, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) any later version of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Utility functions common to Dulwich tests."""
+
+
+import os
+import shutil
+import tempfile
+
+from dulwich.repo import Repo
+
+
+def open_repo(name):
+    """Open a copy of a repo in a temporary directory.
+
+    Use this function for accessing repos in dulwich/tests/data/repos to avoid
+    accidentally or intentionally modifying those repos in place. Use
+    tear_down_repo to delete any temp files created.
+
+    :param name: The name of the repository, relative to
+        dulwich/tests/data/repos
+    :returns: An initialized Repo object that lives in a temporary directory.
+    """
+    temp_dir = tempfile.mkdtemp()
+    repo_dir = os.path.join(os.path.dirname(__file__), 'data', 'repos', name)
+    temp_repo_dir = os.path.join(temp_dir, name)
+    shutil.copytree(repo_dir, temp_repo_dir, symlinks=True)
+    return Repo(temp_repo_dir)
+
+
+def tear_down_repo(repo):
+    """Tear down a test repository."""
+    temp_dir = os.path.dirname(repo.path.rstrip(os.sep))
+    shutil.rmtree(temp_dir)

dulwich/web.py

@@ -23,10 +23,6 @@ import cgi
 import re
 import time
 
-from dulwich.objects import (
-    Tag,
-    num_type_map,
-    )
 from dulwich.server import (
     ReceivePackHandler,
     UploadPackHandler,
@@ -99,23 +95,23 @@ def get_loose_object(req, backend, mat):
 def get_pack_file(req, backend, mat):
     req.cache_forever()
     return send_file(req, backend.repo.get_named_file(mat.group()),
-                     'application/x-git-packed-objects', False)
+                     'application/x-git-packed-objects')
 
 
 def get_idx_file(req, backend, mat):
     req.cache_forever()
     return send_file(req, backend.repo.get_named_file(mat.group()),
-                     'application/x-git-packed-objects-toc', False)
+                     'application/x-git-packed-objects-toc')
 
 
-services = {'git-upload-pack': UploadPackHandler,
-            'git-receive-pack': ReceivePackHandler}
+default_services = {'git-upload-pack': UploadPackHandler,
+                    'git-receive-pack': ReceivePackHandler}
 def get_info_refs(req, backend, mat, services=None):
     if services is None:
-        services = services
+        services = default_services
     params = cgi.parse_qs(req.environ['QUERY_STRING'])
     service = params.get('service', [None])[0]
-    if service:
+    if service and not req.dumb:
         handler_cls = services.get(service, None)
         if handler_cls is None:
             yield req.forbidden('Unsupported service %s' % service)
@@ -146,15 +142,9 @@ def get_info_refs(req, backend, mat, services=None):
             if not o:
                 continue
             yield '%s\t%s\n' % (sha, name)
-            obj_type = num_type_map[o.type]
-            if obj_type == Tag:
-                while obj_type == Tag:
-                    num_type, sha = o.object
-                    obj_type = num_type_map[num_type]
-                    o = backend.repo[sha]
-                if not o:
-                    continue
-                yield '%s\t%s^{}\n' % (o.sha(), name)
+            peeled_sha = backend.repo.get_peeled(name)
+            if peeled_sha != sha:
+                yield '%s\t%s^{}\n' % (peeled_sha, name)
 
 
 def get_info_packs(req, backend, mat):
@@ -185,9 +175,10 @@ class _LengthLimitedFile(object):
 
     # TODO: support more methods as necessary
 
-def handle_service_request(req, backend, mat, services=services):
+
+def handle_service_request(req, backend, mat, services=None):
     if services is None:
-        services = services
+        services = default_services
     service = mat.group().lstrip('/')
     handler_cls = services.get(service, None)
     if handler_cls is None:
@@ -215,8 +206,9 @@ class HTTPGitRequest(object):
     :ivar environ: the WSGI environment for the request.
     """
 
-    def __init__(self, environ, start_response):
+    def __init__(self, environ, start_response, dumb=False):
         self.environ = environ
+        self.dumb = dumb
         self._start_response = start_response
         self._cache_headers = []
         self._headers = []
@@ -285,13 +277,14 @@ class HTTPGitApplication(object):
         ('POST', re.compile('/git-receive-pack$')): handle_service_request,
     }
 
-    def __init__(self, backend):
+    def __init__(self, backend, dumb=False):
         self.backend = backend
+        self.dumb = dumb
 
     def __call__(self, environ, start_response):
         path = environ['PATH_INFO']
         method = environ['REQUEST_METHOD']
-        req = HTTPGitRequest(environ, start_response)
+        req = HTTPGitRequest(environ, start_response, self.dumb)
         # environ['QUERY_STRING'] has qs args
         handler = None
         for smethod, spath in self.services.iterkeys():