* New upstream release.

* Bump standards version to 3.9.1.

AUTHORS

@@ -2,3 +2,5 @@ Jelmer Vernooij <jelmer@samba.org>
 James Westby <jw+debian@jameswestby.net>
 John Carr <john.carr@unrouted.co.uk>
 Dave Borowitz <dborowitz@google.com>
+
+See the revision history for a full list of contributors.

NEWS

@@ -1,3 +1,51 @@
+0.6.1	2010-07-22
+
+ BUG FIXES
+
+  * Fix memory leak in C implementation of sorted_tree_items. (Dave Borowitz)
+
+  * Use correct path separators for named repo files. (Dave Borowitz)
+
+  * python > 2.7 and testtools-based test runners will now also pick up skipped
+    tests correctly. (Jelmer Vernooij)
+
+ FEATURES
+
+  * Move named file initilization to BaseRepo. (Dave Borowitz)
+
+  * Add logging utilities and git/HTTP server logging. (Dave Borowitz)
+
+  * The GitClient interface has been cleaned up and instances are now reusable.
+    (Augie Fackler)
+
+  * Allow overriding paths to executables in GitSSHClient. 
+    (Ross Light, Jelmer Vernooij, #585204)
+
+  * Add PackBasedObjectStore.pack_loose_objects(). (Jelmer Vernooij)
+
+ TESTS
+
+  * Add tests for sorted_tree_items and C implementation. (Dave Borowitz)
+
+  * Add a MemoryRepo that stores everything in memory. (Dave Borowitz)
+
+  * Quiet logging output from web tests. (Dave Borowitz)
+
+ CLEANUP
+
+  * Clean up file headers. (Dave Borowitz)
+
+ API CHANGES
+
+  * dulwich.pack.write_pack_index_v{1,2} now take a file-like object
+    rather than a filename. (Jelmer Vernooij)
+
+  * Make dul-daemon/dul-web trivial wrappers around server functionality.
+    (Dave Borowitz)
+
+  * Move reference WSGI handler to web.py. (Dave Borowitz)
+
+
 0.6.0	2010-05-22
 
 note: This list is most likely incomplete for 0.6.0.

bin/dul-daemon

@@ -1,32 +1,23 @@
 #!/usr/bin/python
-# dul-daemon - Simple git-daemon a like
+# dul-daemon - Simple git-daemon-like server
 # Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # or (at your option) a later version of the License.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-import sys
+from dulwich.server import main
-from dulwich.repo import Repo
-from dulwich.server import DictBackend, TCPGitServer
 
-if __name__ == "__main__":
+if __name__ == '__main__':
-    if len(sys.argv) > 1:
+    main()
-        gitdir = sys.argv[1]
-    else:
-        gitdir = "."
-
-    backend = DictBackend({"/": Repo(gitdir)})
-    server = TCPGitServer(backend, 'localhost')
-    server.serve_forever()

bin/dul-receive-pack

@@ -1,17 +1,17 @@
 #!/usr/bin/python
 # dul-receive-pack - git-receive-pack in python
 # Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # or (at your option) a later version of the License.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

bin/dul-upload-pack

@@ -1,17 +1,17 @@
 #!/usr/bin/python
 # dul-upload-pack - git-upload-pack in python
 # Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # or (at your option) a later version of the License.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

bin/dul-web

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # dul-web - HTTP-based git server
-# Copyright (C) 2010 David Borowitz <dborowitz@google.com>
+# Copyright (C) 2010 Google, Inc. <dborowitz@google.com>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -17,21 +17,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-import os
+from dulwich.web import main
-import sys
-from dulwich.repo import Repo
-from dulwich.server import DictBackend
-from dulwich.web import HTTPGitApplication
-from wsgiref.simple_server import make_server
 
-if __name__ == "__main__":
+if __name__ == '__main__':
-    if len(sys.argv) > 1:
+    main()
-        gitdir = sys.argv[1]
-    else:
-        gitdir = os.getcwd()
-
-    backend = DictBackend({"/": Repo(gitdir)})
-    app = HTTPGitApplication(backend)
-    # TODO: allow serving on other ports via command-line flag
-    server = make_server('', 8000, app)
-    server.serve_forever()

bin/dulwich

@@ -1,17 +1,17 @@
 #!/usr/bin/python
 # dulwich - Simple command-line interface to Dulwich
 # Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # or (at your option) a later version of the License.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

debian/changelog

@@ -1,8 +1,9 @@
-dulwich (0.6.0-5ubuntu1) UNRELEASED; urgency=low
+dulwich (0.6.1-1) unstable; urgency=low
 
-  * Bump standards version to 3.9.0.
+  * New upstream release.
+  * Bump standards version to 3.9.1.
 
- -- Jelmer Vernooij <jelmer@debian.org>  Tue, 20 Jul 2010 18:32:01 +0200
+ -- Jelmer Vernooij <jelmer@debian.org>  Fri, 30 Jul 2010 13:22:11 +0200
 
 dulwich (0.6.0-5) unstable; urgency=low
 

debian/control

@@ -4,7 +4,7 @@ Priority: optional
 Maintainer: Jelmer Vernooij <jelmer@debian.org>
 Homepage: http://samba.org/~jelmer/dulwich
 Build-Depends: python-support (>= 0.90), cdbs (>= 0.4.43), python-all-dev, debhelper (>= 5.0.37.2), python-nose, git (>= 1:1.7.0.4-2) | git-core
-Standards-Version: 3.9.0
+Standards-Version: 3.9.1
 XS-Python-Version: >= 2.4
 Vcs-Bzr: http://bzr.debian.org/users/jelmer/dulwich/unstable
 

dulwich/__init__.py

@@ -1,18 +1,18 @@
 # __init__.py -- The git module of dulwich
 # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
 # Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
-# of the License or (at your option) any later version of 
+# of the License or (at your option) any later version of
 # the License.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
@@ -27,4 +27,4 @@ import protocol
 import repo
 import server
 
-__version__ = (0, 6, 0)
+__version__ = (0, 6, 1)

dulwich/_objects.c

@@ -1,16 +1,16 @@
-/* 
+/*
  * Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; version 2
  * of the License or (at your option) a later version of the License.
- * 
+ *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
- * 
+ *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
@@ -35,7 +35,7 @@ static PyObject *sha_to_pyhex(const unsigned char *sha)
 		hexsha[i*2] = bytehex((sha[i] & 0xF0) >> 4);
 		hexsha[i*2+1] = bytehex(sha[i] & 0x0F);
 	}
-	
+
 	return PyString_FromStringAndSize(hexsha, 40);
 }
 
@@ -121,7 +121,7 @@ int cmp_tree_item(const void *_a, const void *_b)
 		common = strlen(b->name);
 		remain_a = a->name + common;
 		remain_b = (S_ISDIR(b->mode)?"/":"");
-	} else if (strlen(b->name) > strlen(a->name)) { 
+	} else if (strlen(b->name) > strlen(a->name)) {
 		common = strlen(a->name);
 		remain_a = (S_ISDIR(a->mode)?"/":"");
 		remain_b = b->name + common;
@@ -136,12 +136,20 @@ int cmp_tree_item(const void *_a, const void *_b)
 	return strcmp(remain_a, remain_b);
 }
 
+static void free_tree_items(struct tree_item *items, int num) {
+	int i;
+	for (i = 0; i < num; i++) {
+		Py_DECREF(items[i].tuple);
+	}
+	free(items);
+}
+
 static PyObject *py_sorted_tree_items(PyObject *self, PyObject *entries)
 {
 	struct tree_item *qsort_entries;
 	int num, i;
 	PyObject *ret;
-	Py_ssize_t pos = 0; 
+	Py_ssize_t pos = 0;
 	PyObject *key, *value;
 
 	if (!PyDict_Check(entries)) {
@@ -159,10 +167,16 @@ static PyObject *py_sorted_tree_items(PyObject *self, PyObject *entries)
 	i = 0;
 	while (PyDict_Next(entries, &pos, &key, &value)) {
 		PyObject *py_mode, *py_int_mode, *py_sha;
-		
+
+		if (!PyString_Check(key)) {
+			PyErr_SetString(PyExc_TypeError, "Name is not a string");
+			free_tree_items(qsort_entries, i);
+			return NULL;
+		}
+
 		if (PyTuple_Size(value) != 2) {
 			PyErr_SetString(PyExc_ValueError, "Tuple has invalid size");
-			free(qsort_entries);
+			free_tree_items(qsort_entries, i);
 			return NULL;
 		}
 
@@ -170,19 +184,21 @@ static PyObject *py_sorted_tree_items(PyObject *self, PyObject *entries)
 		py_int_mode = PyNumber_Int(py_mode);
 		if (!py_int_mode) {
 			PyErr_SetString(PyExc_TypeError, "Mode is not an integral type");
-			free(qsort_entries);
+			free_tree_items(qsort_entries, i);
 			return NULL;
 		}
 
 		py_sha = PyTuple_GET_ITEM(value, 1);
-		if (!PyString_CheckExact(key)) {
+		if (!PyString_Check(py_sha)) {
-			PyErr_SetString(PyExc_TypeError, "Name is not a string");
+			PyErr_SetString(PyExc_TypeError, "SHA is not a string");
-			free(qsort_entries);
+			Py_DECREF(py_int_mode);
+			free_tree_items(qsort_entries, i);
 			return NULL;
 		}
 		qsort_entries[i].name = PyString_AS_STRING(key);
 		qsort_entries[i].mode = PyInt_AS_LONG(py_mode);
-		qsort_entries[i].tuple = PyTuple_Pack(3, key, py_mode, py_sha);
+		qsort_entries[i].tuple = PyTuple_Pack(3, key, py_int_mode, py_sha);
+		Py_DECREF(py_int_mode);
 		i++;
 	}
 
@@ -190,7 +206,7 @@ static PyObject *py_sorted_tree_items(PyObject *self, PyObject *entries)
 
 	ret = PyList_New(num);
 	if (ret == NULL) {
-		free(qsort_entries);
+		free_tree_items(qsort_entries, i);
 		PyErr_NoMemory();
 		return NULL;
 	}

dulwich/client.py

@@ -21,13 +21,11 @@
 
 __docformat__ = 'restructuredText'
 
-import os
 import select
 import socket
 import subprocess
 
 from dulwich.errors import (
-    ChecksumMismatch,
     SendPackError,
     UpdateRefsError,
     )
@@ -58,36 +56,83 @@ class GitClient(object):
 
     """
 
-    def __init__(self, can_read, read, write, thin_packs=True,
+    def __init__(self, thin_packs=True, report_activity=None):
-        report_activity=None):
         """Create a new GitClient instance.
 
-        :param can_read: Function that returns True if there is data available
-            to be read.
-        :param read: Callback for reading data, takes number of bytes to read
-        :param write: Callback for writing data
         :param thin_packs: Whether or not thin packs should be retrieved
         :param report_activity: Optional callback for reporting transport
             activity.
         """
-        self.proto = Protocol(read, write, report_activity)
+        self._report_activity = report_activity
-        self._can_read = can_read
         self._fetch_capabilities = list(FETCH_CAPABILITIES)
         self._send_capabilities = list(SEND_CAPABILITIES)
         if thin_packs:
             self._fetch_capabilities.append("thin-pack")
 
-    def read_refs(self):
+    def _connect(self, cmd, path):
+        """Create a connection to the server.
+
+        This method is abstract - concrete implementations should
+        implement their own variant which connects to the server and
+        returns an initialized Protocol object with the service ready
+        for use and a can_read function which may be used to see if
+        reads would block.
+
+        :param cmd: The git service name to which we should connect.
+        :param path: The path we should pass to the service.
+        """
+        raise NotImplementedError()
+
+    def read_refs(self, proto):
         server_capabilities = None
         refs = {}
         # Receive refs from server
-        for pkt in self.proto.read_pkt_seq():
+        for pkt in proto.read_pkt_seq():
             (sha, ref) = pkt.rstrip("\n").split(" ", 1)
             if server_capabilities is None:
                 (ref, server_capabilities) = extract_capabilities(ref)
             refs[ref] = sha
         return refs, server_capabilities
 
+    def _parse_status_report(self, proto):
+        unpack = proto.read_pkt_line().strip()
+        if unpack != 'unpack ok':
+            st = True
+            # flush remaining error data
+            while st is not None:
+                st = proto.read_pkt_line()
+            raise SendPackError(unpack)
+        statuses = []
+        errs = False
+        ref_status = proto.read_pkt_line()
+        while ref_status:
+            ref_status = ref_status.strip()
+            statuses.append(ref_status)
+            if not ref_status.startswith('ok '):
+                errs = True
+            ref_status = proto.read_pkt_line()
+
+        if errs:
+            ref_status = {}
+            ok = set()
+            for status in statuses:
+                if ' ' not in status:
+                    # malformed response, move on to the next one
+                    continue
+                status, ref = status.split(' ', 1)
+
+                if status == 'ng':
+                    if ' ' in ref:
+                        ref, status = ref.split(' ', 1)
+                else:
+                    ok.add(ref)
+                ref_status[ref] = status
+            raise UpdateRefsError('%s failed to update' %
+                                  ', '.join([ref for ref in ref_status
+                                             if ref not in ok]),
+                                  ref_status=ref_status)
+
+
     # TODO(durin42): add side-band-64k capability support here and advertise it
     def send_pack(self, path, determine_wants, generate_pack_contents):
         """Upload a pack to a remote repository.
@@ -100,12 +145,13 @@ class GitClient(object):
         :raises UpdateRefsError: if the server supports report-status
                                  and rejects ref updates
         """
-        old_refs, server_capabilities = self.read_refs()
+        proto, unused_can_read = self._connect('receive-pack', path)
+        old_refs, server_capabilities = self.read_refs(proto)
         if 'report-status' not in server_capabilities:
             self._send_capabilities.remove('report-status')
         new_refs = determine_wants(old_refs)
         if not new_refs:
-            self.proto.write_pkt_line(None)
+            proto.write_pkt_line(None)
             return {}
         want = []
         have = [x for x in old_refs.values() if not x == ZERO_SHA]
@@ -115,61 +161,26 @@ class GitClient(object):
             new_sha1 = new_refs.get(refname, ZERO_SHA)
             if old_sha1 != new_sha1:
                 if sent_capabilities:
-                    self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1,
+                    proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1,
                                                             refname))
                 else:
-                    self.proto.write_pkt_line(
+                    proto.write_pkt_line(
                       "%s %s %s\0%s" % (old_sha1, new_sha1, refname,
                                         ' '.join(self._send_capabilities)))
                     sent_capabilities = True
             if new_sha1 not in have and new_sha1 != ZERO_SHA:
                 want.append(new_sha1)
-        self.proto.write_pkt_line(None)
+        proto.write_pkt_line(None)
         if not want:
             return new_refs
         objects = generate_pack_contents(have, want)
-        (entries, sha) = write_pack_data(self.proto.write_file(), objects,
+        entries, sha = write_pack_data(proto.write_file(), objects,
-                                         len(objects))
+                                       len(objects))
 
         if 'report-status' in self._send_capabilities:
-            unpack = self.proto.read_pkt_line().strip()
+            self._parse_status_report(proto)
-            if unpack != 'unpack ok':
-                st = True
-                # flush remaining error data
-                while st is not None:
-                    st = self.proto.read_pkt_line()
-                raise SendPackError(unpack)
-            statuses = []
-            errs = False
-            ref_status = self.proto.read_pkt_line()
-            while ref_status:
-                ref_status = ref_status.strip()
-                statuses.append(ref_status)
-                if not ref_status.startswith('ok '):
-                    errs = True
-                ref_status = self.proto.read_pkt_line()
-
-            if errs:
-                ref_status = {}
-                ok = set()
-                for status in statuses:
-                    if ' ' not in status:
-                        # malformed response, move on to the next one
-                        continue
-                    status, ref = status.split(' ', 1)
-
-                    if status == 'ng':
-                        if ' ' in ref:
-                            ref, status = ref.split(' ', 1)
-                    else:
-                        ok.add(ref)
-                    ref_status[ref] = status
-                raise UpdateRefsError('%s failed to update' %
-                                      ', '.join([ref for ref in ref_status
-                                                 if ref not in ok]),
-                                      ref_status=ref_status)
         # wait for EOF before returning
-        data = self.proto.read()
+        data = proto.read()
         if data:
             raise SendPackError('Unexpected response %r' % data)
         return new_refs
@@ -202,39 +213,40 @@ class GitClient(object):
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
         """
-        (refs, server_capabilities) = self.read_refs()
+        proto, can_read = self._connect('upload-pack', path)
+        (refs, server_capabilities) = self.read_refs(proto)
         wants = determine_wants(refs)
         if not wants:
-            self.proto.write_pkt_line(None)
+            proto.write_pkt_line(None)
             return refs
         assert isinstance(wants, list) and type(wants[0]) == str
-        self.proto.write_pkt_line("want %s %s\n" % (
+        proto.write_pkt_line("want %s %s\n" % (
             wants[0], ' '.join(self._fetch_capabilities)))
         for want in wants[1:]:
-            self.proto.write_pkt_line("want %s\n" % want)
+            proto.write_pkt_line("want %s\n" % want)
-        self.proto.write_pkt_line(None)
+        proto.write_pkt_line(None)
         have = graph_walker.next()
         while have:
-            self.proto.write_pkt_line("have %s\n" % have)
+            proto.write_pkt_line("have %s\n" % have)
-            if self._can_read():
+            if can_read():
-                pkt = self.proto.read_pkt_line()
+                pkt = proto.read_pkt_line()
                 parts = pkt.rstrip("\n").split(" ")
                 if parts[0] == "ACK":
                     graph_walker.ack(parts[1])
                     assert parts[2] == "continue"
             have = graph_walker.next()
-        self.proto.write_pkt_line("done\n")
+        proto.write_pkt_line("done\n")
-        pkt = self.proto.read_pkt_line()
+        pkt = proto.read_pkt_line()
         while pkt:
             parts = pkt.rstrip("\n").split(" ")
             if parts[0] == "ACK":
                 graph_walker.ack(pkt.split(" ")[1])
             if len(parts) < 3 or parts[2] != "continue":
                 break
-            pkt = self.proto.read_pkt_line()
+            pkt = proto.read_pkt_line()
         # TODO(durin42): this is broken if the server didn't support the
         # side-band-64k capability.
-        for pkt in self.proto.read_pkt_seq():
+        for pkt in proto.read_pkt_seq():
             channel = ord(pkt[0])
             pkt = pkt[1:]
             if channel == 1:
@@ -247,96 +259,48 @@ class GitClient(object):
         return refs
 
 
+def can_read(f):
+    """Check if a file descriptor is readable.
+
+    :param f: either the number of the file descriptor or a file-like
+             object which returns the fileno when f.fileno() is called.
+    """
+    return len(select.select([f], [], [], 0)[0]) > 0
+
+
 class TCPGitClient(GitClient):
     """A Git Client that works over TCP directly (i.e. git://)."""
 
     def __init__(self, host, port=None, *args, **kwargs):
-        self._socket = socket.socket(type=socket.SOCK_STREAM)
         if port is None:
             port = TCP_GIT_PORT
-        self._socket.connect((host, port))
+        self._host = host
-        self.rfile = self._socket.makefile('rb', -1)
+        self._port = port
-        self.wfile = self._socket.makefile('wb', 0)
+        GitClient.__init__(self, *args, **kwargs)
-        self.host = host
+
-        super(TCPGitClient, self).__init__(lambda: _fileno_can_read(self._socket.fileno()), self.rfile.read, self.wfile.write, *args, **kwargs)
+    def _connect(self, cmd, path):
-
+        s = socket.socket(type=socket.SOCK_STREAM)
-    def send_pack(self, path, changed_refs, generate_pack_contents):
+        s.connect((self._host, self._port))
-        """Send a pack to a remote host.
+        # -1 means system default buffering
-
+        rfile = s.makefile('rb', -1)
-        :param path: Path of the repository on the remote host
+        # 0 means unbuffered
-        """
+        wfile = s.makefile('wb', 0)
-        self.proto.send_cmd("git-receive-pack", path, "host=%s" % self.host)
+        proto = Protocol(rfile.read, wfile.write,
-        return super(TCPGitClient, self).send_pack(path, changed_refs, generate_pack_contents)
+                         report_activity=self._report_activity)
-
+        proto.send_cmd('git-%s' % cmd, path, 'host=%s' % self._host)
-    def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
+        return proto, lambda: can_read(s)
-        """Fetch a pack from the remote host.
+
-
+
-        :param path: Path of the reposiutory on the remote host
+class SubprocessWrapper(object):
-        :param determine_wants: Callback that receives available refs dict and
+    """A socket-like object that talks to a subprocess via pipes."""
-            should return list of sha's to fetch.
-        :param graph_walker: GraphWalker instance used to find missing shas
-        :param pack_data: Callback for writing pack data
-        :param progress: Callback for writing progress
-        """
-        self.proto.send_cmd("git-upload-pack", path, "host=%s" % self.host)
-        return super(TCPGitClient, self).fetch_pack(path, determine_wants,
-            graph_walker, pack_data, progress)
-
-
-class SubprocessGitClient(GitClient):
-    """Git client that talks to a server using a subprocess."""
-
-    def __init__(self, *args, **kwargs):
-        self.proc = None
-        self._args = args
-        self._kwargs = kwargs
-
-    def _connect(self, service, *args, **kwargs):
-        argv = [service] + list(args)
-        self.proc = subprocess.Popen(argv, bufsize=0,
-                                stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE)
-        def read_fn(size):
-            return self.proc.stdout.read(size)
-        def write_fn(data):
-            self.proc.stdin.write(data)
-            self.proc.stdin.flush()
-        return GitClient(lambda: _fileno_can_read(self.proc.stdout.fileno()), read_fn, write_fn, *args, **kwargs)
-
-    def send_pack(self, path, changed_refs, generate_pack_contents):
-        """Upload a pack to the server.
-
-        :param path: Path to the git repository on the server
-        :param changed_refs: Dictionary with new values for the refs
-        :param generate_pack_contents: Function that returns an iterator over
-            objects to send
-        """
-        client = self._connect("git-receive-pack", path)
-        return client.send_pack(path, changed_refs, generate_pack_contents)
-
-    def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-        progress):
-        """Retrieve a pack from the server
-
-        :param path: Path to the git repository on the server
-        :param determine_wants: Function that receives existing refs
-            on the server and returns a list of desired shas
-        :param graph_walker: GraphWalker instance
-        :param pack_data: Function that can write pack data
-        :param progress: Function that can write progress texts
-        """
-        client = self._connect("git-upload-pack", path)
-        return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
-                                 progress)
-
-
-class SSHSubprocess(object):
-    """A socket-like object that talks to an ssh subprocess via pipes."""
 
     def __init__(self, proc):
         self.proc = proc
-        self.read = self.recv = proc.stdout.read
+        self.read = proc.stdout.read
-        self.write = self.send = proc.stdin.write
+        self.write = proc.stdin.write
+
+    def can_read(self):
+        return can_read(self.proc.stdout.fileno())
 
     def close(self):
         self.proc.stdin.close()
@@ -344,6 +308,21 @@ class SSHSubprocess(object):
         self.proc.wait()
 
 
+class SubprocessGitClient(GitClient):
+    """Git client that talks to a server using a subprocess."""
+
+    def __init__(self, *args, **kwargs):
+        self._connection = None
+        GitClient.__init__(self, *args, **kwargs)
+
+    def _connect(self, service, path):
+        argv = ['git', service, path]
+        p = SubprocessWrapper(
+            subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE))
+        return Protocol(p.read, p.write,
+                        report_activity=self._report_activity), p.can_read
+
 class SSHVendor(object):
 
     def connect_ssh(self, host, command, username=None, port=None):
@@ -357,7 +336,7 @@ class SSHVendor(object):
         proc = subprocess.Popen(args + command,
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE)
-        return SSHSubprocess(proc)
+        return SubprocessWrapper(proc)
 
 # Can be overridden by users
 get_ssh_vendor = SSHVendor
@@ -369,22 +348,17 @@ class SSHGitClient(GitClient):
         self.host = host
         self.port = port
         self.username = username
-        self._args = args
+        GitClient.__init__(self, *args, **kwargs)
-        self._kwargs = kwargs
+        self.alternative_paths = {}
 
-    def send_pack(self, path, determine_wants, generate_pack_contents):
+    def _get_cmd_path(self, cmd):
-        remote = get_ssh_vendor().connect_ssh(
+        return self.alternative_paths.get(cmd, 'git-%s' % cmd)
-            self.host, ["git-receive-pack '%s'" % path],
-            port=self.port, username=self.username)
-        client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
-        return client.send_pack(path, determine_wants, generate_pack_contents)
 
-    def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
+    def _connect(self, cmd, path):
-        progress):
+        con = get_ssh_vendor().connect_ssh(
-        remote = get_ssh_vendor().connect_ssh(self.host, ["git-upload-pack '%s'" % path], port=self.port, username=self.username)
+            self.host, ["%s '%s'" % (self._get_cmd_path(cmd), path)],
-        client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
+            port=self.port, username=self.username)
-        return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
+        return Protocol(con.read, con.write), con.can_read
-                                 progress)
 
 
 def get_transport_and_path(uri):
@@ -398,5 +372,6 @@ def get_transport_and_path(uri):
         if uri.startswith(handler):
             host, path = uri[len(handler):].split("/", 1)
             return transport(host), "/"+path
+    # FIXME: Parse rsync-like git URLs (user@host:/path), bug 568493
     # if its not git or git+ssh, try a local url..
     return SubprocessGitClient(), uri

dulwich/fastexport.py

@@ -1,17 +1,17 @@
 # __init__.py -- Fast export/import functionality
 # Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
-# of the License or (at your option) any later version of 
+# of the License or (at your option) any later version of
 # the License.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
@@ -21,7 +21,6 @@
 """Fast export/import functionality."""
 
 from dulwich.objects import (
-    Tree,
     format_timezone,
     )
 

dulwich/file.py

@@ -16,10 +16,8 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-
 """Safe access to git files."""
 
-
 import errno
 import os
 import tempfile
@@ -65,7 +63,9 @@ def fancy_rename(oldname, newname):
 def GitFile(filename, mode='r', bufsize=-1):
     """Create a file object that obeys the git file locking protocol.
 
-    See _GitFile for a description of the file locking protocol.
+    :return: a builtin file object or a _GitFile object
+
+    :note: See _GitFile for a description of the file locking protocol.
 
     Only read-only and write-only (binary) modes are supported; r+, w+, and a
     are not.  To read and write from the same file, you can take advantage of
@@ -86,8 +86,6 @@ def GitFile(filename, mode='r', bufsize=-1):
     Traceback (most recent call last):
         ...
     OSError: [Errno 17] File exists: 'filename.lock'
-
-    :return: a builtin file object or a _GitFile object
     """
     if 'a' in mode:
         raise IOError('append mode not supported for Git files')

dulwich/index.py

@@ -1,16 +1,16 @@
-# index.py -- File parser/write for the git index file
+# index.py -- File parser/writer for the git index file
 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
- 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

dulwich/log_utils.py

@@ -0,0 +1,67 @@
+# log_utils.py -- Logging utilities for Dulwich
+# Copyright (C) 2010 Google, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA  02110-1301, USA.
+
+"""Logging utilities for Dulwich.
+
+Any module that uses logging needs to do compile-time initialization to set up
+the logging environment. Since Dulwich is also used as a library, clients may
+not want to see any logging output. In that case, we need to use a special
+handler to suppress spurious warnings like "No handlers could be found for
+logger dulwich.foo".
+
+For details on the _NullHandler approach, see:
+http://docs.python.org/library/logging.html#configuring-logging-for-a-library
+
+For many modules, the only function from the logging module they need is
+getLogger; this module exports that function for convenience. If a calling
+module needs something else, it can import the standard logging module directly.
+"""
+
+import logging
+import sys
+
+getLogger = logging.getLogger
+
+
+class _NullHandler(logging.Handler):
+    """No-op logging handler to avoid unexpected logging warnings."""
+
+    def emit(self, record):
+        pass
+
+
+_NULL_HANDLER = _NullHandler()
+_DULWICH_LOGGER = getLogger('dulwich')
+_DULWICH_LOGGER.addHandler(_NULL_HANDLER)
+
+
+def default_logging_config():
+    """Set up the default Dulwich loggers."""
+    remove_null_handler()
+    logging.basicConfig(level=logging.INFO, stream=sys.stderr,
+                        format='%(asctime)s %(levelname)s: %(message)s')
+
+
+def remove_null_handler():
+    """Remove the null handler from the Dulwich loggers.
+
+    If a caller wants to set up logging using something other than
+    default_logging_config, calling this function first is a minor optimization
+    to avoid the overhead of using the _NullHandler.
+    """
+    _DULWICH_LOGGER.removeHandler(_NULL_HANDLER)

dulwich/lru_cache.py

@@ -1,3 +1,4 @@
+# lru_cache.py -- Simple LRU cache for dulwich
 # Copyright (C) 2006, 2008 Canonical Ltd
 #
 # This program is free software; you can redistribute it and/or modify

dulwich/misc.py

@@ -1,20 +1,21 @@
 # misc.py -- For dealing with python2.4 oddness
 # Copyright (C) 2008 Canonical Ltd.
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your option) a later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
+
 """Misc utilities to work with python <2.6.
 
 These utilities can all be deleted when dulwich decides it wants to stop

dulwich/object_store.py

@@ -53,6 +53,7 @@ from dulwich.pack import (
     write_pack_index_v2,
     )
 
+INFODIR = 'info'
 PACKDIR = 'pack'
 
 
@@ -272,11 +273,28 @@ class PackBasedObjectStore(BaseObjectStore):
         return self._pack_cache
 
     def _iter_loose_objects(self):
+        """Iterate over the SHAs of all loose objects."""
         raise NotImplementedError(self._iter_loose_objects)
 
     def _get_loose_object(self, sha):
         raise NotImplementedError(self._get_loose_object)
 
+    def _remove_loose_object(self, sha):
+        raise NotImplementedError(self._remove_loose_object)
+
+    def pack_loose_objects(self):
+        """Pack loose objects.
+        
+        :return: Number of objects packed
+        """
+        objects = set()
+        for sha in self._iter_loose_objects():
+            objects.add((self._get_loose_object(sha), None))
+        self.add_objects(objects)
+        for obj, path in objects:
+            self._remove_loose_object(obj.id)
+        return len(objects)
+
     def __iter__(self):
         """Iterate over the SHAs that are present in this store."""
         iterables = self.packs + [self._iter_loose_objects()]
@@ -385,6 +403,9 @@ class DiskObjectStore(PackBasedObjectStore):
                 return None
             raise
 
+    def _remove_loose_object(self, sha):
+        os.remove(self._get_shafile_path(sha))
+
     def move_in_thin_pack(self, path):
         """Move a specific file containing a pack into the pack directory.
 
@@ -425,7 +446,11 @@ class DiskObjectStore(PackBasedObjectStore):
         entries = p.sorted_entries()
         basename = os.path.join(self.pack_dir,
             "pack-%s" % iter_sha1(entry[0] for entry in entries))
-        write_pack_index_v2(basename+".idx", entries, p.get_stored_checksum())
+        f = GitFile(basename+".idx", "wb")
+        try:
+            write_pack_index_v2(f, entries, p.get_stored_checksum())
+        finally:
+            f.close()
         p.close()
         os.rename(path, basename + ".pack")
         final_pack = Pack(basename)

dulwich/objects.py

@@ -17,7 +17,6 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-
 """Access to base git objects."""
 
 
@@ -25,7 +24,6 @@ import binascii
 from cStringIO import (
     StringIO,
     )
-import mmap
 import os
 import stat
 import zlib
@@ -182,7 +180,9 @@ class ShaFile(object):
         start = 0
         end = -1
         while end < 0:
-            header += decomp.decompress(f.read(bufsize))
+            extra = f.read(bufsize)
+            header += decomp.decompress(extra)
+            magic += extra
             end = header.find("\0", start)
             start = len(header)
         header = header[:end]
@@ -191,19 +191,16 @@ class ShaFile(object):
         obj_class = object_class(type_name)
         if not obj_class:
             raise ObjectFormatException("Not a known type: %s" % type_name)
-        return obj_class()
+        ret = obj_class()
+        ret._magic = magic
+        return ret
 
-    def _parse_legacy_object(self, f):
+    def _parse_legacy_object(self, map):
         """Parse a legacy object, setting the raw string."""
-        size = os.path.getsize(f.name)
+        text = _decompress(map)
-        map = mmap.mmap(f.fileno(), size, access=mmap.ACCESS_READ)
-        try:
-            text = _decompress(map)
-        finally:
-            map.close()
         header_end = text.find('\0')
         if header_end < 0:
-            raise ObjectFormatException("Invalid object header")
+            raise ObjectFormatException("Invalid object header, no \\0")
         self.set_raw_string(text[header_end+1:])
 
     def as_legacy_object_chunks(self):
@@ -240,6 +237,7 @@ class ShaFile(object):
             if not self._chunked_text:
                 if self._file is not None:
                     self._parse_file(self._file)
+                    self._file = None
                 elif self._path is not None:
                     self._parse_path()
                 else:
@@ -266,25 +264,22 @@ class ShaFile(object):
         num_type = (ord(magic[0]) >> 4) & 7
         obj_class = object_class(num_type)
         if not obj_class:
-            raise ObjectFormatException("Not a known type: %d" % num_type)
+            raise ObjectFormatException("Not a known type %d" % num_type)
-        return obj_class()
+        ret = obj_class()
+        ret._magic = magic
+        return ret
 
-    def _parse_object(self, f):
+    def _parse_object(self, map):
         """Parse a new style object, setting self._text."""
-        size = os.path.getsize(f.name)
+        # skip type and size; type must have already been determined, and
-        map = mmap.mmap(f.fileno(), size, access=mmap.ACCESS_READ)
+        # we trust zlib to fail if it's otherwise corrupted
-        try:
+        byte = ord(map[0])
-            # skip type and size; type must have already been determined, and
+        used = 1
-            # we trust zlib to fail if it's otherwise corrupted
+        while (byte & 0x80) != 0:
-            byte = ord(map[0])
+            byte = ord(map[used])
-            used = 1
+            used += 1
-            while (byte & 0x80) != 0:
+        raw = map[used:]
-                byte = ord(map[used])
+        self.set_raw_string(_decompress(raw))
-                used += 1
-            raw = map[used:]
-            self.set_raw_string(_decompress(raw))
-        finally:
-            map.close()
 
     @classmethod
     def _is_legacy_object(cls, magic):
@@ -305,6 +300,7 @@ class ShaFile(object):
         self._sha = None
         self._path = None
         self._file = None
+        self._magic = None
         self._chunked_text = []
         self._needs_parsing = False
         self._needs_serialization = True
@@ -323,11 +319,14 @@ class ShaFile(object):
             f.close()
 
     def _parse_file(self, f):
-        magic = f.read(2)
+        magic = self._magic
-        if self._is_legacy_object(magic):
+        if magic is None:
-            self._parse_legacy_object(f)
+            magic = f.read(2)
+        map = magic + f.read()
+        if self._is_legacy_object(magic[:2]):
+            self._parse_legacy_object(map)
         else:
-            self._parse_object(f)
+            self._parse_object(map)
 
     @classmethod
     def from_path(cls, path):
@@ -337,6 +336,7 @@ class ShaFile(object):
             obj._path = path
             obj._sha = FixedSha(filename_to_hex(path))
             obj._file = None
+            obj._magic = None
             return obj
         finally:
             f.close()
@@ -530,9 +530,9 @@ def _parse_tag_or_commit(text):
     """Parse tag or commit text.
 
     :param text: the raw text of the tag or commit object.
-    :yield: tuples of (field, value), one per header line, in the order read
+    :return: iterator of tuples of (field, value), one per header line, in the
-        from the text, possibly including duplicates. Includes a field named
+        order read from the text, possibly including duplicates. Includes a
-        None for the freeform tag/commit text.
+        field named None for the freeform tag/commit text.
     """
     f = StringIO(text)
     for l in f:
@@ -677,18 +677,26 @@ def parse_tree(text):
     """Parse a tree text.
 
     :param text: Serialized text to parse
-    :yields: tuples of (name, mode, sha)
+    :return: iterator of tuples of (name, mode, sha)
     """
     count = 0
     l = len(text)
     while count < l:
         mode_end = text.index(' ', count)
-        mode = int(text[count:mode_end], 8)
+        mode_text = text[count:mode_end]
+        assert mode_text[0] != '0'
+        try:
+            mode = int(mode_text, 8)
+        except ValueError:
+            raise ObjectFormatException("Invalid mode '%s'" % mode_text)
         name_end = text.index('\0', mode_end)
         name = text[mode_end+1:name_end]
         count = name_end+21
         sha = text[name_end+1:count]
-        yield (name, mode, sha_to_hex(sha))
+        if len(sha) != 20:
+            raise ObjectFormatException("Sha has invalid length")
+        hexsha = sha_to_hex(sha)
+        yield (name, mode, hexsha)
 
 
 def serialize_tree(items):
@@ -706,10 +714,15 @@ def sorted_tree_items(entries):
     the items would be serialized.
 
     :param entries: Dictionary mapping names to (mode, sha) tuples
-    :return: Iterator over (name, mode, sha)
+    :return: Iterator over (name, mode, hexsha)
     """
     for name, entry in sorted(entries.iteritems(), cmp=cmp_entry):
-        yield name, entry[0], entry[1]
+        mode, hexsha = entry
+        # Stricter type checks than normal to mirror checks in the C version.
+        mode = int(mode)
+        if not isinstance(hexsha, str):
+            raise TypeError('Expected a string for SHA, got %r' % hexsha)
+        yield name, mode, hexsha
 
 
 def cmp_entry((name1, value1), (name2, value2)):

dulwich/pack.py

@@ -1,6 +1,6 @@
-# pack.py -- For dealing wih packed git objects.
+# pack.py -- For dealing with packed git objects.
 # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
-# Copryight (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -310,7 +310,8 @@ class PackIndex(object):
     def iterentries(self):
         """Iterate over the entries in this pack index.
 
-        :yields: tuples with object name, offset in packfile and crc32 checksum.
+        :return: iterator over tuples with object name, offset in packfile and
+            crc32 checksum.
         """
         for i in range(len(self)):
             yield self._unpack_entry(i)
@@ -787,9 +788,9 @@ class PackData(object):
     def iterentries(self, progress=None):
         """Yield entries summarizing the contents of this pack.
 
-        :param progress: Progress function, called with current and total object
+        :param progress: Progress function, called with current and total
-            count.
+            object count.
-        :yields: tuples with (sha, offset, crc32)
+        :return: iterator of tuples with (sha, offset, crc32)
         """
         for offset, type, obj, crc32 in self.iterobjects(progress=progress):
             assert isinstance(offset, int)
@@ -801,8 +802,8 @@ class PackData(object):
     def sorted_entries(self, progress=None):
         """Return entries in this pack, sorted by SHA.
 
-        :param progress: Progress function, called with current and total object
+        :param progress: Progress function, called with current and total
-            count
+            object count
         :return: List of tuples with (sha, offset, crc32)
         """
         ret = list(self.iterentries(progress=progress))
@@ -814,18 +815,28 @@ class PackData(object):
 
         :param filename: Index filename.
         :param progress: Progress report function
+        :return: Checksum of index file
         """
         entries = self.sorted_entries(progress=progress)
-        write_pack_index_v1(filename, entries, self.calculate_checksum())
+        f = GitFile(filename, 'wb')
+        try:
+            return write_pack_index_v1(f, entries, self.calculate_checksum())
+        finally:
+            f.close()
 
     def create_index_v2(self, filename, progress=None):
         """Create a version 2 index file for this data file.
 
         :param filename: Index filename.
         :param progress: Progress report function
+        :return: Checksum of index file
         """
         entries = self.sorted_entries(progress=progress)
-        write_pack_index_v2(filename, entries, self.calculate_checksum())
+        f = GitFile(filename, 'wb')
+        try:
+            return write_pack_index_v2(f, entries, self.calculate_checksum())
+        finally:
+            f.close()
 
     def create_index(self, filename, progress=None,
                      version=2):
@@ -833,11 +844,12 @@ class PackData(object):
 
         :param filename: Index filename.
         :param progress: Progress report function
+        :return: Checksum of index file
         """
         if version == 1:
-            self.create_index_v1(filename, progress)
+            return self.create_index_v1(filename, progress)
         elif version == 2:
-            self.create_index_v2(filename, progress)
+            return self.create_index_v2(filename, progress)
         else:
             raise ValueError("unknown index format %d" % version)
 
@@ -1034,6 +1046,7 @@ def write_pack(filename, objects, num_objects):
     :param filename: Path to the new pack file (without .pack extension)
     :param objects: Iterable over (object, path) tuples to write
     :param num_objects: Number of objects to write
+    :return: Tuple with checksum of pack file and index file
     """
     f = GitFile(filename + ".pack", 'wb')
     try:
@@ -1041,7 +1054,11 @@ def write_pack(filename, objects, num_objects):
     finally:
         f.close()
     entries.sort()
-    write_pack_index_v2(filename + ".idx", entries, data_sum)
+    f = GitFile(filename + ".idx", 'wb')
+    try:
+        return data_sum, write_pack_index_v2(f, entries, data_sum)
+    finally:
+        f.close()
 
 
 def write_pack_data(f, objects, num_objects, window=10):
@@ -1091,30 +1108,28 @@ def write_pack_data(f, objects, num_objects, window=10):
     return entries, f.write_sha()
 
 
-def write_pack_index_v1(filename, entries, pack_checksum):
+def write_pack_index_v1(f, entries, pack_checksum):
     """Write a new pack index file.
 
-    :param filename: The filename of the new pack index file.
+    :param f: A file-like object to write to
     :param entries: List of tuples with object name (sha), offset_in_pack,
         and crc32_checksum.
     :param pack_checksum: Checksum of the pack file.
+    :return: The SHA of the written index file
     """
-    f = GitFile(filename, 'wb')
+    f = SHA1Writer(f)
-    try:
+    fan_out_table = defaultdict(lambda: 0)
-        f = SHA1Writer(f)
+    for (name, offset, entry_checksum) in entries:
-        fan_out_table = defaultdict(lambda: 0)
+        fan_out_table[ord(name[0])] += 1
-        for (name, offset, entry_checksum) in entries:
+    # Fan-out table
-            fan_out_table[ord(name[0])] += 1
+    for i in range(0x100):
-        # Fan-out table
+        f.write(struct.pack(">L", fan_out_table[i]))
-        for i in range(0x100):
+        fan_out_table[i+1] += fan_out_table[i]
-            f.write(struct.pack(">L", fan_out_table[i]))
+    for (name, offset, entry_checksum) in entries:
-            fan_out_table[i+1] += fan_out_table[i]
+        f.write(struct.pack(">L20s", offset, name))
-        for (name, offset, entry_checksum) in entries:
+    assert len(pack_checksum) == 20
-            f.write(struct.pack(">L20s", offset, name))
+    f.write(pack_checksum)
-        assert len(pack_checksum) == 20
+    return f.write_sha()
-        f.write(pack_checksum)
-    finally:
-        f.close()
 
 
 def create_delta(base_buf, target_buf):
@@ -1242,38 +1257,36 @@ def apply_delta(src_buf, delta):
     return out
 
 
-def write_pack_index_v2(filename, entries, pack_checksum):
+def write_pack_index_v2(f, entries, pack_checksum):
     """Write a new pack index file.
 
-    :param filename: The filename of the new pack index file.
+    :param f: File-like object to write to
     :param entries: List of tuples with object name (sha), offset_in_pack, and
         crc32_checksum.
     :param pack_checksum: Checksum of the pack file.
+    :return: The SHA of the index file written
     """
-    f = GitFile(filename, 'wb')
+    f = SHA1Writer(f)
-    try:
+    f.write('\377tOc') # Magic!
-        f = SHA1Writer(f)
+    f.write(struct.pack(">L", 2))
-        f.write('\377tOc') # Magic!
+    fan_out_table = defaultdict(lambda: 0)
-        f.write(struct.pack(">L", 2))
+    for (name, offset, entry_checksum) in entries:
-        fan_out_table = defaultdict(lambda: 0)
+        fan_out_table[ord(name[0])] += 1
-        for (name, offset, entry_checksum) in entries:
+    # Fan-out table
-            fan_out_table[ord(name[0])] += 1
+    for i in range(0x100):
-        # Fan-out table
+        f.write(struct.pack(">L", fan_out_table[i]))
-        for i in range(0x100):
+        fan_out_table[i+1] += fan_out_table[i]
-            f.write(struct.pack(">L", fan_out_table[i]))
+    for (name, offset, entry_checksum) in entries:
-            fan_out_table[i+1] += fan_out_table[i]
+        f.write(name)
-        for (name, offset, entry_checksum) in entries:
+    for (name, offset, entry_checksum) in entries:
-            f.write(name)
+        f.write(struct.pack(">L", entry_checksum))
-        for (name, offset, entry_checksum) in entries:
+    for (name, offset, entry_checksum) in entries:
-            f.write(struct.pack(">L", entry_checksum))
+        # FIXME: handle if MSBit is set in offset
-        for (name, offset, entry_checksum) in entries:
+        f.write(struct.pack(">L", offset))
-            # FIXME: handle if MSBit is set in offset
+    # FIXME: handle table for pack files > 8 Gb
-            f.write(struct.pack(">L", offset))
+    assert len(pack_checksum) == 20
-        # FIXME: handle table for pack files > 8 Gb
+    f.write(pack_checksum)
-        assert len(pack_checksum) == 20
+    return f.write_sha()
-        f.write(pack_checksum)
-    finally:
-        f.close()
 
 
 class Pack(object):
@@ -1281,18 +1294,28 @@ class Pack(object):
 
     def __init__(self, basename):
         self._basename = basename
-        self._data_path = self._basename + ".pack"
-        self._idx_path = self._basename + ".idx"
         self._data = None
         self._idx = None
+        self._idx_path = self._basename + ".idx"
+        self._data_path = self._basename + ".pack"
+        self._data_load = lambda: PackData(self._data_path)
+        self._idx_load = lambda: load_pack_index(self._idx_path)
+
+    @classmethod
+    def from_lazy_objects(self, data_fn, idx_fn):
+        """Create a new pack object from callables to load pack data and 
+        index objects."""
+        ret = Pack("")
+        ret._data_load = data_fn
+        ret._idx_load = idx_fn
+        return ret
 
     @classmethod
     def from_objects(self, data, idx):
         """Create a new pack object from pack data and index objects."""
         ret = Pack("")
-        ret._data = data
+        ret._data_load = lambda: data
-        ret._idx = idx
+        ret._idx_load = lambda: idx
-        data.pack = ret
         return ret
 
     def name(self):
@@ -1303,7 +1326,7 @@ class Pack(object):
     def data(self):
         """The pack data object being used."""
         if self._data is None:
-            self._data = PackData(self._data_path)
+            self._data = self._data_load()
             self._data.pack = self
             assert len(self.index) == len(self._data)
             idx_stored_checksum = self.index.get_pack_checksum()
@@ -1320,7 +1343,7 @@ class Pack(object):
         :note: This may be an in-memory index
         """
         if self._idx is None:
-            self._idx = load_pack_index(self._idx_path)
+            self._idx = self._idx_load()
         return self._idx
 
     def close(self):

dulwich/patch.py

@@ -1,16 +1,16 @@
-# patch.py -- For dealing wih packed-style patches.
+# patch.py -- For dealing with packed-style patches.
-# Copryight (C) 2009 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your option) a later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

dulwich/protocol.py

@@ -1,5 +1,5 @@
 # protocol.py -- Shared parts of the git protocols
-# Copryight (C) 2008 John Carr <john.carr@unrouted.co.uk>
+# Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
 # Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
 #
 # This program is free software; you can redistribute it and/or

dulwich/repo.py

@@ -1,4 +1,4 @@
-# repo.py -- For dealing wih git repositories.
+# repo.py -- For dealing with git repositories.
 # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
 #
@@ -21,7 +21,7 @@
 
 """Repository access."""
 
-
+from cStringIO import StringIO
 import errno
 import os
 
@@ -42,6 +42,7 @@ from dulwich.file import (
     )
 from dulwich.object_store import (
     DiskObjectStore,
+    MemoryObjectStore,
     )
 from dulwich.objects import (
     Blob,
@@ -674,9 +675,8 @@ def _split_ref_line(line):
 def read_packed_refs(f):
     """Read a packed refs file.
 
-    Yields tuples with SHA1s and ref names.
-
     :param f: file-like object to read from
+    :return: Iterator over tuples with SHA1s and ref names.
     """
     for l in f:
         if l[0] == "#":
@@ -750,6 +750,16 @@ class BaseRepo(object):
         self.object_store = object_store
         self.refs = refs
 
+    def _init_files(self):
+        """Initialize a default set of named files."""
+        self._put_named_file('description', "Unnamed repository")
+        self._put_named_file('config', ('[core]\n'
+                                        'repositoryformatversion = 0\n'
+                                        'filemode = true\n'
+                                        'bare = false\n'
+                                        'logallrefupdates = true\n'))
+        self._put_named_file(os.path.join('info', 'exclude'), '')
+
     def get_named_file(self, path):
         """Get a file from the control dir with a specific name.
 
@@ -762,6 +772,14 @@ class BaseRepo(object):
         """
         raise NotImplementedError(self.get_named_file)
 
+    def _put_named_file(self, path, contents):
+        """Write a file to the control dir with the given name and contents.
+
+        :param path: The path to the file, relative to the control dir.
+        :param contents: A string to write to the file.
+        """
+        raise NotImplementedError(self._put_named_file)
+
     def open_index(self):
         """Open the index for this repository.
 
@@ -1072,8 +1090,12 @@ class Repo(BaseRepo):
         return self._controldir
 
     def _put_named_file(self, path, contents):
-        """Write a file from the control dir with a specific name and contents.
+        """Write a file to the control dir with the given name and contents.
+
+        :param path: The path to the file, relative to the control dir.
+        :param contents: A string to write to the file.
         """
+        path = path.lstrip(os.path.sep)
         f = GitFile(os.path.join(self.controldir(), path), 'wb')
         try:
             f.write(contents)
@@ -1090,8 +1112,11 @@ class Repo(BaseRepo):
         :param path: The path to the file, relative to the control dir.
         :return: An open file object, or None if the file does not exist.
         """
+        # TODO(dborowitz): sanitize filenames, since this is used directly by
+        # the dumb web serving code.
+        path = path.lstrip(os.path.sep)
         try:
-            return open(os.path.join(self.controldir(), path.lstrip('/')), 'rb')
+            return open(os.path.join(self.controldir(), path), 'rb')
         except (IOError, OSError), e:
             if e.errno == errno.ENOENT:
                 return None
@@ -1162,14 +1187,57 @@ class Repo(BaseRepo):
         DiskObjectStore.init(os.path.join(path, OBJECTDIR))
         ret = cls(path)
         ret.refs.set_symbolic_ref("HEAD", "refs/heads/master")
-        ret._put_named_file('description', "Unnamed repository")
+        ret._init_files()
-        ret._put_named_file('config', """[core]
-    repositoryformatversion = 0
-    filemode = true
-    bare = false
-    logallrefupdates = true
-""")
-        ret._put_named_file(os.path.join('info', 'exclude'), '')
         return ret
 
     create = init_bare
+
+
+class MemoryRepo(BaseRepo):
+    """Repo that stores refs, objects, and named files in memory.
+
+    MemoryRepos are always bare: they have no working tree and no index, since
+    those have a stronger dependency on the filesystem.
+    """
+
+    def __init__(self):
+        BaseRepo.__init__(self, MemoryObjectStore(), DictRefsContainer({}))
+        self._named_files = {}
+        self.bare = True
+
+    def _put_named_file(self, path, contents):
+        """Write a file to the control dir with the given name and contents.
+
+        :param path: The path to the file, relative to the control dir.
+        :param contents: A string to write to the file.
+        """
+        self._named_files[path] = contents
+
+    def get_named_file(self, path):
+        """Get a file from the control dir with a specific name.
+
+        Although the filename should be interpreted as a filename relative to
+        the control dir in a disk-baked Repo, the object returned need not be
+        pointing to a file in that location.
+
+        :param path: The path to the file, relative to the control dir.
+        :return: An open file object, or None if the file does not exist.
+        """
+        contents = self._named_files.get(path, None)
+        if contents is None:
+            return None
+        return StringIO(contents)
+
+    def open_index(self):
+        """Fail to open index for this repo, since it is bare."""
+        raise NoIndexPresent()
+
+    @classmethod
+    def init_bare(cls, objects, refs):
+        ret = cls()
+        for obj in objects:
+            ret.object_store.add_object(obj)
+        for refname, sha in refs.iteritems():
+            ret.refs[refname] = sha
+        ret._init_files()
+        return ret

dulwich/server.py

@@ -1,5 +1,5 @@
 # server.py -- Implementation of the server side git protocols
-# Copryight (C) 2008 John Carr <john.carr@unrouted.co.uk>
+# Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -16,20 +16,21 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-
 """Git smart network protocol server implementation.
 
 For more detailed implementation on the network protocol, see the
 Documentation/technical directory in the cgit distribution, and in particular:
-    Documentation/technical/protocol-capabilities.txt
+
-    Documentation/technical/pack-protocol.txt
+* Documentation/technical/protocol-capabilities.txt
+* Documentation/technical/pack-protocol.txt
 """
 
 
 import collections
 import socket
-import zlib
 import SocketServer
+import sys
+import zlib
 
 from dulwich.errors import (
     ApplyDeltaError,
@@ -37,6 +38,7 @@ from dulwich.errors import (
     GitProtocolError,
     ObjectFormatException,
     )
+from dulwich import log_utils
 from dulwich.objects import (
     hex_to_sha,
     )
@@ -56,7 +58,12 @@ from dulwich.protocol import (
     extract_capabilities,
     extract_want_line_capabilities,
     )
+from dulwich.repo import (
+    Repo,
+    )
+
 
+logger = log_utils.getLogger(__name__)
 
 
 class Backend(object):
@@ -141,6 +148,7 @@ class DictBackend(Backend):
         self.repos = repos
 
     def open_repository(self, path):
+        logger.debug('Opening repository at %s', path)
         # FIXME: What to do in case there is no repo ?
         return self.repos[path]
 
@@ -178,6 +186,7 @@ class Handler(object):
                 raise GitProtocolError('Client does not support required '
                                        'capability %s.' % cap)
         self._client_capabilities = set(caps)
+        logger.info('Client capabilities: %s', caps)
 
     def has_capability(self, cap):
         if self._client_capabilities is None:
@@ -671,6 +680,7 @@ class TCPGitRequestHandler(SocketServer.StreamRequestHandler):
     def handle(self):
         proto = ReceivableProtocol(self.connection.recv, self.wfile.write)
         command, args = proto.read_cmd()
+        logger.info('Handling %s request, args=%s', command, args)
 
         cls = self.handlers.get(command, None)
         if not callable(cls):
@@ -690,5 +700,27 @@ class TCPGitServer(SocketServer.TCPServer):
     def __init__(self, backend, listen_addr, port=TCP_GIT_PORT, handlers=None):
         self.backend = backend
         self.handlers = handlers
+        logger.info('Listening for TCP connections on %s:%d', listen_addr, port)
         SocketServer.TCPServer.__init__(self, (listen_addr, port),
                                         self._make_handler)
+
+    def verify_request(self, request, client_address):
+        logger.info('Handling request from %s', client_address)
+        return True
+
+    def handle_error(self, request, client_address):
+        logger.exception('Exception happened during processing of request '
+                         'from %s', client_address)
+
+
+def main(argv=sys.argv):
+    """Entry point for starting a TCP git server."""
+    if len(argv) > 1:
+        gitdir = argv[1]
+    else:
+        gitdir = '.'
+
+    log_utils.default_logging_config()
+    backend = DictBackend({'/': Repo(gitdir)})
+    server = TCPGitServer(backend, 'localhost')
+    server.serve_forever()

dulwich/tests/__init__.py

@@ -1,17 +1,17 @@
 # __init__.py -- The tests for dulwich
 # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
-# of the License or (at your option) any later version of 
+# of the License or (at your option) any later version of
 # the License.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
@@ -21,10 +21,37 @@
 
 import unittest
 
-# XXX: Ideally we should allow other test runners as well, 
+try:
-# but unfortunately unittest doesn't have a SkipTest/TestSkipped
+    from testtools.testcase import TestCase
-# exception.
+except ImportError:
-from nose import SkipTest as TestSkipped
+    from unittest import TestCase
+
+try:
+    # If Python itself provides an exception, use that
+    from unittest import SkipTest as TestSkipped
+except ImportError:
+    # Check if the nose exception can be used
+    try:
+        import nose
+    except ImportError:
+        try:
+            import testtools.testcase
+        except ImportError:
+            class TestSkipped(Exception):
+                def __init__(self, msg):
+                    self.msg = msg
+        else:
+            TestSkipped = testtools.testcase.TestCase.skipException
+    else:
+        TestSkipped = nose.SkipTest
+        try:
+            import testtools.testcase
+        except ImportError:
+            pass
+        else:
+            # Make testtools use the same exception class as nose
+            testtools.testcase.TestCase.skipException = TestSkipped
+
 
 def test_suite():
     names = [

dulwich/tests/compat/test_client.py

@@ -22,6 +22,7 @@
 import os
 import shutil
 import signal
+import subprocess
 import tempfile
 
 from dulwich import client
@@ -29,7 +30,6 @@ from dulwich import errors
 from dulwich import file
 from dulwich import index
 from dulwich import protocol
-from dulwich import object_store
 from dulwich import objects
 from dulwich import repo
 from dulwich.tests import (
@@ -43,36 +43,16 @@ from utils import (
     run_git,
     )
 
-class DulwichClientTest(CompatTestCase):
+class DulwichClientTestBase(object):
     """Tests for client/server compatibility."""
 
     def setUp(self):
-        if check_for_daemon(limit=1):
-            raise TestSkipped('git-daemon was already running on port %s' %
-                              protocol.TCP_GIT_PORT)
-        CompatTestCase.setUp(self)
-        fd, self.pidfile = tempfile.mkstemp(prefix='dulwich-test-git-client',
-                                            suffix=".pid")
-        os.fdopen(fd).close()
         self.gitroot = os.path.dirname(import_repo_to_dir('server_new.export'))
         dest = os.path.join(self.gitroot, 'dest')
         file.ensure_dir_exists(dest)
-        run_git(['init', '--bare'], cwd=dest)
+        run_git(['init', '--quiet', '--bare'], cwd=dest)
-        run_git(
-            ['daemon', '--verbose', '--export-all',
-             '--pid-file=%s' % self.pidfile, '--base-path=%s' % self.gitroot,
-             '--detach', '--reuseaddr', '--enable=receive-pack',
-             '--listen=localhost', self.gitroot], cwd=self.gitroot)
-        if not check_for_daemon():
-            raise TestSkipped('git-daemon failed to start')
 
     def tearDown(self):
-        CompatTestCase.tearDown(self)
-        try:
-            os.kill(int(open(self.pidfile).read().strip()), signal.SIGKILL)
-            os.unlink(self.pidfile)
-        except (OSError, IOError):
-            pass
         shutil.rmtree(self.gitroot)
 
     def assertDestEqualsSrc(self):
@@ -80,13 +60,19 @@ class DulwichClientTest(CompatTestCase):
         dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
         self.assertReposEqual(src, dest)
 
+    def _client(self):
+        raise NotImplementedError()
+
+    def _build_path(self):
+        raise NotImplementedError()
+
     def _do_send_pack(self):
-        c = client.TCPGitClient('localhost')
+        c = self._client()
         srcpath = os.path.join(self.gitroot, 'server_new.export')
         src = repo.Repo(srcpath)
         sendrefs = dict(src.get_refs())
         del sendrefs['HEAD']
-        c.send_pack('/dest', lambda _: sendrefs,
+        c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
                     src.object_store.generate_pack_contents)
 
     def test_send_pack(self):
@@ -100,20 +86,21 @@ class DulwichClientTest(CompatTestCase):
         self._do_send_pack()
 
     def test_send_without_report_status(self):
-        c = client.TCPGitClient('localhost')
+        c = self._client()
         c._send_capabilities.remove('report-status')
         srcpath = os.path.join(self.gitroot, 'server_new.export')
         src = repo.Repo(srcpath)
         sendrefs = dict(src.get_refs())
         del sendrefs['HEAD']
-        c.send_pack('/dest', lambda _: sendrefs,
+        c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
                     src.object_store.generate_pack_contents)
         self.assertDestEqualsSrc()
 
     def disable_ff_and_make_dummy_commit(self):
         # disable non-fast-forward pushes to the server
         dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
-        run_git(['config', 'receive.denyNonFastForwards', 'true'], cwd=dest.path)
+        run_git(['config', 'receive.denyNonFastForwards', 'true'],
+                cwd=dest.path)
         b = objects.Blob.from_string('hi')
         dest.object_store.add_object(b)
         t = index.commit_tree(dest.object_store, [('hi', b.id, 0100644)])
@@ -137,9 +124,9 @@ class DulwichClientTest(CompatTestCase):
         dest, dummy_commit = self.disable_ff_and_make_dummy_commit()
         dest.refs['refs/heads/master'] = dummy_commit
         sendrefs, gen_pack = self.compute_send()
-        c = client.TCPGitClient('localhost')
+        c = self._client()
         try:
-            c.send_pack('/dest', lambda _: sendrefs, gen_pack)
+            c.send_pack(self._build_path('/dest'), lambda _: sendrefs, gen_pack)
         except errors.UpdateRefsError, e:
             self.assertEqual('refs/heads/master failed to update', str(e))
             self.assertEqual({'refs/heads/branch': 'ok',
@@ -151,9 +138,9 @@ class DulwichClientTest(CompatTestCase):
         # set up for two non-ff errors
         dest.refs['refs/heads/branch'] = dest.refs['refs/heads/master'] = dummy
         sendrefs, gen_pack = self.compute_send()
-        c = client.TCPGitClient('localhost')
+        c = self._client()
         try:
-            c.send_pack('/dest', lambda _: sendrefs, gen_pack)
+            c.send_pack(self._build_path('/dest'), lambda _: sendrefs, gen_pack)
         except errors.UpdateRefsError, e:
             self.assertEqual('refs/heads/branch, refs/heads/master failed to '
                              'update', str(e))
@@ -162,9 +149,9 @@ class DulwichClientTest(CompatTestCase):
                              e.ref_status)
 
     def test_fetch_pack(self):
-        c = client.TCPGitClient('localhost')
+        c = self._client()
         dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
-        refs = c.fetch('/server_new.export', dest)
+        refs = c.fetch(self._build_path('/server_new.export'), dest)
         map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
         self.assertDestEqualsSrc()
 
@@ -172,8 +159,87 @@ class DulwichClientTest(CompatTestCase):
         self.test_fetch_pack()
         dest, dummy = self.disable_ff_and_make_dummy_commit()
         dest.refs['refs/heads/master'] = dummy
-        c = client.TCPGitClient('localhost')
+        c = self._client()
         dest = repo.Repo(os.path.join(self.gitroot, 'server_new.export'))
-        refs = c.fetch('/dest', dest)
+        refs = c.fetch(self._build_path('/dest'), dest)
         map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
         self.assertDestEqualsSrc()
+
+
+class DulwichTCPClientTest(CompatTestCase, DulwichClientTestBase):
+    def setUp(self):
+        CompatTestCase.setUp(self)
+        DulwichClientTestBase.setUp(self)
+        if check_for_daemon(limit=1):
+            raise TestSkipped('git-daemon was already running on port %s' %
+                              protocol.TCP_GIT_PORT)
+        fd, self.pidfile = tempfile.mkstemp(prefix='dulwich-test-git-client',
+                                            suffix=".pid")
+        os.fdopen(fd).close()
+        run_git(
+            ['daemon', '--verbose', '--export-all',
+             '--pid-file=%s' % self.pidfile, '--base-path=%s' % self.gitroot,
+             '--detach', '--reuseaddr', '--enable=receive-pack',
+             '--listen=localhost', self.gitroot], cwd=self.gitroot)
+        if not check_for_daemon():
+            raise TestSkipped('git-daemon failed to start')
+
+    def tearDown(self):
+        try:
+            os.kill(int(open(self.pidfile).read().strip()), signal.SIGKILL)
+            os.unlink(self.pidfile)
+        except (OSError, IOError):
+            pass
+        DulwichClientTestBase.tearDown(self)
+        CompatTestCase.tearDown(self)
+
+    def _client(self):
+        return client.TCPGitClient('localhost')
+
+    def _build_path(self, path):
+        return path
+
+
+class TestSSHVendor(object):
+    @staticmethod
+    def connect_ssh(host, command, username=None, port=None):
+        cmd, path = command[0].replace("'", '').split(' ')
+        cmd = cmd.split('-', 1)
+        p = subprocess.Popen(cmd + [path], stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        return client.SubprocessWrapper(p)
+
+
+class DulwichMockSSHClientTest(CompatTestCase, DulwichClientTestBase):
+    def setUp(self):
+        CompatTestCase.setUp(self)
+        DulwichClientTestBase.setUp(self)
+        self.real_vendor = client.get_ssh_vendor
+        client.get_ssh_vendor = TestSSHVendor
+
+    def tearDown(self):
+        DulwichClientTestBase.tearDown(self)
+        CompatTestCase.tearDown(self)
+        client.get_ssh_vendor = self.real_vendor
+
+    def _client(self):
+        return client.SSHGitClient('localhost')
+
+    def _build_path(self, path):
+        return self.gitroot + path
+
+
+class DulwichSubprocessClientTest(CompatTestCase, DulwichClientTestBase):
+    def setUp(self):
+        CompatTestCase.setUp(self)
+        DulwichClientTestBase.setUp(self)
+
+    def tearDown(self):
+        DulwichClientTestBase.tearDown(self)
+        CompatTestCase.tearDown(self)
+
+    def _client(self):
+        return client.SubprocessGitClient()
+
+    def _build_path(self, path):
+        return self.gitroot + path

dulwich/tests/compat/test_pack.py

@@ -1,4 +1,4 @@
-# test_pack.py -- Compatibilty tests for git packs.
+# test_pack.py -- Compatibility tests for git packs.
 # Copyright (C) 2010 Google, Inc.
 #
 # This program is free software; you can redistribute it and/or
@@ -17,7 +17,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-"""Compatibilty tests for git packs."""
+"""Compatibility tests for git packs."""
 
 
 import binascii

dulwich/tests/compat/test_server.py

@@ -1,4 +1,4 @@
-# test_server.py -- Compatibilty tests for git server.
+# test_server.py -- Compatibility tests for git server.
 # Copyright (C) 2010 Google, Inc.
 #
 # This program is free software; you can redistribute it and/or
@@ -17,7 +17,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-"""Compatibilty tests between Dulwich and the cgit server.
+"""Compatibility tests between Dulwich and the cgit server.
 
 Warning: these tests should be fairly stable, but when writing/debugging new
 tests, deadlocks may freeze the test process such that it cannot be Ctrl-C'ed.
@@ -30,9 +30,6 @@ from dulwich.server import (
     DictBackend,
     TCPGitServer,
     )
-from dulwich.tests import (
-    TestSkipped,
-    )
 from server_utils import (
     ServerTests,
     ShutdownServerMixIn,

dulwich/tests/compat/test_web.py

@@ -1,4 +1,4 @@
-# test_web.py -- Compatibilty tests for the git web server.
+# test_web.py -- Compatibility tests for the git web server.
 # Copyright (C) 2010 Google, Inc.
 #
 # This program is free software; you can redistribute it and/or
@@ -17,7 +17,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-"""Compatibilty tests between Dulwich and the cgit HTTP server.
+"""Compatibility tests between Dulwich and the cgit HTTP server.
 
 Warning: these tests should be fairly stable, but when writing/debugging new
 tests, deadlocks may freeze the test process such that it cannot be Ctrl-C'ed.
@@ -27,6 +27,7 @@ On *nix, you can kill the tests with Ctrl-Z, "kill %".
 import threading
 from wsgiref import simple_server
 
+import dulwich
 from dulwich.server import (
     DictBackend,
     )
@@ -35,6 +36,7 @@ from dulwich.tests import (
     )
 from dulwich.web import (
     HTTPGitApplication,
+    HTTPGitRequestHandler,
     )
 
 from server_utils import (
@@ -72,8 +74,9 @@ class WebTests(ServerTests):
     def _start_server(self, repo):
         backend = DictBackend({'/': repo})
         app = self._make_app(backend)
-        dul_server = simple_server.make_server('localhost', 0, app,
+        dul_server = simple_server.make_server(
-                                               server_class=WSGIServer)
+          'localhost', 0, app, server_class=WSGIServer,
+          handler_class=HTTPGitRequestHandler)
         threading.Thread(target=dul_server.serve_forever).start()
         self._server = dul_server
         _, port = dul_server.socket.getsockname()

dulwich/tests/compat/utils.py

@@ -25,12 +25,12 @@ import socket
 import subprocess
 import tempfile
 import time
-import unittest
 
 from dulwich.repo import Repo
 from dulwich.protocol import TCP_GIT_PORT
 
 from dulwich.tests import (
+    TestCase,
     TestSkipped,
     )
 
@@ -127,7 +127,7 @@ def import_repo_to_dir(name):
                                'repos', name)
     temp_repo_dir = os.path.join(temp_dir, name)
     export_file = open(export_path, 'rb')
-    run_git_or_fail(['init', '--bare', temp_repo_dir])
+    run_git_or_fail(['init', '--quiet', '--bare', temp_repo_dir])
     run_git_or_fail(['fast-import'], input=export_file.read(),
                     cwd=temp_repo_dir)
     export_file.close()
@@ -171,7 +171,7 @@ def check_for_daemon(limit=10, delay=0.1, timeout=0.1, port=TCP_GIT_PORT):
     return False
 
 
-class CompatTestCase(unittest.TestCase):
+class CompatTestCase(TestCase):
     """Test case that requires git for compatibility checks.
 
     Subclasses can change the git version required by overriding
@@ -181,6 +181,7 @@ class CompatTestCase(unittest.TestCase):
     min_git_version = (1, 5, 0)
 
     def setUp(self):
+        super(CompatTestCase, self).setUp()
         require_git_version(self.min_git_version)
 
     def assertReposEqual(self, repo1, repo2):

dulwich/tests/test_client.py

@@ -17,21 +17,39 @@
 # MA  02110-1301, USA.
 
 from cStringIO import StringIO
-from unittest import TestCase
 
 from dulwich.client import (
     GitClient,
+    SSHGitClient,
     )
+from dulwich.tests import (
+    TestCase,
+    )
+from dulwich.protocol import (
+    Protocol,
+    )
+
+
+class DummyClient(GitClient):
+    def __init__(self, can_read, read, write):
+        self.can_read = can_read
+        self.read = read
+        self.write = write
+        GitClient.__init__(self)
+
+    def _connect(self, service, path):
+        return Protocol(self.read, self.write), self.can_read
 
 
 # TODO(durin42): add unit-level tests of GitClient
 class GitClientTests(TestCase):
 
     def setUp(self):
+        super(GitClientTests, self).setUp()
         self.rout = StringIO()
         self.rin = StringIO()
-        self.client = GitClient(lambda x: True, self.rin.read,
+        self.client = DummyClient(lambda x: True, self.rin.read,
-            self.rout.write)
+                                  self.rout.write)
 
     def test_caps(self):
         self.assertEquals(set(['multi_ack', 'side-band-64k', 'ofs-delta',
@@ -47,3 +65,18 @@ class GitClientTests(TestCase):
         self.rin.seek(0)
         self.client.fetch_pack("bla", lambda heads: [], None, None, None)
         self.assertEquals(self.rout.getvalue(), "0000")
+
+
+class SSHGitClientTests(TestCase):
+
+    def setUp(self):
+        super(SSHGitClientTests, self).setUp()
+        self.client = SSHGitClient("git.samba.org")
+
+    def test_default_command(self):
+        self.assertEquals("git-upload-pack", self.client._get_cmd_path("upload-pack"))
+
+    def test_alternative_command_path(self):
+        self.client.alternative_paths["upload-pack"] = "/usr/lib/git/git-upload-pack"
+        self.assertEquals("/usr/lib/git/git-upload-pack", self.client._get_cmd_path("upload-pack"))
+

dulwich/tests/test_fastexport.py

@@ -1,17 +1,17 @@
 # test_fastexport.py -- Fast export/import functionality
 # Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
-# of the License or (at your option) any later version of 
+# of the License or (at your option) any later version of
 # the License.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
@@ -19,7 +19,6 @@
 
 from cStringIO import StringIO
 import stat
-from unittest import TestCase
 
 from dulwich.fastexport import (
     FastExporter,
@@ -32,6 +31,9 @@ from dulwich.objects import (
     Commit,
     Tree,
     )
+from dulwich.tests import (
+    TestCase,
+    )
 
 
 class FastExporterTests(TestCase):

dulwich/tests/test_file.py

@@ -16,21 +16,23 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-
 import errno
 import os
 import shutil
 import sys
 import tempfile
-import unittest
 
 from dulwich.file import GitFile, fancy_rename
-from dulwich.tests import TestSkipped
+from dulwich.tests import (
+    TestCase,
+    TestSkipped,
+    )
 
 
-class FancyRenameTests(unittest.TestCase):
+class FancyRenameTests(TestCase):
 
     def setUp(self):
+        super(FancyRenameTests, self).setUp()
         self._tempdir = tempfile.mkdtemp()
         self.foo = self.path('foo')
         self.bar = self.path('bar')
@@ -38,6 +40,7 @@ class FancyRenameTests(unittest.TestCase):
 
     def tearDown(self):
         shutil.rmtree(self._tempdir)
+        super(FancyRenameTests, self).tearDown()
 
     def path(self, filename):
         return os.path.join(self._tempdir, filename)
@@ -83,9 +86,10 @@ class FancyRenameTests(unittest.TestCase):
         new_f.close()
 
 
-class GitFileTests(unittest.TestCase):
+class GitFileTests(TestCase):
 
     def setUp(self):
+        super(GitFileTests, self).setUp()
         self._tempdir = tempfile.mkdtemp()
         f = open(self.path('foo'), 'wb')
         f.write('foo contents')
@@ -93,6 +97,7 @@ class GitFileTests(unittest.TestCase):
 
     def tearDown(self):
         shutil.rmtree(self._tempdir)
+        super(GitFileTests, self).tearDown()
 
     def path(self, filename):
         return os.path.join(self._tempdir, filename)
@@ -192,6 +197,9 @@ class GitFileTests(unittest.TestCase):
     def test_abort_close_removed(self):
         foo = self.path('foo')
         f = GitFile(foo, 'wb')
+
+        f._file.close()
         os.remove(foo+".lock")
+
         f.abort()
         self.assertTrue(f._closed)

dulwich/tests/test_index.py

@@ -16,7 +16,6 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-
 """Tests for the index."""
 
 
@@ -28,7 +27,6 @@ import shutil
 import stat
 import struct
 import tempfile
-from unittest import TestCase
 
 from dulwich.index import (
     Index,
@@ -44,6 +42,7 @@ from dulwich.object_store import (
 from dulwich.objects import (
     Blob,
     )
+from dulwich.tests import TestCase
 
 
 class IndexTestCase(TestCase):

dulwich/tests/test_lru_cache.py

@@ -19,10 +19,12 @@
 from dulwich import (
     lru_cache,
     )
-import unittest
+from dulwich.tests import (
+    TestCase,
+    )
 
 
-class TestLRUCache(unittest.TestCase):
+class TestLRUCache(TestCase):
     """Test that LRU cache properly keeps track of entries."""
 
     def test_cache_size(self):
@@ -285,7 +287,7 @@ class TestLRUCache(unittest.TestCase):
         self.assertEqual([6, 7, 8, 9, 10, 11], sorted(cache.keys()))
 
 
-class TestLRUSizeCache(unittest.TestCase):
+class TestLRUSizeCache(TestCase):
 
     def test_basic_init(self):
         cache = lru_cache.LRUSizeCache()

dulwich/tests/test_object_store.py

@@ -16,14 +16,12 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-
 """Tests for the object store interface."""
 
 
 import os
 import shutil
 import tempfile
-from unittest import TestCase
 
 from dulwich.objects import (
     Blob,
@@ -32,6 +30,9 @@ from dulwich.object_store import (
     DiskObjectStore,
     MemoryObjectStore,
     )
+from dulwich.tests import (
+    TestCase,
+    )
 from utils import (
     make_object,
     )
@@ -82,7 +83,23 @@ class MemoryObjectStoreTests(ObjectStoreTests, TestCase):
         self.store = MemoryObjectStore()
 
 
-class DiskObjectStoreTests(ObjectStoreTests, TestCase):
+class PackBasedObjectStoreTests(ObjectStoreTests):
+
+    def test_empty_packs(self):
+        self.assertEquals([], self.store.packs)
+
+    def test_pack_loose_objects(self):
+        b1 = make_object(Blob, data="yummy data")
+        self.store.add_object(b1)
+        b2 = make_object(Blob, data="more yummy data")
+        self.store.add_object(b2)
+        self.assertEquals([], self.store.packs)
+        self.assertEquals(2, self.store.pack_loose_objects())
+        self.assertNotEquals([], self.store.packs)
+        self.assertEquals(0, self.store.pack_loose_objects())
+
+
+class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
 
     def setUp(self):
         TestCase.setUp(self)
@@ -97,9 +114,4 @@ class DiskObjectStoreTests(ObjectStoreTests, TestCase):
         o = DiskObjectStore(self.store_dir)
         self.assertEquals(os.path.join(self.store_dir, "pack"), o.pack_dir)
 
-    def test_empty_packs(self):
-        o = DiskObjectStore(self.store_dir)
-        self.assertEquals([], o.packs)
-
-
 # TODO: MissingObjectFinderTests

dulwich/tests/test_objects.py

@@ -1,35 +1,33 @@
 # test_objects.py -- tests for objects.py
 # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
-# of the License or (at your option) any later version of 
+# of the License or (at your option) any later version of
 # the License.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-
 """Tests for git base objects."""
 
 # TODO: Round-trip parse-serialize-parse and serialize-parse-serialize tests.
 
 
+from cStringIO import StringIO
 import datetime
 import os
 import stat
-import unittest
 
 from dulwich.errors import (
-    ChecksumMismatch,
     ObjectFormatException,
     )
 from dulwich.objects import (
@@ -46,8 +44,11 @@ from dulwich.objects import (
     parse_timezone,
     parse_tree,
     _parse_tree_py,
+    sorted_tree_items,
+    _sorted_tree_items_py,
     )
 from dulwich.tests import (
+    TestCase,
     TestSkipped,
     )
 from utils import (
@@ -96,7 +97,7 @@ except ImportError:
                 return
 
 
-class TestHexToSha(unittest.TestCase):
+class TestHexToSha(TestCase):
 
     def test_simple(self):
         self.assertEquals("\xab\xcd" * 10, hex_to_sha("abcd" * 10))
@@ -105,7 +106,7 @@ class TestHexToSha(unittest.TestCase):
         self.assertEquals("abcd" * 10, sha_to_hex("\xab\xcd" * 10))
 
 
-class BlobReadTests(unittest.TestCase):
+class BlobReadTests(TestCase):
     """Test decompression of blobs"""
 
     def get_sha_file(self, cls, base, sha):
@@ -147,6 +148,13 @@ class BlobReadTests(unittest.TestCase):
         self.assertEqual(b.data, string)
         self.assertEqual(b.sha().hexdigest(), b_sha)
 
+    def test_legacy_from_file(self):
+        b1 = Blob.from_string("foo")
+        b_raw = b1.as_legacy_object()
+        open('x', 'w+').write(b_raw)
+        b2 = b1.from_file(StringIO(b_raw))
+        self.assertEquals(b1, b2)
+
     def test_chunks(self):
         string = 'test 5\n'
         b = Blob.from_string(string)
@@ -228,7 +236,7 @@ class BlobReadTests(unittest.TestCase):
         self.assertEqual(c.message, 'Merge ../b\n')
 
 
-class ShaFileCheckTests(unittest.TestCase):
+class ShaFileCheckTests(TestCase):
 
     def assertCheckFails(self, cls, data):
         obj = cls()
@@ -243,7 +251,7 @@ class ShaFileCheckTests(unittest.TestCase):
         self.assertEqual(None, obj.check())
 
 
-class CommitSerializationTests(unittest.TestCase):
+class CommitSerializationTests(TestCase):
 
     def make_commit(self, **kwargs):
         attrs = {'tree': 'd80c186a03f423a81b39df39dc87fd269736ca86',
@@ -404,6 +412,19 @@ class CommitParseTests(ShaFileCheckTests):
                 self.assertCheckFails(Commit, text)
 
 
+_TREE_ITEMS = {
+  'a.c': (0100755, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
+  'a': (stat.S_IFDIR, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
+  'a/c': (stat.S_IFDIR, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
+  }
+
+_SORTED_TREE_ITEMS = [
+  ('a.c', 0100755, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
+  ('a', stat.S_IFDIR, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
+  ('a/c', stat.S_IFDIR, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
+  ]
+
+
 class TreeTests(ShaFileCheckTests):
 
     def test_simple(self):
@@ -422,10 +443,9 @@ class TreeTests(ShaFileCheckTests):
 
     def test_tree_dir_sort(self):
         x = Tree()
-        x["a.c"] = (0100755, "d80c186a03f423a81b39df39dc87fd269736ca86")
+        for name, item in _TREE_ITEMS.iteritems():
-        x["a"] = (stat.S_IFDIR, "d80c186a03f423a81b39df39dc87fd269736ca86")
+            x[name] = item
-        x["a/c"] = (stat.S_IFDIR, "d80c186a03f423a81b39df39dc87fd269736ca86")
+        self.assertEquals(_SORTED_TREE_ITEMS, list(x.iteritems()))
-        self.assertEquals(["a.c", "a", "a/c"], [p[0] for p in x.iteritems()])
 
     def _do_test_parse_tree(self, parse_tree):
         dir = os.path.join(os.path.dirname(__file__), 'data', 'trees')
@@ -441,6 +461,32 @@ class TreeTests(ShaFileCheckTests):
             raise TestSkipped('parse_tree extension not found')
         self._do_test_parse_tree(parse_tree)
 
+    def _do_test_sorted_tree_items(self, sorted_tree_items):
+        def do_sort(entries):
+            return list(sorted_tree_items(entries))
+
+        self.assertEqual(_SORTED_TREE_ITEMS, do_sort(_TREE_ITEMS))
+
+        # C/Python implementations may differ in specific error types, but
+        # should all error on invalid inputs.
+        # For example, the C implementation has stricter type checks, so may
+        # raise TypeError where the Python implementation raises AttributeError.
+        errors = (TypeError, ValueError, AttributeError)
+        self.assertRaises(errors, do_sort, 'foo')
+        self.assertRaises(errors, do_sort, {'foo': (1, 2, 3)})
+
+        myhexsha = 'd80c186a03f423a81b39df39dc87fd269736ca86'
+        self.assertRaises(errors, do_sort, {'foo': ('xxx', myhexsha)})
+        self.assertRaises(errors, do_sort, {'foo': (0100755, 12345)})
+
+    def test_sorted_tree_items(self):
+        self._do_test_sorted_tree_items(_sorted_tree_items_py)
+
+    def test_sorted_tree_items_extension(self):
+        if sorted_tree_items is _sorted_tree_items_py:
+            raise TestSkipped('sorted_tree_items extension not found')
+        self._do_test_sorted_tree_items(sorted_tree_items)
+
     def test_check(self):
         t = Tree
         sha = hex_to_sha(a_sha)
@@ -478,7 +524,7 @@ class TreeTests(ShaFileCheckTests):
         self.assertEquals(set(["foo"]), set(t))
 
 
-class TagSerializeTests(unittest.TestCase):
+class TagSerializeTests(TestCase):
 
     def test_serialize_simple(self):
         x = make_object(Tag,
@@ -590,7 +636,7 @@ class TagParseTests(ShaFileCheckTests):
                 self.assertCheckFails(Tag, text)
 
 
-class CheckTests(unittest.TestCase):
+class CheckTests(TestCase):
 
     def test_check_hexsha(self):
         check_hexsha(a_sha, "failed to check good sha")
@@ -621,7 +667,7 @@ class CheckTests(unittest.TestCase):
                           "trailing characters")
 
 
-class TimezoneTests(unittest.TestCase):
+class TimezoneTests(TestCase):
 
     def test_parse_timezone_utc(self):
         self.assertEquals((0, False), parse_timezone("+0000"))

dulwich/tests/test_pack.py

@@ -17,7 +17,6 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-
 """Tests for Dulwich packs."""
 
 
@@ -25,12 +24,14 @@ from cStringIO import StringIO
 import os
 import shutil
 import tempfile
-import unittest
 import zlib
 
 from dulwich.errors import (
     ChecksumMismatch,
     )
+from dulwich.file import (
+    GitFile,
+    )
 from dulwich.objects import (
     hex_to_sha,
     sha_to_hex,
@@ -42,13 +43,14 @@ from dulwich.pack import (
     apply_delta,
     create_delta,
     load_pack_index,
-    hex_to_sha,
     read_zlib_chunks,
-    sha_to_hex,
     write_pack_index_v1,
     write_pack_index_v2,
     write_pack,
     )
+from dulwich.tests import (
+    TestCase,
+    )
 
 pack1_sha = 'bc63ddad95e7321ee734ea11a7a62d314e0d7481'
 
@@ -57,14 +59,16 @@ tree_sha = 'b2a2766a2879c209ab1176e7e778b81ae422eeaa'
 commit_sha = 'f18faa16531ac570a3fdc8c7ca16682548dafd12'
 
 
-class PackTests(unittest.TestCase):
+class PackTests(TestCase):
     """Base class for testing packs"""
 
     def setUp(self):
+        super(PackTests, self).setUp()
         self.tempdir = tempfile.mkdtemp()
 
     def tearDown(self):
         shutil.rmtree(self.tempdir)
+        super(PackTests, self).tearDown()
 
     datadir = os.path.join(os.path.dirname(__file__), 'data/packs')
 
@@ -126,7 +130,7 @@ class PackIndexTests(PackTests):
         self.assertEquals(set([tree_sha, commit_sha, a_sha]), set(p))
 
 
-class TestPackDeltas(unittest.TestCase):
+class TestPackDeltas(TestCase):
 
     test_string1 = 'The answer was flailing in the wind'
     test_string2 = 'The answer was falling down the pipe'
@@ -290,9 +294,17 @@ class BaseTestPackIndexWriting(object):
         except ChecksumMismatch, e:
             self.fail(e)
 
+    def writeIndex(self, filename, entries, pack_checksum):
+        # FIXME: Write to StringIO instead rather than hitting disk ?
+        f = GitFile(filename, "wb")
+        try:
+            self._write_fn(f, entries, pack_checksum)
+        finally:
+            f.close()
+
     def test_empty(self):
         filename = os.path.join(self.tempdir, 'empty.idx')
-        self._write_fn(filename, [], pack_checksum)
+        self.writeIndex(filename, [], pack_checksum)
         idx = load_pack_index(filename)
         self.assertSucceeds(idx.check)
         self.assertEquals(idx.get_pack_checksum(), pack_checksum)
@@ -302,7 +314,7 @@ class BaseTestPackIndexWriting(object):
         entry_sha = hex_to_sha('6f670c0fb53f9463760b7295fbb814e965fb20c8')
         my_entries = [(entry_sha, 178, 42)]
         filename = os.path.join(self.tempdir, 'single.idx')
-        self._write_fn(filename, my_entries, pack_checksum)
+        self.writeIndex(filename, my_entries, pack_checksum)
         idx = load_pack_index(filename)
         self.assertEquals(idx.version, self._expected_version)
         self.assertSucceeds(idx.check)
@@ -321,35 +333,35 @@ class BaseTestPackIndexWriting(object):
                 self.assertTrue(actual_crc is None)
 
 
-class TestPackIndexWritingv1(unittest.TestCase, BaseTestPackIndexWriting):
+class TestPackIndexWritingv1(TestCase, BaseTestPackIndexWriting):
 
     def setUp(self):
-        unittest.TestCase.setUp(self)
+        TestCase.setUp(self)
         BaseTestPackIndexWriting.setUp(self)
         self._has_crc32_checksum = False
         self._expected_version = 1
         self._write_fn = write_pack_index_v1
 
     def tearDown(self):
-        unittest.TestCase.tearDown(self)
+        TestCase.tearDown(self)
         BaseTestPackIndexWriting.tearDown(self)
 
 
-class TestPackIndexWritingv2(unittest.TestCase, BaseTestPackIndexWriting):
+class TestPackIndexWritingv2(TestCase, BaseTestPackIndexWriting):
 
     def setUp(self):
-        unittest.TestCase.setUp(self)
+        TestCase.setUp(self)
         BaseTestPackIndexWriting.setUp(self)
         self._has_crc32_checksum = True
         self._expected_version = 2
         self._write_fn = write_pack_index_v2
 
     def tearDown(self):
-        unittest.TestCase.tearDown(self)
+        TestCase.tearDown(self)
         BaseTestPackIndexWriting.tearDown(self)
 
 
-class ReadZlibTests(unittest.TestCase):
+class ReadZlibTests(TestCase):
 
     decomp = (
       'tree 4ada885c9196b6b6fa08744b5862bf92896fc002\n'
@@ -362,6 +374,7 @@ class ReadZlibTests(unittest.TestCase):
     extra = 'nextobject'
 
     def setUp(self):
+        super(ReadZlibTests, self).setUp()
         self.read = StringIO(self.comp + self.extra).read
 
     def test_decompress_size(self):

dulwich/tests/test_patch.py

@@ -1,16 +1,16 @@
 # test_patch.py -- tests for patch.py
-# Copryight (C) 2010 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your option) a later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
@@ -19,7 +19,6 @@
 """Tests for patch.py."""
 
 from cStringIO import StringIO
-from unittest import TestCase
 
 from dulwich.objects import (
     Commit,
@@ -29,6 +28,7 @@ from dulwich.patch import (
     git_am_patch_split,
     write_commit_patch,
     )
+from dulwich.tests import TestCase
 
 
 class WriteCommitPatchTests(TestCase):

dulwich/tests/test_protocol.py

@@ -1,27 +1,25 @@
 # test_protocol.py -- Tests for the git protocol
 # Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
-# 
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # or (at your option) any later version of the License.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-
 """Tests for the smart protocol utility functions."""
 
 
 from StringIO import StringIO
-from unittest import TestCase
 
 from dulwich.protocol import (
     Protocol,
@@ -33,6 +31,8 @@ from dulwich.protocol import (
     MULTI_ACK,
     MULTI_ACK_DETAILED,
     )
+from dulwich.tests import TestCase
+
 
 class BaseProtocolTests(object):
 

dulwich/tests/test_repository.py

@@ -17,14 +17,12 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-
 """Tests for the repository."""
 
 from cStringIO import StringIO
 import os
 import shutil
 import tempfile
-import unittest
 import warnings
 
 from dulwich import errors
@@ -36,11 +34,15 @@ from dulwich.repo import (
     check_ref_format,
     DictRefsContainer,
     Repo,
+    MemoryRepo,
     read_packed_refs,
     read_packed_refs_with_peeled,
     write_packed_refs,
     _split_ref_line,
     )
+from dulwich.tests import (
+    TestCase,
+    )
 from dulwich.tests.utils import (
     open_repo,
     tear_down_repo,
@@ -49,25 +51,48 @@ from dulwich.tests.utils import (
 missing_sha = 'b91fa4d900e17e99b433218e988c4eb4a3e9a097'
 
 
-class CreateRepositoryTests(unittest.TestCase):
+class CreateRepositoryTests(TestCase):
+
+    def assertFileContentsEqual(self, expected, repo, path):
+        f = repo.get_named_file(path)
+        if not f:
+            self.assertEqual(expected, None)
+        else:
+            try:
+                self.assertEqual(expected, f.read())
+            finally:
+                f.close()
 
-    def test_create(self):
+    def _check_repo_contents(self, repo):
+        self.assertTrue(repo.bare)
+        self.assertFileContentsEqual('Unnamed repository', repo, 'description')
+        self.assertFileContentsEqual('', repo, os.path.join('info', 'exclude'))
+        self.assertFileContentsEqual(None, repo, 'nonexistent file')
+
+    def test_create_disk(self):
         tmp_dir = tempfile.mkdtemp()
         try:
             repo = Repo.init_bare(tmp_dir)
             self.assertEquals(tmp_dir, repo._controldir)
+            self._check_repo_contents(repo)
         finally:
             shutil.rmtree(tmp_dir)
 
+    def test_create_memory(self):
+        repo = MemoryRepo.init_bare([], {})
+        self._check_repo_contents(repo)
+
 
-class RepositoryTests(unittest.TestCase):
+class RepositoryTests(TestCase):
 
     def setUp(self):
+        super(RepositoryTests, self).setUp()
         self._repo = None
 
     def tearDown(self):
         if self._repo is not None:
             tear_down_repo(self._repo)
+        super(RepositoryTests, self).tearDown()
 
     def test_simple_props(self):
         r = self._repo = open_repo('a.git')
@@ -307,7 +332,7 @@ class RepositoryTests(unittest.TestCase):
             shutil.rmtree(r2_dir)
 
 
-class BuildRepoTests(unittest.TestCase):
+class BuildRepoTests(TestCase):
     """Tests that build on-disk repos from scratch.
 
     Repos live in a temp dir and are torn down after each test. They start with
@@ -315,6 +340,7 @@ class BuildRepoTests(unittest.TestCase):
     """
 
     def setUp(self):
+        super(BuildRepoTests, self).setUp()
         repo_dir = os.path.join(tempfile.mkdtemp(), 'test')
         os.makedirs(repo_dir)
         r = self._repo = Repo.init(repo_dir)
@@ -338,6 +364,7 @@ class BuildRepoTests(unittest.TestCase):
 
     def tearDown(self):
         tear_down_repo(self._repo)
+        super(BuildRepoTests, self).tearDown()
 
     def test_build_repo(self):
         r = self._repo
@@ -377,7 +404,7 @@ class BuildRepoTests(unittest.TestCase):
         self.assertEqual([self._root_commit], r[commit_sha].parents)
         self.assertEqual([], list(r.open_index()))
         tree = r[r[commit_sha].tree]
-        self.assertEqual([], tree.iteritems())
+        self.assertEqual([], list(tree.iteritems()))
 
     def test_commit_fail_ref(self):
         r = self._repo
@@ -410,7 +437,7 @@ class BuildRepoTests(unittest.TestCase):
         r.stage(['a'])  # double-stage a deleted path
 
 
-class CheckRefFormatTests(unittest.TestCase):
+class CheckRefFormatTests(TestCase):
     """Tests for the check_ref_format function.
 
     These are the same tests as in the git test suite.
@@ -441,7 +468,7 @@ TWOS = "2" * 40
 THREES = "3" * 40
 FOURS = "4" * 40
 
-class PackedRefsFileTests(unittest.TestCase):
+class PackedRefsFileTests(TestCase):
 
     def test_split_ref_line_errors(self):
         self.assertRaises(errors.PackedRefsException, _split_ref_line,
@@ -598,20 +625,23 @@ class RefsContainerTests(object):
         self.assertFalse('refs/tags/refs-0.2' in self._refs)
 
 
-class DictRefsContainerTests(RefsContainerTests, unittest.TestCase):
+class DictRefsContainerTests(RefsContainerTests, TestCase):
 
     def setUp(self):
+        TestCase.setUp(self)
         self._refs = DictRefsContainer(dict(_TEST_REFS))
 
 
-class DiskRefsContainerTests(RefsContainerTests, unittest.TestCase):
+class DiskRefsContainerTests(RefsContainerTests, TestCase):
 
     def setUp(self):
+        TestCase.setUp(self)
         self._repo = open_repo('refs.git')
         self._refs = self._repo.refs
 
     def tearDown(self):
         tear_down_repo(self._repo)
+        TestCase.tearDown(self)
 
     def test_get_packed_refs(self):
         self.assertEqual({
@@ -740,7 +770,7 @@ class DiskRefsContainerTests(RefsContainerTests, unittest.TestCase):
         f = open(refs_file)
         refs_data = f.read()
         f.close()
-        f = open(refs_file, 'w')
+        f = open(refs_file, 'wb')
         f.write('\n'.join(l for l in refs_data.split('\n')
                           if not l or l[0] not in '#^'))
         f.close()

dulwich/tests/test_server.py

@@ -16,12 +16,9 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-
 """Tests for the smart protocol server."""
 
 
-from unittest import TestCase
-
 from dulwich.errors import (
     GitProtocolError,
     )
@@ -36,6 +33,8 @@ from dulwich.server import (
     SingleAckGraphWalkerImpl,
     UploadPackHandler,
     )
+from dulwich.tests import TestCase
+
 
 
 ONE = '1' * 40
@@ -80,6 +79,7 @@ class TestProto(object):
 class HandlerTestCase(TestCase):
 
     def setUp(self):
+        super(HandlerTestCase, self).setUp()
         self._handler = Handler(Backend(), None)
         self._handler.capabilities = lambda: ('cap1', 'cap2', 'cap3')
         self._handler.required_capabilities = lambda: ('cap2',)
@@ -123,6 +123,7 @@ class HandlerTestCase(TestCase):
 class UploadPackHandlerTestCase(TestCase):
 
     def setUp(self):
+        super(UploadPackHandlerTestCase, self).setUp()
         self._backend = DictBackend({"/": BackendRepo()})
         self._handler = UploadPackHandler(self._backend,
                 ["/", "host=lolcathost"], None, None)
@@ -214,6 +215,7 @@ class TestUploadPackHandler(Handler):
 class ProtocolGraphWalkerTestCase(TestCase):
 
     def setUp(self):
+        super(ProtocolGraphWalkerTestCase, self).setUp()
         # Create the following commit tree:
         #   3---5
         #  /
@@ -358,6 +360,7 @@ class AckGraphWalkerImplTestCase(TestCase):
     """Base setup and asserts for AckGraphWalker tests."""
 
     def setUp(self):
+        super(AckGraphWalkerImplTestCase, self).setUp()
         self._walker = TestProtocolGraphWalker()
         self._walker.lines = [
           ('have', TWO),

dulwich/tests/test_web.py

@@ -1,5 +1,5 @@
 # test_web.py -- Tests for the git HTTP server
-# Copryight (C) 2010 Google, Inc.
+# Copyright (C) 2010 Google, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -20,11 +20,13 @@
 
 from cStringIO import StringIO
 import re
-from unittest import TestCase
 
 from dulwich.objects import (
     Blob,
     )
+from dulwich.tests import (
+    TestCase,
+    )
 from dulwich.web import (
     HTTP_OK,
     HTTP_NOT_FOUND,
@@ -42,6 +44,7 @@ class WebTestCase(TestCase):
     """Base TestCase that sets up some useful instance vars."""
 
     def setUp(self):
+        super(WebTestCase, self).setUp()
         self._environ = {}
         self._req = HTTPGitRequest(self._environ, self._start_response,
                                    handlers=self._handlers())
@@ -282,7 +285,9 @@ class HTTPGitRequestTestCase(WebTestCase):
 
 
 class HTTPGitApplicationTestCase(TestCase):
+
     def setUp(self):
+        super(HTTPGitApplicationTestCase, self).setUp()
         self._app = HTTPGitApplication('backend')
 
     def test_call(self):

dulwich/web.py

@@ -1,5 +1,5 @@
 # web.py -- WSGI smart-http server
-# Copryight (C) 2010 Google, Inc.
+# Copyright (C) 2010 Google, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -19,23 +19,31 @@
 """HTTP server for dulwich that implements the git smart HTTP protocol."""
 
 from cStringIO import StringIO
+import os
 import re
+import sys
 import time
 
 try:
     from urlparse import parse_qs
 except ImportError:
     from dulwich.misc import parse_qs
+from dulwich import log_utils
 from dulwich.protocol import (
     ReceivableProtocol,
     )
+from dulwich.repo import (
+    Repo,
+    )
 from dulwich.server import (
-    ReceivePackHandler,
+    DictBackend,
-    UploadPackHandler,
     DEFAULT_HANDLERS,
     )
 
 
+logger = log_utils.getLogger(__name__)
+
+
 # HTTP error strings
 HTTP_OK = '200 OK'
 HTTP_NOT_FOUND = '404 Not Found'
@@ -77,7 +85,7 @@ def send_file(req, f, content_type):
     :param req: The HTTPGitRequest object to send output to.
     :param f: An open file-like object to send; will be closed.
     :param content_type: The MIME type for the file.
-    :yield: The contents of the file.
+    :return: Iterator over the contents of the file, as chunks.
     """
     if f is None:
         yield req.not_found('File not found')
@@ -98,14 +106,21 @@ def send_file(req, f, content_type):
         raise
 
 
+def _url_to_path(url):
+    return url.replace('/', os.path.sep)
+
+
 def get_text_file(req, backend, mat):
     req.nocache()
-    return send_file(req, get_repo(backend, mat).get_named_file(mat.group()),
+    path = _url_to_path(mat.group())
+    logger.info('Sending plain text file %s', path)
+    return send_file(req, get_repo(backend, mat).get_named_file(path),
                      'text/plain')
 
 
 def get_loose_object(req, backend, mat):
     sha = mat.group(1) + mat.group(2)
+    logger.info('Sending loose object %s', sha)
     object_store = get_repo(backend, mat).object_store
     if not object_store.contains_loose(sha):
         yield req.not_found('Object not found')
@@ -121,13 +136,17 @@ def get_loose_object(req, backend, mat):
 
 def get_pack_file(req, backend, mat):
     req.cache_forever()
-    return send_file(req, get_repo(backend, mat).get_named_file(mat.group()),
+    path = _url_to_path(mat.group())
+    logger.info('Sending pack file %s', path)
+    return send_file(req, get_repo(backend, mat).get_named_file(path),
                      'application/x-git-packed-objects')
 
 
 def get_idx_file(req, backend, mat):
     req.cache_forever()
-    return send_file(req, get_repo(backend, mat).get_named_file(mat.group()),
+    path = _url_to_path(mat.group())
+    logger.info('Sending pack file %s', path)
+    return send_file(req, get_repo(backend, mat).get_named_file(path),
                      'application/x-git-packed-objects-toc')
 
 
@@ -154,6 +173,7 @@ def get_info_refs(req, backend, mat):
         # TODO: select_getanyfile() (see http-backend.c)
         req.nocache()
         req.respond(HTTP_OK, 'text/plain')
+        logger.info('Emulating dumb info/refs')
         repo = get_repo(backend, mat)
         refs = repo.get_refs()
         for name in sorted(refs.iterkeys()):
@@ -174,6 +194,7 @@ def get_info_refs(req, backend, mat):
 def get_info_packs(req, backend, mat):
     req.nocache()
     req.respond(HTTP_OK, 'text/plain')
+    logger.info('Emulating dumb info/packs')
     for pack in get_repo(backend, mat).object_store.packs:
         yield 'P pack-%s.pack\n' % pack.name()
 
@@ -203,6 +224,7 @@ class _LengthLimitedFile(object):
 
 def handle_service_request(req, backend, mat):
     service = mat.group().lstrip('/')
+    logger.info('Handling service request for %s', service)
     handler_cls = req.handlers.get(service, None)
     if handler_cls is None:
         yield req.forbidden('Unsupported service %s' % service)
@@ -255,12 +277,14 @@ class HTTPGitRequest(object):
     def not_found(self, message):
         """Begin a HTTP 404 response and return the text of a message."""
         self._cache_headers = []
+        logger.info('Not found: %s', message)
         self.respond(HTTP_NOT_FOUND, 'text/plain')
         return message
 
     def forbidden(self, message):
         """Begin a HTTP 403 response and return the text of a message."""
         self._cache_headers = []
+        logger.info('Forbidden: %s', message)
         self.respond(HTTP_FORBIDDEN, 'text/plain')
         return message
 
@@ -324,3 +348,55 @@ class HTTPGitApplication(object):
         if handler is None:
             return req.not_found('Sorry, that method is not supported')
         return handler(req, self.backend, mat)
+
+
+# The reference server implementation is based on wsgiref, which is not
+# distributed with python 2.4. If wsgiref is not present, users will not be able
+# to use the HTTP server without a little extra work.
+try:
+    from wsgiref.simple_server import (
+        WSGIRequestHandler,
+        make_server,
+        )
+
+    class HTTPGitRequestHandler(WSGIRequestHandler):
+        """Handler that uses dulwich's logger for logging exceptions."""
+
+        def log_exception(self, exc_info):
+            logger.exception('Exception happened during processing of request',
+                             exc_info=exc_info)
+
+        def log_message(self, format, *args):
+            logger.info(format, *args)
+
+        def log_error(self, *args):
+            logger.error(*args)
+
+
+    def main(argv=sys.argv):
+        """Entry point for starting an HTTP git server."""
+        if len(argv) > 1:
+            gitdir = argv[1]
+        else:
+            gitdir = os.getcwd()
+
+        # TODO: allow serving on other addresses/ports via command-line flag
+        listen_addr=''
+        port = 8000
+
+        log_utils.default_logging_config()
+        backend = DictBackend({'/': Repo(gitdir)})
+        app = HTTPGitApplication(backend)
+        server = make_server(listen_addr, port, app,
+                             handler_class=HTTPGitRequestHandler)
+        logger.info('Listening for HTTP connections on %s:%d', listen_addr,
+                    port)
+        server.serve_forever()
+
+except ImportError:
+    # No wsgiref found; don't provide the reference functionality, but leave the
+    # rest of the WSGI-based implementation.
+    def main(argv=sys.argv):
+        """Stub entry point for failing to start a server without wsgiref."""
+        sys.stderr.write('Sorry, the wsgiref module is required for dul-web.\n')
+        sys.exit(1)

setup.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Setup file for bzr-git
+# Setup file for dulwich
 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
 
 try:
@@ -8,7 +8,7 @@ except ImportError:
     from distutils.core import setup, Extension
 from distutils.core import Distribution
 
-dulwich_version_string = '0.6.0'
+dulwich_version_string = '0.6.1'
 
 include_dirs = []
 # Windows MSVC support