merge upstream snapshot.

AUTHORS

@@ -2,6 +2,7 @@ Jelmer Vernooij <jelmer@samba.org>
 James Westby <jw+debian@jameswestby.net>
 John Carr <john.carr@unrouted.co.uk>
 Dave Borowitz <dborowitz@google.com>
+Chris Eberle <eberle1080@gmail.com>
 
 Hervé Cauwelier <herve@itaapy.com> wrote the original tutorial.
 

NEWS

@@ -10,6 +10,19 @@
 
   * Avoid calling free_objects() on NULL in error cases. (Chris Eberle)
 
+  * Fix use --bare argument to 'dulwich init'. (Chris Eberle)
+
+  * Properly abort connections when the determine_wants function
+    raises an exception. (Jelmer Vernooij, #856769)
+
+ FEATURES
+
+  * Add support for retrieving tarballs from remote servers.
+    (Jelmer Vernooij, #379087)
+
+  * New method ``update_server_info`` which generates data
+    for dumb server access. (Jelmer Vernooij, #731235)
+
 0.8.1	2011-10-31
 
  FEATURES

README

@@ -11,3 +11,11 @@ maintained by Jelmer Vernooij et al.
 Please file bugs in the Dulwich project on Launchpad: 
 
 https://bugs.launchpad.net/dulwich/+filebug
+
+The dulwich documentation can be found in doc/ and on the web:
+
+http://www.samba.org/~jelmer/dulwich/docs/
+
+The API reference can be generated using pydoctor, by running "make pydoctor", or on the web:
+
+http://www.samba.org/~jelmer/dulwich/apidocs

bin/dulwich

@@ -1,6 +1,8 @@
 #!/usr/bin/python -u
+#
 # dulwich - Simple command-line interface to Dulwich
-# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2011 Jelmer Vernooij <jelmer@samba.org>
+# vim: expandtab
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -33,6 +35,14 @@ from dulwich.errors import ApplyDeltaError
 from dulwich.index import Index
 from dulwich.pack import Pack, sha_to_hex
 from dulwich.repo import Repo
+from dulwich.server import update_server_info
+
+
+def cmd_archive(args):
+    opts, args = getopt(args, "", [])
+    client, path = get_transport_and_path(args.pop(0))
+    committish = args.pop(0)
+    client.archive(path, committish, sys.stdout.write, sys.stderr.write)
 
 
 def cmd_fetch_pack(args):
@@ -114,7 +124,7 @@ def cmd_dump_index(args):
 
 
 def cmd_init(args):
-    opts, args = getopt(args, "", ["--bare"])
+    opts, args = getopt(args, "", ["bare"])
     opts = dict(opts)
 
     if args == []:
@@ -165,6 +175,11 @@ def cmd_commit(args):
     r.do_commit(committer=committer, author=author, message=opts["--message"])
 
 
+def cmd_update_server_info(args):
+    r = Repo(".")
+    update_server_info(r)
+
+
 commands = {
     "commit": cmd_commit,
     "fetch-pack": cmd_fetch_pack,
@@ -173,6 +188,8 @@ commands = {
     "init": cmd_init,
     "log": cmd_log,
     "clone": cmd_clone,
+    "archive": cmd_archive,
+    "update-server-info": cmd_update_server_info,
     }
 
 if len(sys.argv) < 2:

debian/changelog

@@ -1,4 +1,4 @@
-dulwich (0.8.1~bzr973-1) UNRELEASED; urgency=low
+dulwich (0.8.1~bzr983-1) UNRELEASED; urgency=low
 
   * Fix Vcs URL.
   * New upstream snapshot.

dulwich/client.py

@@ -419,7 +419,11 @@ class TraditionalGitClient(GitClient):
         negotiated_capabilities = list(self._send_capabilities)
         if 'report-status' not in server_capabilities:
             negotiated_capabilities.remove('report-status')
-        new_refs = determine_wants(old_refs)
+        try:
+            new_refs = determine_wants(old_refs)
+        except:
+            proto.write_pkt_line(None)
+            raise
         if new_refs is None:
             proto.write_pkt_line(None)
             return old_refs
@@ -446,7 +450,11 @@ class TraditionalGitClient(GitClient):
         proto, can_read = self._connect('upload-pack', path)
         (refs, server_capabilities) = self._read_refs(proto)
         negotiated_capabilities = list(self._fetch_capabilities)
-        wants = determine_wants(refs)
+        try:
+            wants = determine_wants(refs)
+        except:
+            proto.write_pkt_line(None)
+            raise
         if not wants:
             proto.write_pkt_line(None)
             return refs
@@ -456,6 +464,24 @@ class TraditionalGitClient(GitClient):
             graph_walker, pack_data, progress)
         return refs
 
+    def archive(self, path, committish, write_data, progress=None):
+        proto, can_read = self._connect('upload-archive', path)
+        proto.write_pkt_line("argument %s" % committish)
+        proto.write_pkt_line(None)
+        pkt = proto.read_pkt_line()
+        if pkt == "NACK\n":
+            return
+        elif pkt == "ACK\n":
+            pass
+        elif pkt.startswith("ERR "):
+            raise GitProtocolError(pkt[4:].rstrip("\n"))
+        else:
+            raise AssertionError("invalid response %r" % pkt)
+        ret = proto.read_pkt_line()
+        if ret is not None:
+            raise AssertionError("expected pkt tail")
+        self._read_side_band64k_data(proto, {1: write_data, 2: progress})
+
 
 class TCPGitClient(TraditionalGitClient):
     """A Git Client that works over TCP directly (i.e. git://)."""
@@ -524,6 +550,10 @@ class SubprocessGitClient(TraditionalGitClient):
 
     def __init__(self, *args, **kwargs):
         self._connection = None
+        self._stderr = None
+        self._stderr = kwargs.get('stderr')
+        if 'stderr' in kwargs:
+            del kwargs['stderr']
         GitClient.__init__(self, *args, **kwargs)
 
     def _connect(self, service, path):
@@ -531,7 +561,8 @@ class SubprocessGitClient(TraditionalGitClient):
         argv = ['git', service, path]
         p = SubprocessWrapper(
             subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
-                             stdout=subprocess.PIPE))
+                             stdout=subprocess.PIPE,
+                             stderr=self._stderr))
         return Protocol(p.read, p.write,
                         report_activity=self._report_activity), p.can_read
 

dulwich/object_store.py

@@ -476,6 +476,7 @@ class DiskObjectStore(PackBasedObjectStore):
 
         # Complete the pack.
         for ext_sha in indexer.ext_refs():
+            assert len(ext_sha) == 20
             type_num, data = self.get_raw(ext_sha)
             offset = f.tell()
             crc32 = write_pack_object(f, type_num, data, sha=new_sha)
@@ -610,9 +611,17 @@ class MemoryObjectStore(BaseObjectStore):
         super(MemoryObjectStore, self).__init__()
         self._data = {}
 
+    def _to_hexsha(self, sha):
+        if len(sha) == 40:
+            return sha
+        elif len(sha) == 20:
+            return sha_to_hex(sha)
+        else:
+            raise ValueError("Invalid sha %r" % sha)
+
     def contains_loose(self, sha):
         """Check if a particular object is present by SHA1 and is loose."""
-        return sha in self._data
+        return self._to_hexsha(sha) in self._data
 
     def contains_packed(self, sha):
         """Check if a particular object is present by SHA1 and is packed."""
@@ -633,15 +642,15 @@ class MemoryObjectStore(BaseObjectStore):
         :param name: sha for the object.
         :return: tuple with numeric type and object contents.
         """
-        obj = self[name]
+        obj = self[self._to_hexsha(name)]
         return obj.type_num, obj.as_raw_string()
 
     def __getitem__(self, name):
-        return self._data[name]
+        return self._data[self._to_hexsha(name)]
 
     def __delitem__(self, name):
         """Delete an object from this store, for testing only."""
-        del self._data[name]
+        del self._data[self._to_hexsha(name)]
 
     def add_object(self, obj):
         """Add a single object to this object store.

dulwich/repo.py

@@ -388,6 +388,40 @@ class DictRefsContainer(RefsContainer):
         self._peeled.update(peeled)
 
 
+class InfoRefsContainer(RefsContainer):
+    """Refs container that reads refs from a info/refs file."""
+
+    def __init__(self, f):
+        self._refs = {}
+        self._peeled = {}
+        for l in f.readlines():
+            sha, name = l.rstrip("\n").split("\t")
+            if name.endswith("^{}"):
+                name = name[:-3]
+                if not check_ref_format(name):
+                    raise ValueError("invalid ref name '%s'" % name)
+                self._peeled[name] = sha
+            else:
+                if not check_ref_format(name):
+                    raise ValueError("invalid ref name '%s'" % name)
+                self._refs[name] = sha
+
+    def allkeys(self):
+        return self._refs.keys()
+
+    def read_loose_ref(self, name):
+        return self._refs.get(name, None)
+
+    def get_packed_refs(self):
+        return {}
+
+    def get_peeled(self, name):
+        try:
+            return self._peeled[name]
+        except KeyError:
+            return self._refs[name]
+
+
 class DiskRefsContainer(RefsContainer):
     """Refs container that reads refs from disk."""
 

dulwich/server.py

@@ -27,6 +27,7 @@ Documentation/technical directory in the cgit distribution, and in particular:
 
 
 import collections
+import os
 import socket
 import SocketServer
 import sys
@@ -787,3 +788,40 @@ def serve_command(handler_cls, argv=sys.argv, backend=None, inf=sys.stdin,
     # FIXME: Catch exceptions and write a single-line summary to outf.
     handler.handle()
     return 0
+
+
+def generate_info_refs(repo):
+    """Generate an info refs file."""
+    refs = repo.get_refs()
+    for name in sorted(refs.iterkeys()):
+        # get_refs() includes HEAD as a special case, but we don't want to
+        # advertise it
+        if name == 'HEAD':
+            continue
+        sha = refs[name]
+        o = repo[sha]
+        if not o:
+            continue
+        yield '%s\t%s\n' % (sha, name)
+        peeled_sha = repo.get_peeled(name)
+        if peeled_sha != sha:
+            yield '%s\t%s^{}\n' % (peeled_sha, name)
+
+
+def generate_objects_info_packs(repo):
+    """Generate an index for for packs."""
+    for pack in repo.object_store.packs:
+        yield 'P pack-%s.pack\n' % pack.name()
+
+
+def update_server_info(repo):
+    """Generate server info for dumb file access.
+
+    This generates info/refs and objects/info/packs,
+    similar to "git update-server-info".
+    """
+    repo._put_named_file(os.path.join('info', 'refs'),
+        "".join(generate_info_refs(repo)))
+
+    repo._put_named_file(os.path.join('objects', 'info', 'packs'),
+        "".join(generate_objects_info_packs(repo)))

dulwich/tests/compat/test_client.py

@@ -19,6 +19,7 @@
 
 """Compatibilty tests between the Dulwich client and the cgit server."""
 
+from cStringIO import StringIO
 import BaseHTTPServer
 import SimpleHTTPServer
 import copy
@@ -27,6 +28,7 @@ import select
 import shutil
 import signal
 import subprocess
+import tarfile
 import tempfile
 import threading
 import urllib
@@ -164,6 +166,14 @@ class DulwichClientTestBase(object):
                               'refs/heads/master': 'non-fast-forward'},
                              e.ref_status)
 
+    def test_archive(self):
+        c = self._client()
+        f = StringIO()
+        c.archive(self._build_path('/server_new.export'), 'HEAD', f.write)
+        f.seek(0)
+        tf = tarfile.open(fileobj=f)
+        self.assertEquals(['baz', 'foo'], tf.getnames())
+
     def test_fetch_pack(self):
         c = self._client()
         dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
@@ -211,7 +221,7 @@ class DulwichTCPClientTest(CompatTestCase, DulwichClientTestBase):
             ['daemon', '--verbose', '--export-all',
              '--pid-file=%s' % self.pidfile, '--base-path=%s' % self.gitroot,
              '--detach', '--reuseaddr', '--enable=receive-pack',
-             '--listen=localhost', self.gitroot], cwd=self.gitroot)
+             '--enable=upload-archive', '--listen=localhost', self.gitroot], cwd=self.gitroot)
         if not check_for_daemon():
             raise SkipTest('git-daemon failed to start')
 
@@ -272,7 +282,7 @@ class DulwichSubprocessClientTest(CompatTestCase, DulwichClientTestBase):
         CompatTestCase.tearDown(self)
 
     def _client(self):
-        return client.SubprocessGitClient()
+        return client.SubprocessGitClient(stderr=subprocess.PIPE)
 
     def _build_path(self, path):
         return self.gitroot + path
@@ -446,3 +456,6 @@ class DulwichHttpClientTest(CompatTestCase, DulwichClientTestBase):
 
     def _build_path(self, path):
         return path
+
+    def test_archive(self):
+        raise SkipTest("exporting archives not supported over http")

dulwich/tests/test_client.py

@@ -67,6 +67,14 @@ class GitClientTests(TestCase):
         self.assertEquals(set(['ofs-delta', 'report-status', 'side-band-64k']),
                           set(self.client._send_capabilities))
 
+    def test_archive_ack(self):
+        self.rin.write(
+            '0009NACK\n'
+            '0000')
+        self.rin.seek(0)
+        self.client.archive('bla', 'HEAD', None, None)
+        self.assertEquals(self.rout.getvalue(), '0011argument HEAD0000')
+
     def test_fetch_pack_none(self):
         self.rin.write(
             '008855dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7 HEAD.multi_ack '

dulwich/tests/test_repository.py

@@ -36,6 +36,7 @@ from dulwich import objects
 from dulwich.repo import (
     check_ref_format,
     DictRefsContainer,
+    InfoRefsContainer,
     Repo,
     MemoryRepo,
     read_packed_refs,
@@ -891,3 +892,56 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
             self._refs.read_ref("refs/heads/packed"))
         self.assertEqual(None,
             self._refs.read_ref("nonexistant"))
+
+
+_TEST_REFS_SERIALIZED = (
+'42d06bd4b77fed026b154d16493e5deab78f02ec\trefs/heads/master\n'
+'42d06bd4b77fed026b154d16493e5deab78f02ec\trefs/heads/packed\n'
+'df6800012397fb85c56e7418dd4eb9405dee075c\trefs/tags/refs-0.1\n'
+'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8\trefs/tags/refs-0.2\n')
+
+
+class InfoRefsContainerTests(TestCase):
+
+    def test_invalid_refname(self):
+        text = _TEST_REFS_SERIALIZED + '00' * 20 + '\trefs/stash\n'
+        refs = InfoRefsContainer(StringIO(text))
+        expected_refs = dict(_TEST_REFS)
+        del expected_refs['HEAD']
+        expected_refs["refs/stash"] = "00" * 20
+        self.assertEquals(expected_refs, refs.as_dict())
+
+    def test_keys(self):
+        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        actual_keys = set(refs.keys())
+        self.assertEqual(set(refs.allkeys()), actual_keys)
+        # ignore the symref loop if it exists
+        actual_keys.discard('refs/heads/loop')
+        expected_refs = dict(_TEST_REFS)
+        del expected_refs['HEAD']
+        self.assertEqual(set(expected_refs.iterkeys()), actual_keys)
+
+        actual_keys = refs.keys('refs/heads')
+        actual_keys.discard('loop')
+        self.assertEqual(['master', 'packed'], sorted(actual_keys))
+        self.assertEqual(['refs-0.1', 'refs-0.2'],
+                         sorted(refs.keys('refs/tags')))
+
+    def test_as_dict(self):
+        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        # refs/heads/loop does not show up even if it exists
+        expected_refs = dict(_TEST_REFS)
+        del expected_refs['HEAD']
+        self.assertEqual(expected_refs, refs.as_dict())
+
+    def test_contains(self):
+        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        self.assertTrue('refs/heads/master' in refs)
+        self.assertFalse('refs/heads/bar' in refs)
+
+    def test_get_peeled(self):
+        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        # refs/heads/loop does not show up even if it exists
+        self.assertEqual(
+            _TEST_REFS['refs/heads/master'],
+            refs.get_peeled('refs/heads/master'))

dulwich/tests/test_server.py

@@ -44,6 +44,7 @@ from dulwich.server import (
     ReceivePackHandler,
     SingleAckGraphWalkerImpl,
     UploadPackHandler,
+    update_server_info,
     )
 from dulwich.tests import TestCase
 from dulwich.tests.utils import (
@@ -689,3 +690,30 @@ class ServeCommandTests(TestCase):
             outlines[0][4:].split("\x00")[0])
         self.assertEquals("0000", outlines[-1])
         self.assertEquals(0, exitcode)
+
+
+class UpdateServerInfoTests(TestCase):
+    """Tests for update_server_info."""
+
+    def setUp(self):
+        super(UpdateServerInfoTests, self).setUp()
+        self.path = tempfile.mkdtemp()
+        self.repo = Repo.init(self.path)
+
+    def test_empty(self):
+        update_server_info(self.repo)
+        self.assertEquals("",
+            open(os.path.join(self.path, ".git", "info", "refs"), 'r').read())
+        self.assertEquals("",
+            open(os.path.join(self.path, ".git", "objects", "info", "packs"), 'r').read())
+
+    def test_simple(self):
+        commit_id = self.repo.do_commit(
+            message="foo",
+            committer="Joe Example <joe@example.com>",
+            ref="refs/heads/foo")
+        update_server_info(self.repo)
+        ref_text = open(os.path.join(self.path, ".git", "info", "refs"), 'r').read()
+        self.assertEquals(ref_text, "%s\trefs/heads/foo\n" % commit_id)
+        packs_text = open(os.path.join(self.path, ".git", "objects", "info", "packs"), 'r').read()
+        self.assertEquals(packs_text, "")

dulwich/tests/test_walk.py

@@ -123,7 +123,7 @@ class WalkerTest(TestCase):
         del self.store[cs[-1].id]
         for i in xrange(1, 11):
             self.assertWalkYields(cs[:i], [cs[0].id], max_entries=i)
-        self.assertRaises(MissingCommitError, Walker, self.store, cs[0].id)
+        self.assertRaises(MissingCommitError, Walker, self.store, [cs[-1].id])
 
     def test_branch(self):
         c1, x2, x3, y4 = self.make_commits([[1], [2, 1], [3, 2], [4, 1]])

dulwich/tests/utils.py

@@ -230,6 +230,7 @@ def build_pack(f, objects_spec, store=None):
     expected = []
     for i in xrange(num_objects):
         type_num, data, sha = full_objects[i]
+        assert len(sha) == 20
         expected.append((offsets[i], type_num, data, sha, crc32s[i]))
 
     sf.write_sha()

dulwich/web.py

@@ -29,6 +29,7 @@ try:
 except ImportError:
     from dulwich._compat import parse_qs
 from dulwich import log_utils
+from dulwich.gzip import GzipConsumer
 from dulwich.protocol import (
     ReceivableProtocol,
     )
@@ -38,6 +39,8 @@ from dulwich.repo import (
 from dulwich.server import (
     DictBackend,
     DEFAULT_HANDLERS,
+    generate_info_refs,
+    generate_objects_info_packs,
     )
 
 
@@ -180,28 +183,15 @@ def get_info_refs(req, backend, mat):
         req.respond(HTTP_OK, 'text/plain')
         logger.info('Emulating dumb info/refs')
         repo = get_repo(backend, mat)
-        refs = repo.get_refs()
-        for name in sorted(refs.iterkeys()):
-            # get_refs() includes HEAD as a special case, but we don't want to
-            # advertise it
-            if name == 'HEAD':
-                continue
-            sha = refs[name]
-            o = repo[sha]
-            if not o:
-                continue
-            yield '%s\t%s\n' % (sha, name)
-            peeled_sha = repo.get_peeled(name)
-            if peeled_sha != sha:
-                yield '%s\t%s^{}\n' % (peeled_sha, name)
+        for text in generate_info_refs(repo):
+            yield text
 
 
 def get_info_packs(req, backend, mat):
     req.nocache()
     req.respond(HTTP_OK, 'text/plain')
     logger.info('Emulating dumb info/packs')
-    for pack in get_repo(backend, mat).object_store.packs:
-        yield 'P pack-%s.pack\n' % pack.name()
+    return generate_objects_info_packs(get_repo(backend, mat))
 
 
 class _LengthLimitedFile(object):