Imported Upstream version 0.9.1

.gitignore

@@ -1,12 +0,0 @@
-_trial_temp
-build
-MANIFEST
-dist
-apidocs
-*,cover
-.testrepository
-*.pyc
-*.so
-*~
-*.swp
-docs/tutorial/index.html

.testr.conf

@@ -1,4 +0,0 @@
-[DEFAULT]
-test_command=PYTHONPATH=. python -m subunit.run $IDOPTION $LISTOPT dulwich.tests.test_suite
-test_id_option=--load-list $IDFILE
-test_list_option=--list

AUTHORS

@@ -1,10 +0,0 @@
-Jelmer Vernooij <jelmer@samba.org>
-James Westby <jw+debian@jameswestby.net>
-John Carr <john.carr@unrouted.co.uk>
-Dave Borowitz <dborowitz@google.com>
-Chris Eberle <eberle1080@gmail.com>
-"milki" <milki@rescomp.berkeley.edu>
-
-Hervé Cauwelier <herve@itaapy.com> wrote the original tutorial.
-
-See the revision history for a full list of contributors.

HACKING

@@ -1,27 +0,0 @@
-All functionality should be available in pure Python. Optional C
-implementations may be written for performance reasons, but should never
-replace the Python implementation. The C implementations should follow the
-kernel/git coding style.
-
-Where possible include updates to NEWS along with your improvements.
-
-New functionality and bug fixes should be accompanied with matching unit tests.
-
-Coding style
-------------
-Where possible, please follow PEP8 with regard to coding style.
-
-Furthermore, triple-quotes should always be """, single quotes are ' unless
-using " would result in less escaping within the string.
-
-Public methods, functions and classes should all have doc strings. Please use
-epydoc style docstrings to document parameters and return values.
-You can generate the documentation by running "make doc".
-
-Running the tests
------------------
-To run the testsuite, you should be able to simply run "make check". This
-will run the tests using unittest on Python 2.7 and higher, and using
-unittest2 (which you will need to have installed) on older versions of Python.
-
- $ make check

MANIFEST.in

@@ -2,4 +2,6 @@ include NEWS
 include README
 include Makefile
 include COPYING
-include docs/protocol.txt
+recursive-include docs conf.py *.txt Makefile make.bat
+recursive-include examples *.py
+graft dulwich/tests/data

NEWS

@@ -1,3 +1,53 @@
+0.9.1	2013-09-22
+
+ BUG FIXES
+
+  * Support lookups of 40-character refs in BaseRepo.__getitem__. (Chow Loong Jin, Jelmer Vernooij)
+
+  * Fix fetching packs with side-band-64k capability disabled. (David Keijser, Jelmer Vernooij)
+
+  * Several fixes in send-pack protocol behaviour - handling of empty pack files and deletes.
+    (milki, #1063087)
+
+  * Fix capability negotiation when fetching packs over HTTP.
+    (#1072461, William Grant)
+
+  * Enforce determine_wants returning an empty list rather than None. (Fabien Boucher, Jelmer Vernooij)
+
+  * In the server, support pushes just removing refs. (Fabien Boucher, Jelmer Vernooij)
+
+ IMPROVEMENTS
+
+  * Support passing a single revision to BaseRepo.get_walker() rather than a list of revisions. 
+    (Alberto Ruiz)
+
+  * Add `Repo.get_description` method. (Jelmer Vernooij)
+
+  * Support thin packs in Pack.iterobjects() and Pack.get_raw().
+    (William Grant)
+
+  * Add `MemoryObjectStore.add_pack` and `MemoryObjectStore.add_thin_pack` methods.
+    (David Bennett)
+
+  * Add paramiko-based SSH vendor. (Aaron O'Mullan)
+
+  * Support running 'dulwich.server' and 'dulwich.web' using 'python -m'.
+    (Jelmer Vernooij)
+
+  * Add ObjectStore.close(). (Jelmer Vernooij)
+
+  * Raise appropriate NotImplementedError when encountering dumb HTTP servers.
+    (Jelmer Vernooij)
+
+ API CHANGES
+
+  * SSHVendor.connect_ssh has been renamed to SSHVendor.run_command.
+    (Jelmer Vernooij)
+
+  * ObjectStore.add_pack() now returns a 3-tuple. The last element will be an
+    abort() method that can be used to cancel the pack operation.
+    (Jelmer Vernooij)
+
 0.9.0	2013-05-31
 
  BUG FIXES

PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 1.0
+Name: dulwich
+Version: 0.9.1
+Summary: Python Git Library
+Home-page: http://samba.org/~jelmer/dulwich
+Author: Jelmer Vernooij
+Author-email: jelmer@samba.org
+License: GPLv2 or later
+Description: 
+              Simple Python implementation of the Git file formats and
+              protocols. Dulwich is the place where Mr. and Mrs. Git live
+              in one of the Monty Python sketches.
+        
+              All functionality is available in pure Python, but (optional)
+              C extensions are also available for better performance.
+              
+Keywords: git
+Platform: UNKNOWN

bin/dulwich

@@ -34,6 +34,7 @@ from dulwich.client import get_transport_and_path
 from dulwich.errors import ApplyDeltaError
 from dulwich.index import Index
 from dulwich.pack import Pack, sha_to_hex
+from dulwich.patch import write_tree_diff
 from dulwich.repo import Repo
 from dulwich.server import update_server_info
 
@@ -98,6 +99,20 @@ def cmd_log(args):
         todo.extend([p for p in commit.parents if p not in done])
 
 
+def cmd_diff(args):
+    opts, args = getopt(args, "", [])
+
+    if args == []:
+        print "Usage: dulwich diff COMMITID"
+        sys.exit(1)
+
+    r = Repo(".")
+    commit_id = args[0]
+    commit = r[commit_id]
+    parent_commit = r[commit.parents[0]]
+    write_tree_diff(sys.stdout, r.object_store, parent_commit.tree, commit.tree)
+
+
 def cmd_dump_pack(args):
     opts, args = getopt(args, "", [])
 
@@ -203,6 +218,7 @@ commands = {
     "clone": cmd_clone,
     "archive": cmd_archive,
     "update-server-info": cmd_update_server_info,
+    "diff": cmd_diff,
     }
 
 if len(sys.argv) < 2:

dulwich.cfg

@@ -1,5 +0,0 @@
-packages: dulwich
-docformat: restructuredtext
-projectname: Dulwich
-projecturl: http://samba.org/~jelmer/dulwich/
-htmloutput: apidocs

dulwich.egg-info/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 1.0
+Name: dulwich
+Version: 0.9.1
+Summary: Python Git Library
+Home-page: http://samba.org/~jelmer/dulwich
+Author: Jelmer Vernooij
+Author-email: jelmer@samba.org
+License: GPLv2 or later
+Description: 
+              Simple Python implementation of the Git file formats and
+              protocols. Dulwich is the place where Mr. and Mrs. Git live
+              in one of the Monty Python sketches.
+        
+              All functionality is available in pure Python, but (optional)
+              C extensions are also available for better performance.
+              
+Keywords: git
+Platform: UNKNOWN

dulwich.egg-info/SOURCES.txt

@@ -0,0 +1,162 @@
+COPYING
+MANIFEST.in
+Makefile
+NEWS
+README
+setup.py
+bin/dul-daemon
+bin/dul-receive-pack
+bin/dul-upload-pack
+bin/dul-web
+bin/dulwich
+docs/Makefile
+docs/conf.py
+docs/index.txt
+docs/make.bat
+docs/performance.txt
+docs/protocol.txt
+docs/tutorial/.gitignore
+docs/tutorial/Makefile
+docs/tutorial/conclusion.txt
+docs/tutorial/index.txt
+docs/tutorial/introduction.txt
+docs/tutorial/object-store.txt
+docs/tutorial/remote.txt
+docs/tutorial/repo.txt
+docs/tutorial/tag.txt
+dulwich/__init__.py
+dulwich/_compat.py
+dulwich/_diff_tree.c
+dulwich/_objects.c
+dulwich/_pack.c
+dulwich/client.py
+dulwich/config.py
+dulwich/diff_tree.py
+dulwich/errors.py
+dulwich/fastexport.py
+dulwich/file.py
+dulwich/hooks.py
+dulwich/index.py
+dulwich/log_utils.py
+dulwich/lru_cache.py
+dulwich/object_store.py
+dulwich/objects.py
+dulwich/pack.py
+dulwich/patch.py
+dulwich/protocol.py
+dulwich/repo.py
+dulwich/server.py
+dulwich/walk.py
+dulwich/web.py
+dulwich.egg-info/PKG-INFO
+dulwich.egg-info/SOURCES.txt
+dulwich.egg-info/dependency_links.txt
+dulwich.egg-info/top_level.txt
+dulwich/tests/__init__.py
+dulwich/tests/test_blackbox.py
+dulwich/tests/test_client.py
+dulwich/tests/test_config.py
+dulwich/tests/test_diff_tree.py
+dulwich/tests/test_fastexport.py
+dulwich/tests/test_file.py
+dulwich/tests/test_hooks.py
+dulwich/tests/test_index.py
+dulwich/tests/test_lru_cache.py
+dulwich/tests/test_missing_obj_finder.py
+dulwich/tests/test_object_store.py
+dulwich/tests/test_objects.py
+dulwich/tests/test_pack.py
+dulwich/tests/test_patch.py
+dulwich/tests/test_protocol.py
+dulwich/tests/test_repository.py
+dulwich/tests/test_server.py
+dulwich/tests/test_utils.py
+dulwich/tests/test_walk.py
+dulwich/tests/test_web.py
+dulwich/tests/utils.py
+dulwich/tests/compat/__init__.py
+dulwich/tests/compat/server_utils.py
+dulwich/tests/compat/test_client.py
+dulwich/tests/compat/test_pack.py
+dulwich/tests/compat/test_repository.py
+dulwich/tests/compat/test_server.py
+dulwich/tests/compat/test_utils.py
+dulwich/tests/compat/test_web.py
+dulwich/tests/compat/utils.py
+dulwich/tests/data/blobs/11/11111111111111111111111111111111111111
+dulwich/tests/data/blobs/6f/670c0fb53f9463760b7295fbb814e965fb20c8
+dulwich/tests/data/blobs/95/4a536f7819d40e6f637f849ee187dd10066349
+dulwich/tests/data/blobs/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391
+dulwich/tests/data/commits/0d/89f20333fbb1d2f3a94da77f4981373d8f4310
+dulwich/tests/data/commits/5d/ac377bdded4c9aeb8dff595f0faeebcc8498cc
+dulwich/tests/data/commits/60/dacdc733de308bb77bb76ce0fb0f9b44c9769e
+dulwich/tests/data/indexes/index
+dulwich/tests/data/packs/pack-bc63ddad95e7321ee734ea11a7a62d314e0d7481.idx
+dulwich/tests/data/packs/pack-bc63ddad95e7321ee734ea11a7a62d314e0d7481.pack
+dulwich/tests/data/repos/server_new.export
+dulwich/tests/data/repos/server_old.export
+dulwich/tests/data/repos/a.git/HEAD
+dulwich/tests/data/repos/a.git/packed-refs
+dulwich/tests/data/repos/a.git/objects/28/237f4dc30d0d462658d6b937b08a0f0b6ef55a
+dulwich/tests/data/repos/a.git/objects/2a/72d929692c41d8554c07f6301757ba18a65d91
+dulwich/tests/data/repos/a.git/objects/4e/f30bbfe26431a69c3820d3a683df54d688f2ec
+dulwich/tests/data/repos/a.git/objects/4f/2e6529203aa6d44b5af6e3292c837ceda003f9
+dulwich/tests/data/repos/a.git/objects/7d/9a07d797595ef11344549b8d08198e48c15364
+dulwich/tests/data/repos/a.git/objects/a2/96d0bb611188cabb256919f36bc30117cca005
+dulwich/tests/data/repos/a.git/objects/a9/0fa2d900a17e99b433217e988c4eb4a2e9a097
+dulwich/tests/data/repos/a.git/objects/b0/931cadc54336e78a1d980420e3268903b57a50
+dulwich/tests/data/repos/a.git/objects/ff/d47d45845a8f6576491e1edb97e3fe6a850e7f
+dulwich/tests/data/repos/a.git/refs/heads/master
+dulwich/tests/data/repos/a.git/refs/tags/mytag
+dulwich/tests/data/repos/empty.git/HEAD
+dulwich/tests/data/repos/empty.git/config
+dulwich/tests/data/repos/empty.git/objects/info/.gitignore
+dulwich/tests/data/repos/empty.git/objects/pack/.gitignore
+dulwich/tests/data/repos/empty.git/refs/heads/.gitignore
+dulwich/tests/data/repos/empty.git/refs/tags/.gitignore
+dulwich/tests/data/repos/ooo_merge.git/HEAD
+dulwich/tests/data/repos/ooo_merge.git/objects/29/69be3e8ee1c0222396a5611407e4769f14e54b
+dulwich/tests/data/repos/ooo_merge.git/objects/38/74e9c60a6d149c44c928140f250d81e6381520
+dulwich/tests/data/repos/ooo_merge.git/objects/6f/670c0fb53f9463760b7295fbb814e965fb20c8
+dulwich/tests/data/repos/ooo_merge.git/objects/70/c190eb48fa8bbb50ddc692a17b44cb781af7f6
+dulwich/tests/data/repos/ooo_merge.git/objects/76/01d7f6231db6a57f7bbb79ee52e4d462fd44d1
+dulwich/tests/data/repos/ooo_merge.git/objects/90/182552c4a85a45ec2a835cadc3451bebdfe870
+dulwich/tests/data/repos/ooo_merge.git/objects/95/4a536f7819d40e6f637f849ee187dd10066349
+dulwich/tests/data/repos/ooo_merge.git/objects/b2/a2766a2879c209ab1176e7e778b81ae422eeaa
+dulwich/tests/data/repos/ooo_merge.git/objects/f5/07291b64138b875c28e03469025b1ea20bc614
+dulwich/tests/data/repos/ooo_merge.git/objects/f9/e39b120c68182a4ba35349f832d0e4e61f485c
+dulwich/tests/data/repos/ooo_merge.git/objects/fb/5b0425c7ce46959bec94d54b9a157645e114f5
+dulwich/tests/data/repos/ooo_merge.git/refs/heads/master
+dulwich/tests/data/repos/refs.git/HEAD
+dulwich/tests/data/repos/refs.git/packed-refs
+dulwich/tests/data/repos/refs.git/objects/3b/9e5457140e738c2dcd39bf6d7acf88379b90d1
+dulwich/tests/data/repos/refs.git/objects/3e/c9c43c84ff242e3ef4a9fc5bc111fd780a76a8
+dulwich/tests/data/repos/refs.git/objects/42/d06bd4b77fed026b154d16493e5deab78f02ec
+dulwich/tests/data/repos/refs.git/objects/a1/8114c31713746a33a2e70d9914d1ef3e781425
+dulwich/tests/data/repos/refs.git/objects/cd/a609072918d7b70057b6bef9f4c2537843fcfe
+dulwich/tests/data/repos/refs.git/objects/df/6800012397fb85c56e7418dd4eb9405dee075c
+dulwich/tests/data/repos/refs.git/refs/heads/40-char-ref-aaaaaaaaaaaaaaaaaa
+dulwich/tests/data/repos/refs.git/refs/heads/loop
+dulwich/tests/data/repos/refs.git/refs/heads/master
+dulwich/tests/data/repos/refs.git/refs/tags/refs-0.2
+dulwich/tests/data/repos/simple_merge.git/HEAD
+dulwich/tests/data/repos/simple_merge.git/objects/0d/89f20333fbb1d2f3a94da77f4981373d8f4310
+dulwich/tests/data/repos/simple_merge.git/objects/1b/6318f651a534b38f9c7aedeebbd56c1e896853
+dulwich/tests/data/repos/simple_merge.git/objects/29/69be3e8ee1c0222396a5611407e4769f14e54b
+dulwich/tests/data/repos/simple_merge.git/objects/4c/ffe90e0a41ad3f5190079d7c8f036bde29cbe6
+dulwich/tests/data/repos/simple_merge.git/objects/5d/ac377bdded4c9aeb8dff595f0faeebcc8498cc
+dulwich/tests/data/repos/simple_merge.git/objects/60/dacdc733de308bb77bb76ce0fb0f9b44c9769e
+dulwich/tests/data/repos/simple_merge.git/objects/6f/670c0fb53f9463760b7295fbb814e965fb20c8
+dulwich/tests/data/repos/simple_merge.git/objects/70/c190eb48fa8bbb50ddc692a17b44cb781af7f6
+dulwich/tests/data/repos/simple_merge.git/objects/90/182552c4a85a45ec2a835cadc3451bebdfe870
+dulwich/tests/data/repos/simple_merge.git/objects/95/4a536f7819d40e6f637f849ee187dd10066349
+dulwich/tests/data/repos/simple_merge.git/objects/ab/64bbdcc51b170d21588e5c5d391ee5c0c96dfd
+dulwich/tests/data/repos/simple_merge.git/objects/d4/bdad6549dfedf25d3b89d21f506aff575b28a7
+dulwich/tests/data/repos/simple_merge.git/objects/d8/0c186a03f423a81b39df39dc87fd269736ca86
+dulwich/tests/data/repos/simple_merge.git/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391
+dulwich/tests/data/repos/simple_merge.git/refs/heads/master
+dulwich/tests/data/repos/submodule/dotgit
+dulwich/tests/data/tags/71/033db03a03c6a36721efcf1968dd8f8e0cf023
+dulwich/tests/data/trees/70/c190eb48fa8bbb50ddc692a17b44cb781af7f6
+examples/clone.py
+examples/diff.py

dulwich.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dulwich.egg-info/top_level.txt

@@ -0,0 +1 @@
+dulwich

dulwich/__init__.py

@@ -21,4 +21,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 9, 0)
+__version__ = (0, 9, 1)

dulwich/client.py

@@ -137,6 +137,31 @@ class ReportStatusParser(object):
                 self._ref_status_ok = False
 
 
+def read_pkt_refs(proto):
+    server_capabilities = None
+    refs = {}
+    # Receive refs from server
+    for pkt in proto.read_pkt_seq():
+        (sha, ref) = pkt.rstrip('\n').split(None, 1)
+        if sha == 'ERR':
+            raise GitProtocolError(ref)
+        if server_capabilities is None:
+            (ref, server_capabilities) = extract_capabilities(ref)
+        refs[ref] = sha
+
+    if len(refs) == 0:
+        return None, set([])
+    return refs, set(server_capabilities)
+
+
+def read_info_refs(f):
+    ret = {}
+    for l in f.readlines():
+        (sha, name) = l.rstrip("\r\n").split("\t", 1)
+        ret[name] = sha
+    return ret
+
+
 # TODO(durin42): this doesn't correctly degrade if the server doesn't
 # support some capabilities. This should work properly with servers
 # that don't support multi_ack.
@@ -153,27 +178,12 @@ class GitClient(object):
             activity.
         """
         self._report_activity = report_activity
+        self._report_status_parser = None
         self._fetch_capabilities = set(FETCH_CAPABILITIES)
         self._send_capabilities = set(SEND_CAPABILITIES)
         if not thin_packs:
             self._fetch_capabilities.remove('thin-pack')
 
-    def _read_refs(self, proto):
-        server_capabilities = None
-        refs = {}
-        # Receive refs from server
-        for pkt in proto.read_pkt_seq():
-            (sha, ref) = pkt.rstrip('\n').split(' ', 1)
-            if sha == 'ERR':
-                raise GitProtocolError(ref)
-            if server_capabilities is None:
-                (ref, server_capabilities) = extract_capabilities(ref)
-            refs[ref] = sha
-
-        if len(refs) == 0:
-            return None, set([])
-        return refs, set(server_capabilities)
-
     def send_pack(self, path, determine_wants, generate_pack_contents,
                   progress=None):
         """Upload a pack to a remote repository.
@@ -201,10 +211,15 @@ class GitClient(object):
         """
         if determine_wants is None:
             determine_wants = target.object_store.determine_wants_all
-        f, commit = target.object_store.add_pack()
-        result = self.fetch_pack(path, determine_wants,
-                target.get_graph_walker(), f.write, progress)
-        commit()
+        f, commit, abort = target.object_store.add_pack()
+        try:
+            result = self.fetch_pack(path, determine_wants,
+                    target.get_graph_walker(), f.write, progress)
+        except:
+            abort()
+            raise
+        else:
+            commit()
         return result
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
@@ -288,9 +303,11 @@ class GitClient(object):
         want = []
         have = [x for x in old_refs.values() if not x == ZERO_SHA]
         sent_capabilities = False
+
         for refname in set(new_refs.keys() + old_refs.keys()):
             old_sha1 = old_refs.get(refname, ZERO_SHA)
             new_sha1 = new_refs.get(refname, ZERO_SHA)
+
             if old_sha1 != new_sha1:
                 if sent_capabilities:
                     proto.write_pkt_line('%s %s %s' % (old_sha1, new_sha1,
@@ -312,24 +329,20 @@ class GitClient(object):
         :param capabilities: List of negotiated capabilities
         :param progress: Optional progress reporting function
         """
-        if 'report-status' in capabilities:
-            report_status_parser = ReportStatusParser()
-        else:
-            report_status_parser = None
         if "side-band-64k" in capabilities:
             if progress is None:
                 progress = lambda x: None
             channel_callbacks = { 2: progress }
             if 'report-status' in capabilities:
                 channel_callbacks[1] = PktLineParser(
-                    report_status_parser.handle_packet).parse
+                    self._report_status_parser.handle_packet).parse
             self._read_side_band64k_data(proto, channel_callbacks)
         else:
             if 'report-status' in capabilities:
                 for pkt in proto.read_pkt_seq():
-                    report_status_parser.handle_packet(pkt)
-        if report_status_parser is not None:
-            report_status_parser.check()
+                    self._report_status_parser.handle_packet(pkt)
+        if self._report_status_parser is not None:
+            self._report_status_parser.check()
         # wait for EOF before returning
         data = proto.read()
         if data:
@@ -402,7 +415,7 @@ class GitClient(object):
                 raise Exception('Unexpected response %r' % data)
         else:
             while True:
-                data = self.read(rbufsize)
+                data = proto.read(rbufsize)
                 if data == "":
                     break
                 pack_data(data)
@@ -439,16 +452,48 @@ class TraditionalGitClient(GitClient):
                                  and rejects ref updates
         """
         proto, unused_can_read = self._connect('receive-pack', path)
-        old_refs, server_capabilities = self._read_refs(proto)
+        old_refs, server_capabilities = read_pkt_refs(proto)
         negotiated_capabilities = self._send_capabilities & server_capabilities
+
+        if 'report-status' in negotiated_capabilities:
+            self._report_status_parser = ReportStatusParser()
+        report_status_parser = self._report_status_parser
+
         try:
-            new_refs = determine_wants(dict(old_refs))
+            new_refs = orig_new_refs = determine_wants(dict(old_refs))
         except:
             proto.write_pkt_line(None)
             raise
+
+        if not 'delete-refs' in server_capabilities:
+            # Server does not support deletions. Fail later.
+            def remove_del(pair):
+                if pair[1] == ZERO_SHA:
+                    if 'report-status' in negotiated_capabilities:
+                        report_status_parser._ref_statuses.append(
+                            'ng %s remote does not support deleting refs'
+                            % pair[1])
+                        report_status_parser._ref_status_ok = False
+                    return False
+                else:
+                    return True
+
+            new_refs = dict(
+                filter(
+                    remove_del,
+                    [(ref, sha) for ref, sha in new_refs.iteritems()]))
+
         if new_refs is None:
             proto.write_pkt_line(None)
             return old_refs
+
+        if len(new_refs) == 0 and len(orig_new_refs):
+            # NOOP - Original new refs filtered out by policy
+            proto.write_pkt_line(None)
+            if self._report_status_parser is not None:
+                self._report_status_parser.check()
+            return old_refs
+
         (have, want) = self._handle_receive_pack_head(proto,
             negotiated_capabilities, old_refs, new_refs)
         if not want and old_refs == new_refs:
@@ -456,6 +501,15 @@ class TraditionalGitClient(GitClient):
         objects = generate_pack_contents(have, want)
         if len(objects) > 0:
             entries, sha = write_pack_objects(proto.write_file(), objects)
+        elif len(set(new_refs.values()) - set([ZERO_SHA])) > 0:
+            # Check for valid create/update refs
+            filtered_new_refs = \
+                dict([(ref, sha) for ref, sha in new_refs.iteritems()
+                     if sha != ZERO_SHA])
+            if len(set(filtered_new_refs.iteritems()) -
+                    set(old_refs.iteritems())) > 0:
+                entries, sha = write_pack_objects(proto.write_file(), objects)
+
         self._handle_receive_pack_tail(proto, negotiated_capabilities,
             progress)
         return new_refs
@@ -470,7 +524,7 @@ class TraditionalGitClient(GitClient):
         :param progress: Callback for progress reports (strings)
         """
         proto, can_read = self._connect('upload-pack', path)
-        refs, server_capabilities = self._read_refs(proto)
+        refs, server_capabilities = read_pkt_refs(proto)
         negotiated_capabilities = self._fetch_capabilities & server_capabilities
 
         if refs is None:
@@ -597,8 +651,33 @@ class SubprocessGitClient(TraditionalGitClient):
 
 
 class SSHVendor(object):
+    """A client side SSH implementation."""
 
     def connect_ssh(self, host, command, username=None, port=None):
+        import warnings
+        warnings.warn(
+            "SSHVendor.connect_ssh has been renamed to SSHVendor.run_command",
+            DeprecationWarning)
+        return self.run_command(host, command, username=username, port=port)
+
+    def run_command(self, host, command, username=None, port=None):
+        """Connect to an SSH server.
+
+        Run a command remotely and return a file-like object for interaction
+        with the remote command.
+
+        :param host: Host name
+        :param command: Command to run
+        :param username: Optional ame of user to log in as
+        :param port: Optional SSH port to use
+        """
+        raise NotImplementedError(self.run_command)
+
+
+class SubprocessSSHVendor(SSHVendor):
+    """SSH vendor that shells out to the local 'ssh' command."""
+
+    def run_command(self, host, command, username=None, port=None):
         import subprocess
         #FIXME: This has no way to deal with passwords..
         args = ['ssh', '-x']
@@ -612,8 +691,112 @@ class SSHVendor(object):
                                 stdout=subprocess.PIPE)
         return SubprocessWrapper(proc)
 
+
+try:
+    import paramiko
+except ImportError:
+    pass
+else:
+    import threading
+
+    class ParamikoWrapper(object):
+        STDERR_READ_N = 2048  # 2k
+
+        def __init__(self, client, channel, progress_stderr=None):
+            self.client = client
+            self.channel = channel
+            self.progress_stderr = progress_stderr
+            self.should_monitor = bool(progress_stderr) or True
+            self.monitor_thread = None
+            self.stderr = ''
+
+            # Channel must block
+            self.channel.setblocking(True)
+
+            # Start
+            if self.should_monitor:
+                self.monitor_thread = threading.Thread(target=self.monitor_stderr)
+                self.monitor_thread.start()
+
+        def monitor_stderr(self):
+            while self.should_monitor:
+                # Block and read
+                data = self.read_stderr(self.STDERR_READ_N)
+
+                # Socket closed
+                if not data:
+                    self.should_monitor = False
+                    break
+
+                # Emit data
+                if self.progress_stderr:
+                    self.progress_stderr(data)
+
+                # Append to buffer
+                self.stderr += data
+
+        def stop_monitoring(self):
+            # Stop StdErr thread
+            if self.should_monitor:
+                self.should_monitor = False
+                self.monitor_thread.join()
+
+                # Get left over data
+                data = self.channel.in_stderr_buffer.empty()
+                self.stderr += data
+
+        def can_read(self):
+            return self.channel.recv_ready()
+
+        def write(self, data):
+            return self.channel.sendall(data)
+
+        def read_stderr(self, n):
+            return self.channel.recv_stderr(n)
+
+        def read(self, n=None):
+            data = self.channel.recv(n)
+            data_len = len(data)
+
+            # Closed socket
+            if not data:
+                return
+
+            # Read more if needed
+            if n and data_len < n:
+                diff_len = n - data_len
+                return data + self.read(diff_len)
+            return data
+
+        def close(self):
+            self.channel.close()
+            self.stop_monitoring()
+
+        def __del__(self):
+            self.close()
+
+    class ParamikoSSHVendor(object):
+
+        def run_command(self, host, command, username=None, port=None,
+                progress_stderr=None, **kwargs):
+            client = paramiko.SSHClient()
+
+            policy = paramiko.client.MissingHostKeyPolicy()
+            client.set_missing_host_key_policy(policy)
+            client.connect(host, username=username, port=port, **kwargs)
+
+            # Open SSH session
+            channel = client.get_transport().open_session()
+
+            # Run commands
+            apply(channel.exec_command, command)
+
+            return ParamikoWrapper(client, channel,
+                    progress_stderr=progress_stderr)
+
+
 # Can be overridden by users
-get_ssh_vendor = SSHVendor
+get_ssh_vendor = SubprocessSSHVendor
 
 
 class SSHGitClient(TraditionalGitClient):
@@ -631,7 +814,7 @@ class SSHGitClient(TraditionalGitClient):
     def _connect(self, cmd, path):
         if path.startswith("/~"):
             path = path[1:]
-        con = get_ssh_vendor().connect_ssh(
+        con = get_ssh_vendor().run_command(
             self.host, ["%s '%s'" % (self._get_cmd_path(cmd), path)],
             port=self.port, username=self.username)
         return (Protocol(con.read, con.write, report_activity=self._report_activity),
@@ -678,14 +861,16 @@ class HttpGitClient(GitClient):
             headers["Content-Type"] = "application/x-%s-request" % service
         resp = self._http_request(url, headers)
         self.dumb = (not resp.info().gettype().startswith("application/x-git-"))
-        proto = Protocol(resp.read, None)
         if not self.dumb:
+            proto = Protocol(resp.read, None)
             # The first line should mention the service
             pkts = list(proto.read_pkt_seq())
             if pkts != [('# service=%s\n' % service)]:
                 raise GitProtocolError(
                     "unexpected first line %r from smart server" % pkts)
-        return self._read_refs(proto)
+            return read_pkt_refs(proto)
+        else:
+            return read_info_refs(resp), set()
 
     def _smart_request(self, service, url, data):
         assert url[-1] == "/"
@@ -714,6 +899,10 @@ class HttpGitClient(GitClient):
         old_refs, server_capabilities = self._discover_references(
             "git-receive-pack", url)
         negotiated_capabilities = self._send_capabilities & server_capabilities
+
+        if 'report-status' in negotiated_capabilities:
+            self._report_status_parser = ReportStatusParser()
+
         new_refs = determine_wants(dict(old_refs))
         if new_refs is None:
             return old_refs
@@ -748,7 +937,7 @@ class HttpGitClient(GitClient):
         url = self._get_url(path)
         refs, server_capabilities = self._discover_references(
             "git-upload-pack", url)
-        negotiated_capabilities = server_capabilities
+        negotiated_capabilities = self._fetch_capabilities & server_capabilities
         wants = determine_wants(refs)
         if wants is not None:
             wants = [cid for cid in wants if cid != ZERO_SHA]

dulwich/object_store.py

@@ -21,6 +21,7 @@
 """Git object store interfaces and implementation."""
 
 
+from cStringIO import StringIO
 import errno
 import itertools
 import os
@@ -50,6 +51,7 @@ from dulwich.objects import (
 from dulwich.pack import (
     Pack,
     PackData,
+    PackInflater,
     iter_sha1,
     write_pack_header,
     write_pack_index_v2,
@@ -245,6 +247,10 @@ class BaseObjectStore(object):
                 queue.extend(cmt.parents)
         return (commits, bases)
 
+    def close(self):
+        """Close any files opened by this object store."""
+        # Default implementation is a NO-OP
+
 
 class PackBasedObjectStore(BaseObjectStore):
 
@@ -291,6 +297,13 @@ class PackBasedObjectStore(BaseObjectStore):
         if self._pack_cache is not None:
             self._pack_cache.append(pack)
 
+    def close(self):
+        pack_cache = self._pack_cache
+        self._pack_cache = None
+        while pack_cache:
+            pack = pack_cache.pop()
+            pack.close()
+
     @property
     def packs(self):
         """List with pack objects."""
@@ -379,9 +392,14 @@ class PackBasedObjectStore(BaseObjectStore):
         if len(objects) == 0:
             # Don't bother writing an empty pack file
             return
-        f, commit = self.add_pack()
-        write_pack_objects(f, objects)
-        return commit()
+        f, commit, abort = self.add_pack()
+        try:
+            write_pack_objects(f, objects)
+        except:
+            abort()
+            raise
+        else:
+            return commit()
 
 
 class DiskObjectStore(PackBasedObjectStore):
@@ -616,8 +634,9 @@ class DiskObjectStore(PackBasedObjectStore):
     def add_pack(self):
         """Add a new pack to this object store.
 
-        :return: Fileobject to write to and a commit function to
-            call when the pack is finished.
+        :return: Fileobject to write to, a commit function to
+            call when the pack is finished and an abort
+            function.
         """
         fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack")
         f = os.fdopen(fd, 'wb')
@@ -629,7 +648,10 @@ class DiskObjectStore(PackBasedObjectStore):
             else:
                 os.remove(path)
                 return None
-        return f, commit
+        def abort():
+            f.close()
+            os.remove(path)
+        return f, commit, abort
 
     def add_object(self, obj):
         """Add a single object to this object store.
@@ -725,6 +747,72 @@ class MemoryObjectStore(BaseObjectStore):
         for obj, path in objects:
             self._data[obj.id] = obj
 
+    def add_pack(self):
+        """Add a new pack to this object store.
+
+        Because this object store doesn't support packs, we extract and add the
+        individual objects.
+
+        :return: Fileobject to write to and a commit function to
+            call when the pack is finished.
+        """
+        f = StringIO()
+        def commit():
+            p = PackData.from_file(StringIO(f.getvalue()), f.tell())
+            f.close()
+            for obj in PackInflater.for_pack_data(p):
+                self._data[obj.id] = obj
+        def abort():
+            pass
+        return f, commit, abort
+
+    def _complete_thin_pack(self, f, indexer):
+        """Complete a thin pack by adding external references.
+
+        :param f: Open file object for the pack.
+        :param indexer: A PackIndexer for indexing the pack.
+        """
+        entries = list(indexer)
+
+        # Update the header with the new number of objects.
+        f.seek(0)
+        write_pack_header(f, len(entries) + len(indexer.ext_refs()))
+
+        # Rescan the rest of the pack, computing the SHA with the new header.
+        new_sha = compute_file_sha(f, end_ofs=-20)
+
+        # Complete the pack.
+        for ext_sha in indexer.ext_refs():
+            assert len(ext_sha) == 20
+            type_num, data = self.get_raw(ext_sha)
+            write_pack_object(f, type_num, data, sha=new_sha)
+        pack_sha = new_sha.digest()
+        f.write(pack_sha)
+
+    def add_thin_pack(self, read_all, read_some):
+        """Add a new thin pack to this object store.
+
+        Thin packs are packs that contain deltas with parents that exist outside
+        the pack. Because this object store doesn't support packs, we extract
+        and add the individual objects.
+
+        :param read_all: Read function that blocks until the number of requested
+            bytes are read.
+        :param read_some: Read function that returns at least one byte, but may
+            not return the number of bytes requested.
+        """
+        f, commit, abort = self.add_pack()
+        try:
+            indexer = PackIndexer(f, resolve_ext_ref=self.get_raw)
+            copier = PackStreamCopier(read_all, read_some, f, delta_iter=indexer)
+            copier.verify()
+            self._complete_thin_pack(f, indexer)
+        except:
+            abort()
+            raise
+        else:
+            commit()
+
 
 class ObjectImporter(object):
     """Interface for importing objects."""

dulwich/objects.py

@@ -88,7 +88,12 @@ def sha_to_hex(sha):
 def hex_to_sha(hex):
     """Takes a hex sha and returns a binary sha"""
     assert len(hex) == 40, "Incorrent length of hexsha: %s" % hex
-    return binascii.unhexlify(hex)
+    try:
+        return binascii.unhexlify(hex)
+    except TypeError, exc:
+        if not isinstance(hex, str):
+            raise
+        raise ValueError(exc.message)
 
 
 def hex_to_filename(path, hex):
@@ -149,7 +154,7 @@ def check_hexsha(hex, error_msg):
     """
     try:
         hex_to_sha(hex)
-    except (TypeError, AssertionError):
+    except (TypeError, AssertionError, ValueError):
         raise ObjectFormatException("%s %s" % (error_msg, hex))
 
 

dulwich/pack.py

@@ -1023,10 +1023,16 @@ class PackData(object):
         # TODO: cache these results
         if self.pack is None:
             raise KeyError(sha)
-        offset = self.pack.index.object_index(sha)
-        if not offset:
+        try:
+            offset = self.pack.index.object_index(sha)
+        except KeyError:
+            offset = None
+        if offset:
+            type, obj = self.get_object_at(offset)
+        elif self.pack is not None and self.pack.resolve_ext_ref:
+            type, obj = self.pack.resolve_ext_ref(sha)
+        else:
             raise KeyError(sha)
-        type, obj = self.get_object_at(offset)
         return offset, type, obj
 
     def resolve_object(self, offset, type, obj, get_ref=None):
@@ -1094,7 +1100,11 @@ class PackData(object):
         :return: iterator of tuples with (sha, offset, crc32)
         """
         num_objects = self._num_objects
-        for i, result in enumerate(PackIndexer.for_pack_data(self)):
+        resolve_ext_ref = (
+            self.pack.resolve_ext_ref if self.pack is not None else None)
+        indexer = PackIndexer.for_pack_data(
+            self, resolve_ext_ref=resolve_ext_ref)
+        for i, result in enumerate(indexer):
             if progress is not None:
                 progress(i, num_objects)
             yield result
@@ -1739,7 +1749,7 @@ def write_pack_index_v2(f, entries, pack_checksum):
 class Pack(object):
     """A Git pack object."""
 
-    def __init__(self, basename):
+    def __init__(self, basename, resolve_ext_ref=None):
         self._basename = basename
         self._data = None
         self._idx = None
@@ -1747,6 +1757,7 @@ class Pack(object):
         self._data_path = self._basename + '.pack'
         self._data_load = lambda: PackData(self._data_path)
         self._idx_load = lambda: load_pack_index(self._idx_path)
+        self.resolve_ext_ref = resolve_ext_ref
 
     @classmethod
     def from_lazy_objects(self, data_fn, idx_fn):
@@ -1851,7 +1862,8 @@ class Pack(object):
 
     def iterobjects(self):
         """Iterate over the objects in this pack."""
-        return iter(PackInflater.for_pack_data(self.data))
+        return iter(PackInflater.for_pack_data(
+            self.data, resolve_ext_ref=self.resolve_ext_ref))
 
     def pack_tuples(self):
         """Provide an iterable for use with write_pack_objects.

dulwich/repo.py

@@ -86,14 +86,6 @@ BASE_DIRECTORIES = [
     ]
 
 
-def read_info_refs(f):
-    ret = {}
-    for l in f.readlines():
-        (sha, name) = l.rstrip("\r\n").split("\t", 1)
-        ret[name] = sha
-    return ret
-
-
 def check_ref_format(refname):
     """Check if a refname is correctly formatted.
 
@@ -896,10 +888,12 @@ class BaseRepo(object):
         :return: iterator over objects, with __len__ implemented
         """
         wants = determine_wants(self.get_refs())
-        if wants is None:
+        if type(wants) is not list:
+            raise TypeError("determine_wants() did not return a list")
+        if wants == []:
             # TODO(dborowitz): find a way to short-circuit that doesn't change
             # this interface.
-            return None
+            return []
         haves = self.object_store.find_common_revisions(graph_walker)
         return self.object_store.iter_shas(
           self.object_store.find_missing_objects(haves, wants, progress,
@@ -979,6 +973,14 @@ class BaseRepo(object):
         """
         raise NotImplementedError(self.get_config)
 
+    def get_description(self):
+        """Retrieve the description for this repository.
+
+        :return: String with the description of the repository
+            as set by the user.
+        """
+        raise NotImplementedError(self.get_description)
+
     def get_config_stack(self):
         """Return a config stack for this repository.
 
@@ -1081,6 +1083,8 @@ class BaseRepo(object):
         from dulwich.walk import Walker
         if include is None:
             include = [self.head()]
+        if isinstance(include, str):
+            include = [include]
         return Walker(self.object_store, include, *args, **kwargs)
 
     def revision_history(self, head):
@@ -1107,7 +1111,7 @@ class BaseRepo(object):
         if len(name) in (20, 40):
             try:
                 return self.object_store[name]
-            except KeyError:
+            except (KeyError, ValueError):
                 pass
         try:
             return self.object_store[self.refs[name]]
@@ -1449,6 +1453,23 @@ class Repo(BaseRepo):
             ret.path = path
             return ret
 
+    def get_description(self):
+        """Retrieve the description of this repository.
+
+        :return: A string describing the repository or None.
+        """
+        path = os.path.join(self._controldir, 'description')
+        try:
+            f = GitFile(path, 'rb')
+            try:
+                return f.read()
+            finally:
+                f.close()
+        except (IOError, OSError), e:
+            if e.errno != errno.ENOENT:
+                raise
+            return None
+
     def __repr__(self):
         return "<Repo at %r>" % self.path
 
@@ -1541,6 +1562,13 @@ class MemoryRepo(BaseRepo):
         from dulwich.config import ConfigFile
         return ConfigFile()
 
+    def get_description(self):
+        """Retrieve the repository description.
+
+        This defaults to None, for no description.
+        """
+        return None
+
     @classmethod
     def init_bare(cls, objects, refs):
         """Create a new bare repository in memory.

dulwich/server.py

@@ -359,7 +359,7 @@ class ProtocolGraphWalker(object):
         if not heads:
             # The repo is empty, so short-circuit the whole process.
             self.proto.write_pkt_line(None)
-            return None
+            return []
         values = set(heads.itervalues())
         if self.advertise_refs or not self.http_req:
             for i, (ref, sha) in enumerate(sorted(heads.iteritems())):
@@ -402,7 +402,7 @@ class ProtocolGraphWalker(object):
             # The client may close the socket at this point, expecting a
             # flush-pkt from the server. We might be ready to send a packfile at
             # this point, so we need to explicitly short-circuit in this case.
-            return None
+            return []
 
         return want_revs
 
@@ -617,15 +617,26 @@ class ReceivePackHandler(Handler):
                           AssertionError, socket.error, zlib.error,
                           ObjectFormatException)
         status = []
-        # TODO: more informative error messages than just the exception string
-        try:
-            recv = getattr(self.proto, "recv", None)
-            p = self.repo.object_store.add_thin_pack(self.proto.read, recv)
+        will_send_pack = False
+
+        for command in refs:
+            if command[1] != ZERO_SHA:
+                will_send_pack = True
+
+        if will_send_pack:
+            # TODO: more informative error messages than just the exception string
+            try:
+                recv = getattr(self.proto, "recv", None)
+                p = self.repo.object_store.add_thin_pack(self.proto.read, recv)
+                status.append(('unpack', 'ok'))
+            except all_exceptions, e:
+                status.append(('unpack', str(e).replace('\n', '')))
+                # The pack may still have been moved in, but it may contain broken
+                # objects. We trust a later GC to clean it up.
+        else:
+            # The git protocol want to find a status entry related to unpack process
+            # even if no pack data has been sent.
             status.append(('unpack', 'ok'))
-        except all_exceptions, e:
-            status.append(('unpack', str(e).replace('\n', '')))
-            # The pack may still have been moved in, but it may contain broken
-            # objects. We trust a later GC to clean it up.
 
         for oldsha, sha, ref in refs:
             ref_status = 'ok'
@@ -769,13 +780,22 @@ class TCPGitServer(SocketServer.TCPServer):
 
 def main(argv=sys.argv):
     """Entry point for starting a TCP git server."""
-    if len(argv) > 1:
-        gitdir = argv[1]
-    else:
-        gitdir = '.'
+    import optparse
+    parser = optparse.OptionParser()
+    parser.add_option("-b", "--backend", dest="backend",
+                      help="Select backend to use.",
+                      choices=["file"], default="file")
+    options, args = parser.parse_args(argv)
 
     log_utils.default_logging_config()
-    backend = DictBackend({'/': Repo(gitdir)})
+    if options.backend == "file":
+        if len(argv) > 1:
+            gitdir = args[1]
+        else:
+            gitdir = '.'
+        backend = DictBackend({'/': Repo(gitdir)})
+    else:
+        raise Exception("No such backend %s." % backend)
     server = TCPGitServer(backend, 'localhost')
     server.serve_forever()
 
@@ -840,3 +860,7 @@ def update_server_info(repo):
 
     repo._put_named_file(os.path.join('objects', 'info', 'packs'),
         "".join(generate_objects_info_packs(repo)))
+
+
+if __name__ == '__main__':
+    main()

dulwich/stdint.h

@@ -1,19 +0,0 @@
-/**
- * Replacement of gcc' stdint.h for MSVC
- */
-
-#ifndef STDINT_H
-#define STDINT_H
-
-typedef signed char       int8_t;
-typedef signed short      int16_t;
-typedef signed int        int32_t;
-typedef signed long long  int64_t;
-
-typedef unsigned char       uint8_t;
-typedef unsigned short      uint16_t;
-typedef unsigned int        uint32_t;
-typedef unsigned long long  uint64_t;
-
-
-#endif

dulwich/tests/compat/server_utils.py

@@ -69,6 +69,28 @@ class ServerTests(object):
                         cwd=self._new_repo.path)
         self.assertReposEqual(self._old_repo, self._new_repo)
 
+    def test_push_to_dulwich_no_op(self):
+        self._old_repo = import_repo('server_old.export')
+        self._new_repo = import_repo('server_old.export')
+        self.assertReposEqual(self._old_repo, self._new_repo)
+        port = self._start_server(self._old_repo)
+
+        run_git_or_fail(['push', self.url(port)] + self.branch_args(),
+                        cwd=self._new_repo.path)
+        self.assertReposEqual(self._old_repo, self._new_repo)
+
+    def test_push_to_dulwich_remove_branch(self):
+        self._old_repo = import_repo('server_old.export')
+        self._new_repo = import_repo('server_old.export')
+        self.assertReposEqual(self._old_repo, self._new_repo)
+        port = self._start_server(self._old_repo)
+
+        run_git_or_fail(['push', self.url(port), ":master"],
+                        cwd=self._new_repo.path)
+
+        self.assertEquals(
+            self._old_repo.get_refs().keys(), ["refs/heads/branch"])
+
     def test_fetch_from_dulwich(self):
         self.import_repos()
         self.assertReposNotEqual(self._old_repo, self._new_repo)

dulwich/tests/compat/test_client.py

@@ -32,7 +32,6 @@ import tarfile
 import tempfile
 import threading
 import urllib
-from socket import gethostname
 
 from dulwich import (
     client,
@@ -193,6 +192,14 @@ class DulwichClientTestBase(object):
         map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
         self.assertDestEqualsSrc()
 
+    def test_fetch_pack_no_side_band_64k(self):
+        c = self._client()
+        c._fetch_capabilities.remove('side-band-64k')
+        dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
+        refs = c.fetch(self._build_path('/server_new.export'), dest)
+        map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
+        self.assertDestEqualsSrc()
+
     def test_fetch_pack_zero_sha(self):
         # zero sha1s are already present on the client, and should
         # be ignored
@@ -254,7 +261,7 @@ class DulwichTCPClientTest(CompatTestCase, DulwichClientTestBase):
 
 class TestSSHVendor(object):
     @staticmethod
-    def connect_ssh(host, command, username=None, port=None):
+    def run_command(host, command, username=None, port=None):
         cmd, path = command[0].replace("'", '').split(' ')
         cmd = cmd.split('-', 1)
         p = subprocess.Popen(cmd + [path], env=get_safe_env(), stdin=subprocess.PIPE,

dulwich/tests/compat/test_web.py

@@ -136,3 +136,7 @@ class DumbWebTestCase(WebTests, CompatTestCase):
     def test_push_to_dulwich(self):
         # Note: remove this if dumb pushing is supported
         raise SkipTest('Dumb web pushing not supported.')
+
+    def test_push_to_dulwich_remove_branch(self):
+        # Note: remove this if dumb pushing is supported
+        raise SkipTest('Dumb web pushing not supported.')

dulwich/tests/compat/utils.py

@@ -134,7 +134,8 @@ def run_git(args, git_path=_DEFAULT_GIT, input=None, capture_stdout=False,
 
 def run_git_or_fail(args, git_path=_DEFAULT_GIT, input=None, **popen_kwargs):
     """Run a git command, capture stdout/stderr, and fail if git fails."""
-    popen_kwargs['stderr'] = subprocess.STDOUT
+    if 'stderr' not in popen_kwargs:
+        popen_kwargs['stderr'] = subprocess.STDOUT
     returncode, stdout = run_git(args, git_path=git_path, input=input,
                                  capture_stdout=True, **popen_kwargs)
     if returncode != 0:

dulwich/tests/data/repos/refs.git/refs/heads/40-char-ref-aaaaaaaaaaaaaaaaaa

@@ -0,0 +1 @@
+42d06bd4b77fed026b154d16493e5deab78f02ec

dulwich/tests/test_client.py

@@ -39,6 +39,13 @@ from dulwich.protocol import (
     TCP_GIT_PORT,
     Protocol,
     )
+from dulwich.pack import (
+    write_pack_objects,
+    )
+from dulwich.objects import (
+    Commit,
+    Tree
+    )
 
 
 class DummyClient(TraditionalGitClient):
@@ -207,7 +214,8 @@ class GitClientTests(TestCase):
     def test_send_pack_no_sideband64k_with_update_ref_error(self):
         # No side-bank-64k reported by server shouldn't try to parse
         # side band data
-        pkts = ['55dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7 capabilities^{}\x00 report-status ofs-delta\n',
+        pkts = ['55dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7 capabilities^{}'
+                '\x00 report-status delete-refs ofs-delta\n',
                 '',
                 "unpack ok",
                 "ng refs/foo/bar pre-receive hook declined",
@@ -218,8 +226,156 @@ class GitClientTests(TestCase):
             else:
                 self.rin.write("%04x%s" % (len(pkt)+4, pkt))
         self.rin.seek(0)
+
+        tree = Tree()
+        commit = Commit()
+        commit.tree = tree
+        commit.parents = []
+        commit.author = commit.committer = 'test user'
+        commit.commit_time = commit.author_time = 1174773719
+        commit.commit_timezone = commit.author_timezone = 0
+        commit.encoding = 'UTF-8'
+        commit.message = 'test message'
+
+        def determine_wants(refs):
+            return {'refs/foo/bar': commit.id, }
+
+        def generate_pack_contents(have, want):
+            return [(commit, None), (tree, ''), ]
+
         self.assertRaises(UpdateRefsError,
-            self.client.send_pack, "blah", lambda x: {}, lambda h,w: [])
+                          self.client.send_pack, "blah",
+                          determine_wants, generate_pack_contents)
+
+    def test_send_pack_none(self):
+        self.rin.write(
+            '0078310ca9477129b8586fa2afc779c1f57cf64bba6c '
+            'refs/heads/master\x00 report-status delete-refs '
+            'side-band-64k quiet ofs-delta\n'
+            '0000')
+        self.rin.seek(0)
+
+        def determine_wants(refs):
+            return {
+                'refs/heads/master': '310ca9477129b8586fa2afc779c1f57cf64bba6c'
+            }
+
+        def generate_pack_contents(have, want):
+            return {}
+
+        self.client.send_pack('/', determine_wants, generate_pack_contents)
+        self.assertEqual(self.rout.getvalue(), '0000')
+
+    def test_send_pack_delete_only(self):
+        self.rin.write(
+            '0063310ca9477129b8586fa2afc779c1f57cf64bba6c '
+            'refs/heads/master\x00report-status delete-refs ofs-delta\n'
+            '0000000eunpack ok\n'
+            '0019ok refs/heads/master\n'
+            '0000')
+        self.rin.seek(0)
+
+        def determine_wants(refs):
+            return {'refs/heads/master': '0' * 40}
+
+        def generate_pack_contents(have, want):
+            return {}
+
+        self.client.send_pack('/', determine_wants, generate_pack_contents)
+        self.assertEqual(
+            self.rout.getvalue(),
+            '007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
+            '0000000000000000000000000000000000000000 '
+            'refs/heads/master\x00report-status ofs-delta0000')
+
+    def test_send_pack_new_ref_only(self):
+        self.rin.write(
+            '0063310ca9477129b8586fa2afc779c1f57cf64bba6c '
+            'refs/heads/master\x00report-status delete-refs ofs-delta\n'
+            '0000000eunpack ok\n'
+            '0019ok refs/heads/blah12\n'
+            '0000')
+        self.rin.seek(0)
+
+        def determine_wants(refs):
+            return {
+                'refs/heads/blah12':
+                '310ca9477129b8586fa2afc779c1f57cf64bba6c',
+                'refs/heads/master': '310ca9477129b8586fa2afc779c1f57cf64bba6c'
+            }
+
+        def generate_pack_contents(have, want):
+            return {}
+
+        f = StringIO()
+        empty_pack = write_pack_objects(f, {})
+        self.client.send_pack('/', determine_wants, generate_pack_contents)
+        self.assertEqual(
+            self.rout.getvalue(),
+            '007f0000000000000000000000000000000000000000 '
+            '310ca9477129b8586fa2afc779c1f57cf64bba6c '
+            'refs/heads/blah12\x00report-status ofs-delta0000%s'
+            % f.getvalue())
+
+    def test_send_pack_new_ref(self):
+        self.rin.write(
+            '0064310ca9477129b8586fa2afc779c1f57cf64bba6c '
+            'refs/heads/master\x00 report-status delete-refs ofs-delta\n'
+            '0000000eunpack ok\n'
+            '0019ok refs/heads/blah12\n'
+            '0000')
+        self.rin.seek(0)
+
+        tree = Tree()
+        commit = Commit()
+        commit.tree = tree
+        commit.parents = []
+        commit.author = commit.committer = 'test user'
+        commit.commit_time = commit.author_time = 1174773719
+        commit.commit_timezone = commit.author_timezone = 0
+        commit.encoding = 'UTF-8'
+        commit.message = 'test message'
+
+        def determine_wants(refs):
+            return {
+                'refs/heads/blah12': commit.id,
+                'refs/heads/master': '310ca9477129b8586fa2afc779c1f57cf64bba6c'
+            }
+
+        def generate_pack_contents(have, want):
+            return [(commit, None), (tree, ''), ]
+
+        f = StringIO()
+        pack = write_pack_objects(f, generate_pack_contents(None, None))
+        self.client.send_pack('/', determine_wants, generate_pack_contents)
+        self.assertEqual(
+            self.rout.getvalue(),
+            '007f0000000000000000000000000000000000000000 %s '
+            'refs/heads/blah12\x00report-status ofs-delta0000%s'
+            % (commit.id, f.getvalue()))
+
+    def test_send_pack_no_deleteref_delete_only(self):
+        pkts = ['310ca9477129b8586fa2afc779c1f57cf64bba6c refs/heads/master'
+                '\x00 report-status ofs-delta\n',
+                '',
+                '']
+        for pkt in pkts:
+            if pkt == '':
+                self.rin.write("0000")
+            else:
+                self.rin.write("%04x%s" % (len(pkt)+4, pkt))
+        self.rin.seek(0)
+
+        def determine_wants(refs):
+            return {'refs/heads/master': '0' * 40}
+
+        def generate_pack_contents(have, want):
+            return {}
+
+        self.assertRaises(UpdateRefsError,
+                          self.client.send_pack, "/",
+                          determine_wants, generate_pack_contents)
+        self.assertEqual(self.rout.getvalue(), '0000')
 
 
 class TestSSHVendor(object):
@@ -230,7 +386,7 @@ class TestSSHVendor(object):
         self.username = None
         self.port = None
 
-    def connect_ssh(self, host, command, username=None, port=None):
+    def run_command(self, host, command, username=None, port=None):
         self.host = host
         self.command = command
         self.username = username

dulwich/tests/test_object_store.py

@@ -188,6 +188,11 @@ class ObjectStoreTests(object):
         self.assertEqual((Blob.type_num, 'yummy data'),
                          self.store.get_raw(testobject.id))
 
+    def test_close(self):
+        # For now, just check that close doesn't barf.
+        self.store.add_object(testobject)
+        self.store.close()
+
 
 class MemoryObjectStoreTests(ObjectStoreTests, TestCase):
 
@@ -195,6 +200,32 @@ class MemoryObjectStoreTests(ObjectStoreTests, TestCase):
         TestCase.setUp(self)
         self.store = MemoryObjectStore()
 
+    def test_add_pack(self):
+        o = MemoryObjectStore()
+        f, commit, abort = o.add_pack()
+        try:
+            b = make_object(Blob, data="more yummy data")
+            write_pack_objects(f, [(b, None)])
+        except:
+            abort()
+            raise
+        else:
+            commit()
+
+    def test_add_thin_pack(self):
+        o = MemoryObjectStore()
+        blob = make_object(Blob, data='yummy data')
+        o.add_object(blob)
+
+        f = StringIO()
+        entries = build_pack(f, [
+          (REF_DELTA, (blob.id, 'more yummy data')),
+          ], store=o)
+        o.add_thin_pack(f.read, None)
+        packed_blob_sha = sha_to_hex(entries[0][3])
+        self.assertEqual((Blob.type_num, 'more yummy data'),
+                         o.get_raw(packed_blob_sha))
+
 
 class PackBasedObjectStoreTests(ObjectStoreTests):
 
@@ -269,10 +300,15 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
 
     def test_add_pack(self):
         o = DiskObjectStore(self.store_dir)
-        f, commit = o.add_pack()
-        b = make_object(Blob, data="more yummy data")
-        write_pack_objects(f, [(b, None)])
-        commit()
+        f, commit, abort = o.add_pack()
+        try:
+            b = make_object(Blob, data="more yummy data")
+            write_pack_objects(f, [(b, None)])
+        except:
+            abort()
+            raise
+        else:
+            commit()
 
     def test_add_thin_pack(self):
         o = DiskObjectStore(self.store_dir)

dulwich/tests/test_pack.py

@@ -420,6 +420,57 @@ class TestPack(PackTests):
         self.assertTrue(isinstance(objs[commit_sha], Commit))
 
 
+class TestThinPack(PackTests):
+
+    def setUp(self):
+        super(TestThinPack, self).setUp()
+        self.store = MemoryObjectStore()
+        self.blobs = {}
+        for blob in ('foo', 'bar', 'foo1234', 'bar2468'):
+            self.blobs[blob] = make_object(Blob, data=blob)
+        self.store.add_object(self.blobs['foo'])
+        self.store.add_object(self.blobs['bar'])
+
+        # Build a thin pack. 'foo' is as an external reference, 'bar' an
+        # internal reference.
+        self.pack_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.pack_dir)
+        self.pack_prefix = os.path.join(self.pack_dir, 'pack')
+        with open(self.pack_prefix + '.pack', 'wb') as f:
+            build_pack(f, [
+                (REF_DELTA, (self.blobs['foo'].id, 'foo1234')),
+                (Blob.type_num, 'bar'),
+                (REF_DELTA, (self.blobs['bar'].id, 'bar2468'))],
+                store=self.store)
+
+        # Index the new pack.
+        pack = self.make_pack(True)
+        data = PackData(pack._data_path)
+        data.pack = pack
+        data.create_index(self.pack_prefix + '.idx')
+
+        del self.store[self.blobs['bar'].id]
+
+    def make_pack(self, resolve_ext_ref):
+        return Pack(
+            self.pack_prefix,
+            resolve_ext_ref=self.store.get_raw if resolve_ext_ref else None)
+
+    def test_get_raw(self):
+        self.assertRaises(
+            KeyError, self.make_pack(False).get_raw, self.blobs['foo1234'].id)
+        self.assertEqual(
+            (3, 'foo1234'),
+            self.make_pack(True).get_raw(self.blobs['foo1234'].id))
+
+    def test_iterobjects(self):
+        self.assertRaises(KeyError, list, self.make_pack(False).iterobjects())
+        self.assertEqual(
+            sorted([self.blobs['foo1234'].id, self.blobs['bar'].id,
+                    self.blobs['bar2468'].id]),
+            sorted(o.id for o in self.make_pack(True).iterobjects()))
+
+
 class WritePackTests(TestCase):
 
     def test_write_pack_header(self):

dulwich/tests/test_repository.py

@@ -165,6 +165,19 @@ class RepositoryTests(TestCase):
         r = self._repo = open_repo('a.git')
         self.assertTrue("HEAD" in r)
 
+    def test_get_no_description(self):
+        r = self._repo = open_repo('a.git')
+        self.assertIs(None, r.get_description())
+
+    def test_get_description(self):
+        r = self._repo = open_repo('a.git')
+        f = open(os.path.join(r.path, 'description'), 'w')
+        try:
+            f.write("Some description")
+        finally:
+            f.close()
+        self.assertEquals("Some description", r.get_description())
+
     def test_contains_missing(self):
         r = self._repo = open_repo('a.git')
         self.assertFalse("bar" in r)
@@ -260,6 +273,9 @@ class RepositoryTests(TestCase):
         self.assertEqual(
             [e.commit.id for e in r.get_walker(['2a72d929692c41d8554c07f6301757ba18a65d91'])],
             ['2a72d929692c41d8554c07f6301757ba18a65d91'])
+        self.assertEqual(
+            [e.commit.id for e in r.get_walker('2a72d929692c41d8554c07f6301757ba18a65d91')],
+            ['2a72d929692c41d8554c07f6301757ba18a65d91'])
 
     def test_linear_history(self):
         r = self._repo = open_repo('a.git')
@@ -829,6 +845,7 @@ class PackedRefsFileTests(TestCase):
 # Dict of refs that we expect all RefsContainerTests subclasses to define.
 _TEST_REFS = {
   'HEAD': '42d06bd4b77fed026b154d16493e5deab78f02ec',
+  'refs/heads/40-char-ref-aaaaaaaaaaaaaaaaaa': '42d06bd4b77fed026b154d16493e5deab78f02ec',
   'refs/heads/master': '42d06bd4b77fed026b154d16493e5deab78f02ec',
   'refs/heads/packed': '42d06bd4b77fed026b154d16493e5deab78f02ec',
   'refs/tags/refs-0.1': 'df6800012397fb85c56e7418dd4eb9405dee075c',
@@ -847,7 +864,9 @@ class RefsContainerTests(object):
 
         actual_keys = self._refs.keys('refs/heads')
         actual_keys.discard('loop')
-        self.assertEqual(['master', 'packed'], sorted(actual_keys))
+        self.assertEqual(
+            ['40-char-ref-aaaaaaaaaaaaaaaaaa', 'master', 'packed'],
+            sorted(actual_keys))
         self.assertEqual(['refs-0.1', 'refs-0.2'],
                          sorted(self._refs.keys('refs/tags')))
 
@@ -1111,6 +1130,7 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
 
 
 _TEST_REFS_SERIALIZED = (
+'42d06bd4b77fed026b154d16493e5deab78f02ec\trefs/heads/40-char-ref-aaaaaaaaaaaaaaaaaa\n'
 '42d06bd4b77fed026b154d16493e5deab78f02ec\trefs/heads/master\n'
 '42d06bd4b77fed026b154d16493e5deab78f02ec\trefs/heads/packed\n'
 'df6800012397fb85c56e7418dd4eb9405dee075c\trefs/tags/refs-0.1\n'
@@ -1139,7 +1159,9 @@ class InfoRefsContainerTests(TestCase):
 
         actual_keys = refs.keys('refs/heads')
         actual_keys.discard('loop')
-        self.assertEqual(['master', 'packed'], sorted(actual_keys))
+        self.assertEqual(
+            ['40-char-ref-aaaaaaaaaaaaaaaaaa', 'master', 'packed'],
+            sorted(actual_keys))
         self.assertEqual(['refs-0.1', 'refs-0.2'],
                          sorted(refs.keys('refs/tags')))
 

dulwich/tests/test_server.py

@@ -50,7 +50,9 @@ from dulwich.tests import TestCase
 from dulwich.tests.utils import (
     make_commit,
     )
-
+from dulwich.protocol import (
+    ZERO_SHA,
+    )
 
 ONE = '1' * 40
 TWO = '2' * 40
@@ -200,6 +202,27 @@ class TestUploadPackHandler(UploadPackHandler):
     def required_capabilities(self):
         return ()
 
+class ReceivePackHandlerTestCase(TestCase):
+
+    def setUp(self):
+        super(ReceivePackHandlerTestCase, self).setUp()
+        self._repo = MemoryRepo.init_bare([], {})
+        backend = DictBackend({'/': self._repo})
+        self._handler = ReceivePackHandler(
+          backend, ['/', 'host=lolcathost'], TestProto())
+
+    def test_apply_pack_del_ref(self):
+        refs = {
+            'refs/heads/master': TWO,
+            'refs/heads/fake-branch': ONE}
+        self._repo.refs._update(refs)
+        update_refs = [[ONE, ZERO_SHA, 'refs/heads/fake-branch'], ]
+        status = self._handler._apply_pack(update_refs)
+        self.assertEqual(status[0][0], 'unpack')
+        self.assertEqual(status[0][1], 'ok')
+        self.assertEqual(status[1][0], 'refs/heads/fake-branch')
+        self.assertEqual(status[1][1], 'ok')
+
 
 class ProtocolGraphWalkerTestCase(TestCase):
 
@@ -264,7 +287,7 @@ class ProtocolGraphWalkerTestCase(TestCase):
         self.assertEqual((None, None), _split_proto_line('', allowed))
 
     def test_determine_wants(self):
-        self.assertEqual(None, self._walker.determine_wants({}))
+        self.assertEqual([], self._walker.determine_wants({}))
         self.assertEqual(None, self._walker.proto.get_received_line())
 
         self._walker.proto.set_output([

dulwich/web.py

@@ -409,7 +409,7 @@ try:
     )
     class ServerHandlerLogger(ServerHandler):
         """ServerHandler that uses dulwich's logger for logging exceptions."""
-        
+
         def log_exception(self, exc_info):
             logger.exception('Exception happened during processing of request',
                              exc_info=exc_info)
@@ -432,20 +432,20 @@ try:
 
         def log_error(self, *args):
             logger.error(*args)
-        
+
         def handle(self):
             """Handle a single HTTP request"""
-    
+
             self.raw_requestline = self.rfile.readline()
             if not self.parse_request(): # An error code has been sent, just exit
                 return
-    
+
             handler = ServerHandlerLogger(
                 self.rfile, self.wfile, self.get_stderr(), self.get_environ()
             )
             handler.request_handler = self      # backpointer for logging
             handler.run(self.server.get_app())
-    
+
     class WSGIServerLogger(WSGIServer):
         def handle_error(self, request, client_address):
             """Handle an error. """
@@ -480,3 +480,7 @@ except ImportError:
         sys.stderr.write(
             'Sorry, the wsgiref module is required for dul-web.\n')
         sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

examples/clone.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python
+# This trivial script demonstrates how to clone a remote repository.
+#
+# Example usage:
+#  python examples/clone.py git://github.com/jelmer/dulwich dulwich-clone
+
+import sys
+from getopt import getopt
+from dulwich.repo import Repo
+from dulwich.client import get_transport_and_path
+
+opts, args = getopt(sys.argv, "", [])
+opts = dict(opts)
+
+if len(args) < 2:
+    print "usage: %s host:path path" % (args[0], )
+    sys.exit(1)
+
+# Connect to the remote repository
+client, host_path = get_transport_and_path(args[1])
+path = args[2]
+
+# Create the local repository
+r = Repo.init(path, mkdir=True)
+
+# Fetch the remote objects
+remote_refs = client.fetch(host_path, r,
+    determine_wants=r.object_store.determine_wants_all,
+    progress=sys.stdout.write)
+
+# Update the local head to point at the right object
+r["HEAD"] = remote_refs["HEAD"]
+
+r._build_tree()

examples/diff.py

@@ -0,0 +1,19 @@
+#!/usr/bin/python
+# This trivial script demonstrates how to extract the unified diff for a single
+# commit in a local repository.
+#
+# Example usage:
+#  python examples/diff.py
+
+from dulwich.repo import Repo
+from dulwich.patch import write_tree_diff
+import sys
+
+repo_path = "."
+commit_id = "a6602654997420bcfd0bee2a0563d9416afe34b4"
+
+r = Repo(repo_path)
+
+commit = r[commit_id]
+parent_commit = r[commit.parents[0]]
+write_tree_diff(sys.stdout, r.object_store, parent_commit.tree, commit.tree)

setup.cfg

@@ -0,0 +1,5 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+tag_svn_revision = 0
+

setup.py

@@ -10,7 +10,7 @@ except ImportError:
     has_setuptools = False
 from distutils.core import Distribution
 
-dulwich_version_string = '0.9.0'
+dulwich_version_string = '0.9.1'
 
 include_dirs = []
 # Windows MSVC support
@@ -58,8 +58,6 @@ setup(name='dulwich',
       keywords='git',
       version=dulwich_version_string,
       url='http://samba.org/~jelmer/dulwich',
-      download_url='http://samba.org/~jelmer/dulwich/'
-                   'dulwich-%s.tar.gz' % dulwich_version_string,
       license='GPLv2 or later',
       author='Jelmer Vernooij',
       author_email='jelmer@samba.org',
@@ -71,8 +69,8 @@ setup(name='dulwich',
       All functionality is available in pure Python, but (optional)
       C extensions are also available for better performance.
       """,
-      packages=['dulwich', 'dulwich.tests'],
-      scripts=['bin/dulwich', 'bin/dul-daemon', 'bin/dul-web'],
+      packages=['dulwich', 'dulwich.tests', 'dulwich.tests.compat'],
+      scripts=['bin/dulwich', 'bin/dul-daemon', 'bin/dul-web', 'bin/dul-receive-pack', 'bin/dul-upload-pack'],
       ext_modules=[
           Extension('dulwich._objects', ['dulwich/_objects.c'],
                     include_dirs=include_dirs),