Merge upstream

bin/dul-fetch-pack

@@ -1,57 +0,0 @@
-#!/usr/bin/python
-# dul-daemon - Simple git smart server client
-# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
-# 
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; version 2
-# or (at your option) a later version of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-# MA  02110-1301, USA.
-
-from dulwich.client import TCPGitClient, SimpleFetchGraphWalker
-from dulwich.repo import Repo
-import sys
-from getopt import getopt
-
-opts, args = getopt(sys.argv[1:], "", ["all"])
-
-opts = dict(opts)
-
-if args == []:
-	print "Usage: dul-fetch-pack host:path"
-	sys.exit(1)
-
-if not ":" in args[0]:
-	print "Usage: dul-fetch-pack host:path"
-	sys.exit(1)
-
-(host, path) = args.pop(0).split(":", 1)
-client = TCPGitClient(host)
-
-if "--all" in opts:
-    determine_wants = lambda x: [y for y in x.values() if not y in r.object_store]
-else:
-    determine_wants = lambda x: [y for y in args if not y in r.object_store]
-
-r = Repo(".")
-
-# FIXME: Will just fetch everything..
-graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
-
-f, commit = r.object_store.add_pack()
-try:
-    client.fetch_pack(path, determine_wants, graphwalker, f.write, sys.stdout.write)
-    f.close()
-    commit()
-except:
-    f.close()
-    raise

bin/dulwich

@@ -0,0 +1,115 @@
+#!/usr/bin/python
+# dul-daemon - Simple git smart server client
+# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
+# 
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# or (at your option) a later version of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+import sys
+from getopt import getopt
+
+def cmd_fetch_pack(args):
+	from dulwich.client import TCPGitClient, SimpleFetchGraphWalker
+	from dulwich.repo import Repo
+	opts, args = getopt(args, "", ["all"])
+	opts = dict(opts)
+	if not ":" in args[0]:
+		print "Usage: dulwich fetch-pack [--all] host:path [REF...]"
+		sys.exit(1)
+	(host, path) = args.pop(0).split(":", 1)
+	client = TCPGitClient(host)
+	if "--all" in opts:
+		determine_wants = lambda x: [y for y in x.values() if not y in r.object_store]
+	else:
+		determine_wants = lambda x: [y for y in args if not y in r.object_store]
+	r = Repo(".")
+	graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
+	f, commit = r.object_store.add_pack()
+	try:
+		client.fetch_pack(path, determine_wants, graphwalker, f.write, sys.stdout.write)
+		f.close()
+		commit()
+	except:
+		f.close()
+		raise
+
+
+def cmd_log(args):
+	from dulwich.repo import Repo
+	opts, args = getopt(args, "", [])
+	r = Repo(".")
+	todo = [r.head()]
+	done = set()
+	while todo:
+		sha = todo.pop()
+		assert isinstance(sha, str)
+		if sha in done:
+			continue
+		done.add(sha)
+		commit = r.commit(sha)
+		print "-" * 50
+		print "commit: %s" % sha
+		if len(commit.parents) > 1:
+			print "merge: %s" % "...".join(commit.parents[1:])
+		print "author: %s" % commit.author
+		print "committer: %s" % commit.committer
+		print ""
+		print commit.message
+		print ""
+		todo.extend([p for p in commit.parents if p not in done])
+
+
+def cmd_dump_pack(args):
+	from dulwich.errors import ApplyDeltaError
+	from dulwich.pack import Pack, sha_to_hex
+	import os
+	import sys
+
+	opts, args = getopt(args, "", [])
+
+	if args == []:
+		print "Usage: dulwich dump-pack FILENAME"
+		sys.exit(1)
+
+	basename, _ = os.path.splitext(args[0])
+	x = Pack(basename)
+	print "Object names checksum: %s" % x.name()
+	print "Checksum: %s" % sha_to_hex(x.get_stored_checksum())
+	if not x.check():
+		print "CHECKSUM DOES NOT MATCH"
+	print "Length: %d" % len(x)
+	for name in x:
+		try:
+			print "\t%s" % x[name]
+		except KeyError, k:
+			print "\t%s: Unable to resolve base %s" % (name, k)
+		except ApplyDeltaError, e:
+			print "\t%s: Unable to apply delta: %r" % (name, e)
+
+commands = {
+	"fetch-pack": cmd_fetch_pack,
+	"dump-pack": cmd_dump_pack,
+	"log": cmd_log,
+	}
+
+if len(sys.argv) < 2:
+	print "Usage: %s <%s> [OPTIONS...]" % (sys.argv[0], "|".join(commands.keys()))
+	sys.exit(1)
+
+cmd = sys.argv[1]
+if not cmd in commands:
+	print "No such subcommand: %s" % cmd
+	sys.exit(1)
+commands[cmd](sys.argv[2:])

bin/dumppack

@@ -1,38 +0,0 @@
-#!/usr/bin/python
-# dumppack - Simple pack dumper for dulwich
-# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
-# 
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; version 2
-# of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-# MA  02110-1301, USA.
-
-
-from dulwich.errors import ApplyDeltaError
-from dulwich.pack import Pack, sha_to_hex
-import sys
-
-basename = sys.argv[1]
-x = Pack(basename)
-print "Object names checksum: %s" % x.name()
-print "Checksum: %s" % sha_to_hex(x.get_stored_checksum())
-if not x.check():
-    print "CHECKSUM DOES NOT MATCH"
-print "Length: %d" % len(x)
-for name in x:
-    try:
-        print "\t%s: %s" % (name, x[name])
-    except KeyError, k:
-        print "\t%s: Unable to resolve base %s" % (name, k)
-    except ApplyDeltaError, e:
-        print "\t%s: Unable to apply delta: %r" % (name, e)

dulwich/__init__.py

@@ -16,5 +16,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
+import client
+import protocol
 import repo
 import server

dulwich/errors.py

@@ -63,3 +63,10 @@ class ApplyDeltaError(Exception):
     
     def __init__(self, *args, **kwargs):
         Exception.__init__(self, *args, **kwargs)
+
+
+class NotGitRepository(Exception):
+    """Indicates that no Git repository was found."""
+
+    def __init__(self, *args, **kwargs):
+        Exception.__init__(self, *args, **kwargs)

dulwich/objects.py

@@ -29,13 +29,13 @@ from errors import (NotCommitError,
                     NotBlobError,
                     )
 
-blob_id = "blob"
-tag_id = "tag"
-tree_id = "tree"
-commit_id = "commit"
-parent_id = "parent"
-author_id = "author"
-committer_id = "committer"
+BLOB_ID = "blob"
+TAG_ID = "tag"
+TREE_ID = "tree"
+COMMIT_ID = "commit"
+PARENT_ID = "parent"
+AUTHOR_ID = "author"
+COMMITTER_ID = "committer"
 
 def _decompress(string):
     dcomp = zlib.decompressobj()
@@ -116,7 +116,7 @@ class ShaFile(object):
     """Don't call this directly"""
 
   def _parse_text(self):
-    """For subclasses to do initialistion time parsing"""
+    """For subclasses to do initialisation time parsing"""
 
   @classmethod
   def from_file(cls, filename):
@@ -162,6 +162,9 @@ class ShaFile(object):
   def id(self):
       return self.sha().hexdigest()
 
+  def __repr__(self):
+    return "<%s %s>" % (self.__class__.__name__, self.id)
+
   def __eq__(self, other):
     """Return true id the sha of the two objects match.
 
@@ -174,7 +177,7 @@ class ShaFile(object):
 class Blob(ShaFile):
   """A Git Blob object."""
 
-  _type = blob_id
+  _type = BLOB_ID
 
   @property
   def data(self):
@@ -199,7 +202,7 @@ class Blob(ShaFile):
 class Tag(ShaFile):
   """A Git Tag object."""
 
-  _type = tag_id
+  _type = TAG_ID
 
   @classmethod
   def from_file(cls, filename):
@@ -219,7 +222,7 @@ class Tag(ShaFile):
 class Tree(ShaFile):
   """A Git tree object"""
 
-  _type = tree_id
+  _type = TREE_ID
 
   @classmethod
   def from_file(cls, filename):
@@ -261,7 +264,7 @@ class Tree(ShaFile):
 class Commit(ShaFile):
   """A git commit object"""
 
-  _type = commit_id
+  _type = COMMIT_ID
 
   @classmethod
   def from_file(cls, filename):
@@ -273,11 +276,11 @@ class Commit(ShaFile):
   def _parse_text(self):
     text = self._text
     count = 0
-    assert text.startswith(tree_id), "Invalid commit object, " \
-         "must start with %s" % tree_id
-    count += len(tree_id)
+    assert text.startswith(TREE_ID), "Invalid commit object, " \
+         "must start with %s" % TREE_ID
+    count += len(TREE_ID)
     assert text[count] == ' ', "Invalid commit object, " \
-         "%s must be followed by space not %s" % (tree_id, text[count])
+         "%s must be followed by space not %s" % (TREE_ID, text[count])
     count += 1
     self._tree = text[count:count+40]
     count = count + 40
@@ -285,10 +288,10 @@ class Commit(ShaFile):
          "tree sha must be followed by newline"
     count += 1
     self._parents = []
-    while text[count:].startswith(parent_id):
-      count += len(parent_id)
+    while text[count:].startswith(PARENT_ID):
+      count += len(PARENT_ID)
       assert text[count] == ' ', "Invalid commit object, " \
-           "%s must be followed by space not %s" % (parent_id, text[count])
+           "%s must be followed by space not %s" % (PARENT_ID, text[count])
       count += 1
       self._parents.append(text[count:count+40])
       count += 40
@@ -296,10 +299,10 @@ class Commit(ShaFile):
            "parent sha must be followed by newline"
       count += 1
     self._author = None
-    if text[count:].startswith(author_id):
-      count += len(author_id)
+    if text[count:].startswith(AUTHOR_ID):
+      count += len(AUTHOR_ID)
       assert text[count] == ' ', "Invalid commit object, " \
-           "%s must be followed by space not %s" % (author_id, text[count])
+           "%s must be followed by space not %s" % (AUTHOR_ID, text[count])
       count += 1
       self._author = ''
       while text[count] != '>':
@@ -312,10 +315,10 @@ class Commit(ShaFile):
         count += 1
       count += 1
     self._committer = None
-    if text[count:].startswith(committer_id):
-      count += len(committer_id)
+    if text[count:].startswith(COMMITTER_ID):
+      count += len(COMMITTER_ID)
       assert text[count] == ' ', "Invalid commit object, " \
-           "%s must be followed by space not %s" % (committer_id, text[count])
+           "%s must be followed by space not %s" % (COMMITTER_ID, text[count])
       count += 1
       self._committer = ''
       while text[count] != '>':
@@ -370,10 +373,10 @@ class Commit(ShaFile):
     return self._commit_time
 
 type_map = {
-  blob_id : Blob,
-  tree_id : Tree,
-  commit_id : Commit,
-  tag_id: Tag,
+  BLOB_ID : Blob,
+  TREE_ID : Tree,
+  COMMIT_ID : Commit,
+  TAG_ID: Tag,
 }
 
 num_type_map = {

dulwich/pack.py

@@ -88,6 +88,7 @@ def hex_to_sha(hex):
     ret += chr(int(hex[i:i+2], 16))
   return ret
 
+
 def sha_to_hex(sha):
   """Convert a binary sha string to a hex sha string."""
   ret = ""
@@ -95,6 +96,7 @@ def sha_to_hex(sha):
       ret += "%02x" % ord(i)
   return ret
 
+
 MAX_MMAP_SIZE = 256 * 1024 * 1024
 
 def simple_mmap(f, offset, size, access=mmap.ACCESS_READ):
@@ -362,8 +364,9 @@ class PackData(object):
     self._filename = filename
     assert os.path.exists(filename), "%s is not a packfile" % filename
     self._size = os.path.getsize(filename)
-    assert self._size >= 12, "%s is too small for a packfile" % filename
-    self._header_size = self._read_header()
+    self._header_size = 12
+    assert self._size >= self._header_size, "%s is too small for a packfile" % filename
+    self._read_header()
 
   def _read_header(self):
     f = open(self._filename, 'rb')
@@ -377,7 +380,6 @@ class PackData(object):
     (version,) = struct.unpack_from(">L", header, 4)
     assert version in (2, 3), "Version was %d" % version
     (self._num_objects,) = struct.unpack_from(">L", header, 8)
-    return 12 # Header size
 
   def __len__(self):
       """Returns the number of objects in this pack."""

dulwich/repo.py

@@ -20,7 +20,7 @@
 import os
 
 from commit import Commit
-from errors import MissingCommitError, NotBlobError, NotTreeError, NotCommitError
+from errors import MissingCommitError, NotBlobError, NotTreeError, NotCommitError, NotGitRepository
 from objects import (ShaFile,
                      Commit,
                      Tree,
@@ -46,22 +46,84 @@ class Repo(object):
   ref_locs = ['', 'refs', 'refs/tags', 'refs/heads', 'refs/remotes']
 
   def __init__(self, root):
-    controldir = os.path.join(root, ".git")
-    if os.path.exists(os.path.join(controldir, "objects")):
+    if os.path.isdir(os.path.join(root, ".git", "objects")):
       self.bare = False
-      self._basedir = controldir
-    else:
+      self._controldir = os.path.join(root, ".git")
+    elif os.path.isdir(os.path.join(root, "objects")):
       self.bare = True
-      self._basedir = root
-    self.path = controldir
+      self._controldir = root
+    else:
+      raise NotGitRepository(root)
+    self.path = root
     self.tags = [Tag(name, ref) for name, ref in self.get_tags().items()]
     self._object_store = None
 
-  def basedir(self):
-    return self._basedir
+  def controldir(self):
+    return self._controldir
+
+  def find_missing_objects(self, determine_wants, graph_walker, progress):
+    """Fetch the missing objects required for a set of revisions.
+
+    :param determine_wants: Function that takes a dictionary with heads 
+        and returns the list of heads to fetch.
+    :param graph_walker: Object that can iterate over the list of revisions 
+        to fetch and has an "ack" method that will be called to acknowledge 
+        that a revision is present.
+    :param progress: Simple progress function that will be called with 
+        updated progress strings.
+    """
+    wants = determine_wants(self.heads())
+    commits_to_send = wants
+    ref = graph_walker.next()
+    while ref:
+        commits_to_send.add(ref)
+        if ref in self.object_store:
+            graph_walker.ack(ref)
+        ref = graph_walker.next()
+    sha_done = set()
+    for sha in commits_to_send:
+        if sha in sha_done:
+            continue
+
+        c = self.commit(sha)
+        sha_done.add(sha)
+
+        def parse_tree(tree, sha_done):
+            for mode, name, x in tree.entries():
+                if not x in sha_done:
+                    try:
+                        t = self.tree(x)
+                        sha_done.add(x)
+                        parse_tree(t, sha_done)
+                    except:
+                        sha_done.add(x)
+
+        treesha = c.tree
+        if treesha not in sha_done:
+            t = self.tree(treesha)
+            sha_done.add(treesha)
+            parse_tree(t, sha_done)
+
+        progress("counting objects: %d\r" % len(sha_done))
+    return sha_done
+
+  def fetch_objects(self, determine_wants, graph_walker, progress):
+    """Fetch the missing objects required for a set of revisions.
+
+    :param determine_wants: Function that takes a dictionary with heads 
+        and returns the list of heads to fetch.
+    :param graph_walker: Object that can iterate over the list of revisions 
+        to fetch and has an "ack" method that will be called to acknowledge 
+        that a revision is present.
+    :param progress: Simple progress function that will be called with 
+        updated progress strings.
+    """
+    shas = self.find_missing_objects(determine_wants, graph_walker, progress)
+    for sha in shas:
+        yield self.get_object(sha)
 
   def object_dir(self):
-    return os.path.join(self.basedir(), OBJECTDIR)
+    return os.path.join(self.controldir(), OBJECTDIR)
 
   @property
   def object_store(self):
@@ -81,37 +143,46 @@ class Repo(object):
         if ref[-1] == '\n':
           ref = ref[:-1]
         return self.ref(ref)
-      assert len(contents) == 41, 'Invalid ref'
+      assert len(contents) == 41, 'Invalid ref in %s' % file
       return contents[:-1]
     finally:
       f.close()
 
   def ref(self, name):
     for dir in self.ref_locs:
-      file = os.path.join(self.basedir(), dir, name)
+      file = os.path.join(self.controldir(), dir, name)
       if os.path.exists(file):
         return self._get_ref(file)
 
+  def get_refs(self):
+    ret = {"HEAD": self.head()}
+    for dir in ["refs/heads", "refs/tags"]:
+        for name in os.listdir(os.path.join(self.controldir(), dir)):
+          path = os.path.join(self.controldir(), dir, name)
+          if os.path.isfile(path):
+            ret["/".join([dir, name])] = self._get_ref(path)
+    return ret
+
   def set_ref(self, name, value):
-    file = os.path.join(self.basedir(), name)
+    file = os.path.join(self.controldir(), name)
     open(file, 'w').write(value+"\n")
 
   def remove_ref(self, name):
-    file = os.path.join(self.basedir(), name)
+    file = os.path.join(self.controldir(), name)
     if os.path.exists(file):
       os.remove(file)
       return
 
   def get_tags(self):
     ret = {}
-    for root, dirs, files in os.walk(os.path.join(self.basedir(), 'refs', 'tags')):
+    for root, dirs, files in os.walk(os.path.join(self.controldir(), 'refs', 'tags')):
       for name in files:
         ret[name] = self._get_ref(os.path.join(root, name))
     return ret
 
   def heads(self):
     ret = {}
-    for root, dirs, files in os.walk(os.path.join(self.basedir(), 'refs', 'heads')):
+    for root, dirs, files in os.walk(os.path.join(self.controldir(), 'refs', 'heads')):
       for name in files:
         ret[name] = self._get_ref(os.path.join(root, name))
     return ret
@@ -181,6 +252,9 @@ class Repo(object):
     history.reverse()
     return history
 
+  def __repr__(self):
+      return "<Repo at %r>" % self.path
+
   @classmethod
   def init_bare(cls, path, mkdir=True):
       for d in [["objects"], 
@@ -219,6 +293,7 @@ class ObjectStore(object):
 
     @property
     def packs(self):
+        """List with pack objects."""
         if self._packs is None:
             self._packs = list(load_packs(self.pack_dir()))
         return self._packs
@@ -233,6 +308,11 @@ class ObjectStore(object):
         return None
 
     def get_raw(self, sha):
+        """Obtain the raw text for an object.
+        
+        :param sha: Sha for the object.
+        :return: tuple with object type and object contents.
+        """
         for pack in self.packs:
             if sha in pack:
                 return pack.get_raw(sha, self.get_raw)
@@ -252,13 +332,26 @@ class ObjectStore(object):
         return ShaFile.from_raw_string(type, uncomp)
 
     def move_in_pack(self, path):
+        """Move a specific file containing a pack into the pack directory.
+
+        :note: The file should be on the same file system as the 
+            packs directory.
+
+        :param path: Path to the pack file.
+        """
         p = PackData(path)
         entries = p.sorted_entries(self.get_raw)
-        basename = os.path.join(self.pack_dir(), "pack-%s" % iter_sha1(entry[0] for entry in entries))
+        basename = os.path.join(self.pack_dir(), 
+            "pack-%s" % iter_sha1(entry[0] for entry in entries))
         write_pack_index_v2(basename+".idx", entries, p.calculate_checksum())
         os.rename(path, basename + ".pack")
 
     def add_pack(self):
+        """Add a new pack to this object store. 
+
+        :return: Fileobject to write to and a commit function to 
+            call when the pack is finished.
+        """
         fd, path = tempfile.mkstemp(dir=self.pack_dir(), suffix=".pack")
         f = os.fdopen(fd, 'w')
         def commit():

dulwich/server.py

@@ -28,15 +28,7 @@ class Backend(object):
         """
         Get all the refs in the repository
 
-        :return: list of tuple(name, sha)
-        """
-        raise NotImplementedError
-
-    def has_revision(self, sha):
-        """
-        Is a given sha in this repository?
-
-        :return: True or False
+        :return: dict of name -> sha
         """
         raise NotImplementedError
 
@@ -48,13 +40,10 @@ class Backend(object):
         """
         raise NotImplementedError
 
-    def generate_pack(self, want, have, write, progress):
+    def fetch_objects(self, determine_wants, graph_waker, progress):
         """
-        Generate a pack containing all commits a client is missing
+        Yield the objects required for a list of commits.
 
-        :param want: is a list of sha's the client desires
-        :param have: is a list of sha's the client has (allowing us to send the minimal pack)
-        :param write: is a callback to write pack data to the client
         :param progress: is a callback to send progress messages to the client
         """
         raise NotImplementedError
@@ -72,15 +61,7 @@ class GitBackend(Backend):
         self.repo = Repo(self.gitdir)
 
     def get_refs(self):
-        refs = []
-        if self.repo.head():
-            refs.append(('HEAD', self.repo.head()))
-        for ref, sha in self.repo.heads().items():
-            refs.append(('refs/heads/'+ref,sha))
-        return refs
-
-    def has_revision(self, sha):
-        return self.repo.get_object(sha) != None
+        return self.repo.get_refs()
 
     def apply_pack(self, refs, read):
         # store the incoming pack in the repository
@@ -103,11 +84,9 @@ class GitBackend(Backend):
 
         print "pack applied"
 
-    def generate_pack(self, want, have, write, progress):
-        progress("dul-daemon says what\n")
-        sha_queue = generate_pack_contents(want, have, self.repo.get_object) 
-        write_pack_data(ProtocolFile(None, write), (self.repo.get_object(sha) for sha in sha_queue), len(sha_queue))
-        progress("how was that, then?\n")
+    def fetch_objects(self, determine_wants, graph_waker, progress):
+        for sha in generate_pack_contents(determine_wants(), have, self.repo.get_object):
+            yield self.repo.get_object(sha)
 
 
 class Handler(object):
@@ -126,59 +105,62 @@ class UploadPackHandler(Handler):
         return ("multi_ack", "side-band-64k", "thin-pack", "ofs-delta")
 
     def handle(self):
-        refs = self.backend.get_refs()
+        def determine_wants(heads):
+            keys = heads.keys()
+            if keys:
+                self.proto.write_pkt_line("%s %s\x00%s\n" % (keys[0], heads[keys[0]], self.capabilities()))
+                for k in keys[1:]:
+                    self.proto.write_pkt_line("%s %s\n" % (k, heads[k]))
+
+            # i'm done..
+            self.proto.write("0000")
+
+            # Now client will either send "0000", meaning that it doesnt want to pull.
+            # or it will start sending want want want commands
+            want = self.proto.read_pkt_line()
+            if want == None:
+                return []
 
-        if refs:
-            self.proto.write_pkt_line("%s %s\x00%s\n" % (refs[0][1], refs[0][0], self.capabilities()))
-            for i in range(1, len(refs)):
-                ref = refs[i]
-                self.proto.write_pkt_line("%s %s\n" % (ref[1], ref[0]))
+            want, self.client_capabilities = extract_capabilities(want)
 
-        # i'm done..
-        self.proto.write("0000")
+            want_revs = []
+            while want and want[:4] == 'want':
+                want_rev = want[5:45]
+                # FIXME: This check probably isnt needed?
+                want_revs.append(want_rev)
+                want = self.proto.read_pkt_line()
+            return want_revs
 
-        # Now client will either send "0000", meaning that it doesnt want to pull.
-        # or it will start sending want want want commands
-        want = self.proto.read_pkt_line()
-        if want == None:
-            return
+        progress = lambda x: self.proto.write_sideband(2, x)
 
-        want, client_capabilities = extract_capabilities(want)
+        class ProtocolGraphWalker(object):
 
-        # Keep reading the list of demands until we hit another "0000" 
-        want_revs = []
-        while want and want[:4] == 'want':
-            want_rev = want[5:45]
-            # FIXME: This check probably isnt needed?
-            if self.backend.has_revision(want_rev):
-               want_revs.append(want_rev)
-            want = self.proto.read_pkt_line()
-        
-        # Client will now tell us which commits it already has - if we have them we ACK them
-        # this allows client to stop looking at that commits parents (main reason why git pull is fast)
-        last_sha = None
-        have_revs = []
-        have = self.proto.read_pkt_line()
-        while have and have[:4] == 'have':
-            have_ref = have[5:45]
-            if self.backend.has_revision(have_ref):
+            def __init__(self):
+                self._last_sha = None
+
+            def ack(self, have_ref):
                 self.proto.write_pkt_line("ACK %s continue\n" % have_ref)
-                last_sha = have_ref
-                have_revs.append(have_ref)
-            have = self.proto.read_pkt_line()
 
-        # At some point client will stop sending commits and will tell us it is done
-        assert(have[:4] == "done")
+            def next(self):
+                have = self.proto.read_pkt_line()
+                if have[:4] == 'have':
+                    return have[5:45]
+
+                if have[:4] == 'done':
+                    return None
 
-        # Oddness: Git seems to resend the last ACK, without the "continue" statement
-        if last_sha:
-            self.proto.write_pkt_line("ACK %s\n" % last_sha)
+                if self._last_sha:
+                    # Oddness: Git seems to resend the last ACK, without the "continue" statement
+                    self.proto.write_pkt_line("ACK %s\n" % self._last_sha)
 
-        # The exchange finishes with a NAK
-        self.proto.write_pkt_line("NAK\n")
-      
-        self.backend.generate_pack(want_revs, have_revs, lambda x: self.proto.write_sideband(1, x), lambda x: self.proto.write_sideband(2, x))
+                # The exchange finishes with a NAK
+                self.proto.write_pkt_line("NAK\n")
 
+        objects = list(self.backend.fetch_objects(determine_wants, graph_walker, progress))
+        progress("dul-daemon says what\n")
+        progress("counting objects: %d, done.\n" % len(objects))
+        write_pack_data(ProtocolFile(None, write), objects, len(objects))
+        progress("how was that, then?\n")
         # we are done
         self.proto.write("0000")
 
@@ -189,7 +171,7 @@ class ReceivePackHandler(Handler):
         return ("report-status", "delete-refs")
 
     def handle(self):
-        refs = self.backend.get_refs()
+        refs = self.backend.get_refs().items()
 
         if refs:
             self.proto.write_pkt_line("%s %s\x00%s\n" % (refs[0][1], refs[0][0], self.capabilities()))

dulwich/tests/test_pack.py

@@ -111,7 +111,7 @@ class TestPackData(PackTests):
 
   def test_iterentries(self):
     p = self.get_pack_data(pack1_sha)
-    self.assertEquals([('og\x0c\x0f\xb5?\x94cv\x0br\x95\xfb\xb8\x14\xe9e\xfb \xc8', 178, -1718046665), ('\xb2\xa2vj(y\xc2\t\xab\x11v\xe7\xe7x\xb8\x1a\xe4"\xee\xaa', 138, -901046474), ('\xf1\x8f\xaa\x16S\x1a\xc5p\xa3\xfd\xc8\xc7\xca\x16h%H\xda\xfd\x12', 12, 1185722901)], list(p.iterentries()))
+    self.assertEquals(set([('og\x0c\x0f\xb5?\x94cv\x0br\x95\xfb\xb8\x14\xe9e\xfb \xc8', 178, -1718046665), ('\xb2\xa2vj(y\xc2\t\xab\x11v\xe7\xe7x\xb8\x1a\xe4"\xee\xaa', 138, -901046474), ('\xf1\x8f\xaa\x16S\x1a\xc5p\xa3\xfd\xc8\xc7\xca\x16h%H\xda\xfd\x12', 12, 1185722901)]), set(p.iterentries()))
 
   def test_create_index_v1(self):
     p = self.get_pack_data(pack1_sha)
@@ -162,8 +162,8 @@ class TestPack(PackTests):
 
     def test_copy(self):
         p = self.get_pack(pack1_sha)
-        write_pack("testcopy", p.iterobjects(), len(p))
-        self.assertEquals(p, Pack("testcopy"))
+        write_pack("Elch", p.iterobjects(), len(p))
+        self.assertEquals(p, Pack("Elch"))
 
     def test_commit_obj(self):
         p = self.get_pack(pack1_sha)

dulwich/tests/test_repository.py

@@ -33,7 +33,7 @@ class RepositoryTests(unittest.TestCase):
   def test_simple_props(self):
     r = self.open_repo('a')
     basedir = os.path.join(os.path.dirname(__file__), 'data/repos/a/.git')
-    self.assertEqual(r.basedir(), basedir)
+    self.assertEqual(r.controldir(), basedir)
     self.assertEqual(r.object_dir(), os.path.join(basedir, 'objects'))
 
   def test_ref(self):
@@ -41,6 +41,13 @@ class RepositoryTests(unittest.TestCase):
     self.assertEqual(r.ref('master'),
                      'a90fa2d900a17e99b433217e988c4eb4a2e9a097')
 
+  def test_get_refs(self):
+    r = self.open_repo('a')
+    self.assertEquals({
+        'HEAD': 'a90fa2d900a17e99b433217e988c4eb4a2e9a097', 
+        'refs/heads/master': 'a90fa2d900a17e99b433217e988c4eb4a2e9a097'
+        }, r.get_refs())
+
   def test_head(self):
     r = self.open_repo('a')
     self.assertEqual(r.head(), 'a90fa2d900a17e99b433217e988c4eb4a2e9a097')

setup.py

@@ -19,4 +19,5 @@ setup(name='dulwich',
       in one of the Monty Python sketches.
       """,
       packages=['dulwich', 'dulwich.tests'],
+      scripts=['bin/dulwich', 'bin/dul-daemon'],
       )