Merge John.

bin/dulwich

@@ -20,16 +20,21 @@
 import sys
 from getopt import getopt
 
+def get_transport_and_path(uri):
+    from dulwich.client import TCPGitClient, SSHGitClient, SubprocessGitClient
+    for handler, transport in (("git://", TCPGitClient), ("git+ssh://", SSHGitClient)):
+        if uri.startswith(handler):
+            host, path = uri[len(handler):].split("/", 1)
+            return transport(host), "/"+path
+    # if its not git or git+ssh, try a local url..
+    return SubprocessGitClient(), uri
+
 def cmd_fetch_pack(args):
-	from dulwich.client import TCPGitClient, SimpleFetchGraphWalker
+	from dulwich.client import SimpleFetchGraphWalker
 	from dulwich.repo import Repo
 	opts, args = getopt(args, "", ["all"])
 	opts = dict(opts)
-	if not ":" in args[0]:
-		print "Usage: dulwich fetch-pack [--all] host:path [REF...]"
-		sys.exit(1)
-	(host, path) = args.pop(0).split(":", 1)
-	client = TCPGitClient(host)
+        client, path = get_transport_and_path(args.pop(0))
 	if "--all" in opts:
 		determine_wants = lambda x: [y for y in x.values() if not y in r.object_store]
 	else:
@@ -137,7 +142,7 @@ def cmd_init(args):
 
 
 def cmd_clone(args):
-	from dulwich.client import TCPGitClient, SimpleFetchGraphWalker
+	from dulwich.client import SimpleFetchGraphWalker
 	from dulwich.repo import Repo
 	import os
 	import sys
@@ -148,11 +153,7 @@ def cmd_clone(args):
 		print "usage: dulwich clone host:path [PATH]"
 		sys.exit(1)
 
-	if not ":" in args[0]:
-		print "Usage: dulwich clone host:path [PATH]"
-		sys.exit(1)
-	(host, host_path) = args.pop(0).split(":", 1)
-	client = TCPGitClient(host)
+        client, host_path = get_transport_and_path(args.pop(0))
 
 	if len(args) > 0:
 		path = args.pop(0)

dulwich/client.py

@@ -16,9 +16,12 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
+import os
 import select
 import socket
+import subprocess
 from dulwich.protocol import Protocol, TCP_GIT_PORT, extract_capabilities
+from dulwich.pack import write_pack_data
 
 class SimpleFetchGraphWalker(object):
 
@@ -68,7 +71,7 @@ class GitClient(object):
                 refs[ref] = sha
         return refs, server_capabilities
 
-    def send_pack(self, path):
+    def send_pack(self, path, generate_pack_contents):
         refs, server_capabilities = self.read_refs()
         changed_refs = [] # FIXME
         if not changed_refs:
@@ -83,9 +86,8 @@ class GitClient(object):
             if changed_refs[0] != "0"*40:
                 have.append(changed_refs[0])
         self.proto.write_pkt_line(None)
-        # FIXME: This is implementation specific
-        # shas = generate_pack_contents(want, have, None)
-        # write_pack_data(self.write, shas, len(shas))
+        shas = generate_pack_contents(want, have, None)
+        write_pack_data(self.write, shas, len(shas))
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
         """Retrieve a pack from a git smart server.
@@ -96,7 +98,6 @@ class GitClient(object):
         :param progress: Callback for progress reports (strings)
         """
         (refs, server_capabilities) = self.read_refs()
-       
         wants = determine_wants(refs)
         if not wants:
             self.proto.write_pkt_line(None)
@@ -152,3 +153,84 @@ class TCPGitClient(GitClient):
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
         self.proto.send_cmd("git-upload-pack", path, "host=%s" % self.host)
         super(TCPGitClient, self).fetch_pack(path, determine_wants, graph_walker, pack_data, progress)
+
+
+class SubprocessGitClient(GitClient):
+
+    def __init__(self):
+        self.proc = None
+
+    def _connect(self, service, *args):
+        argv = [service] + list(args)
+        self.proc = subprocess.Popen(argv, bufsize=0,
+                                stdin=subprocess.PIPE,
+                                stdout=subprocess.PIPE)
+        def read_fn(size):
+            return self.proc.stdout.read(size)
+        def write_fn(data):
+            self.proc.stdin.write(data)
+            self.proc.stdin.flush()
+        return GitClient(self.proc.stdout.fileno(), read_fn, write_fn)
+
+    def send_pack(self, path):
+        client = self._connect("git-receive-pack", path)
+        client.send_pack(path)
+
+    def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
+        client = self._connect("git-upload-pack", path)
+        client.fetch_pack(path, determine_wants, graph_walker, pack_data, progress)
+
+
+class SSHSubprocess(object):
+    """A socket-like object that talks to an ssh subprocess via pipes."""
+
+    def __init__(self, proc):
+        self.proc = proc
+
+    def send(self, data):
+        return os.write(self.proc.stdin.fileno(), data)
+
+    def recv(self, count):
+        return self.proc.stdout.read(count)
+
+    def close(self):
+        self.proc.stdin.close()
+        self.proc.stdout.close()
+        self.proc.wait()
+
+
+class SSHVendor(object):
+
+    def connect_ssh(self, host, command, username=None, port=None):
+        #FIXME: This has no way to deal with passwords..
+        args = ['ssh', '-x']
+        if port is not None:
+            args.extend(['-p', str(port)])
+        if username is not None:
+            host = "%s@%s" % (username, host)
+        args.append(host)
+        proc = subprocess.Popen(args + command,
+                                stdin=subprocess.PIPE,
+                                stdout=subprocess.PIPE)
+        return SSHSubprocess(proc)
+
+# Can be overridden by users
+get_ssh_vendor = SSHVendor
+
+
+class SSHGitClient(GitClient):
+
+    def __init__(self, host, port=None):
+        self.host = host
+        self.port = port
+
+    def send_pack(self, path):
+        remote = get_ssh_vendor().connect_ssh(self.host, ["git-receive-pack %s" % path], port=self.port)
+        client = GitClient(remote.proc.stdout.fileno(), remote.recv, remote.send)
+        client.send_pack(path)
+
+    def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
+        remote = get_ssh_vendor().connect_ssh(self.host, ["git-upload-pack %s" % path], port=self.port)
+        client = GitClient(remote.proc.stdout.fileno(), remote.recv, remote.send)
+        client.fetch_pack(path, determine_wants, graph_walker, pack_data, progress)
+

dulwich/objects.py

@@ -173,6 +173,10 @@ class ShaFile(object):
   def id(self):
       return self.sha().hexdigest()
 
+  @property
+  def type(self):
+      return self._num_type
+
   def __repr__(self):
     return "<%s %s>" % (self.__class__.__name__, self.id)
 

dulwich/pack.py

@@ -589,8 +589,8 @@ def write_pack_data(f, objects, num_objects, window=10):
     # Build a list of objects ordered by the magic Linus heuristic
     # This helps us find good objects to diff against us
     magic = []
-    for o in recency:
-        magic.append( (o._num_type, "filename", 1, -len(o.as_raw_string()[1]), o) )
+    for obj, path in recency:
+        magic.append( (obj.type, path, 1, -len(obj.as_raw_string()[1]), obj) )
     magic.sort()
     # Build a map of objects and their index in magic - so we can find preceeding objects
     # to diff against
@@ -603,7 +603,7 @@ def write_pack_data(f, objects, num_objects, window=10):
     f.write("PACK")               # Pack header
     f.write(struct.pack(">L", 2)) # Pack version
     f.write(struct.pack(">L", num_objects)) # Number of objects in pack
-    for o in recency:
+    for o, path in recency:
         sha1 = o.sha().digest()
         crc32 = o.crc32()
         orig_t, raw = o.as_raw_string()
@@ -612,7 +612,7 @@ def write_pack_data(f, objects, num_objects, window=10):
         #for i in range(offs[o]-window, window):
         #    if i < 0 or i >= len(offs): continue
         #    b = magic[i][4]
-        #    if b._num_type != orig_t: continue
+        #    if b.type != orig_t: continue
         #    _, base = b.as_raw_string()
         #    delta = create_delta(base, raw)
         #    if len(delta) < len(winner):

dulwich/repo.py

@@ -18,7 +18,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-import os
+import os, stat
 
 from commit import Commit
 from errors import (
@@ -101,36 +101,32 @@ class Repo(object):
     sha_done = set()
     ref = graph_walker.next()
     while ref:
-        sha_done.add(ref)
         if ref in self.object_store:
             graph_walker.ack(ref)
         ref = graph_walker.next()
     while commits_to_send:
-        sha = commits_to_send.pop()
+        sha = (commits_to_send.pop(), None)
         if sha in sha_done:
             continue
 
         c = self.commit(sha)
         assert isinstance(c, Commit)
-        sha_done.add(sha)
+        sha_done.add((sha, None))
 
         commits_to_send.update([p for p in c.parents if not p in sha_done])
 
         def parse_tree(tree, sha_done):
-            for mode, name, x in tree.entries():
-                if not x in sha_done:
-                    try:
-                        t = self.tree(x)
-                        sha_done.add(x)
-                        parse_tree(t, sha_done)
-                    except:
-                        sha_done.add(x)
+            for mode, name, sha in tree.entries():
+                if (sha, name) in sha_done:
+                    continue
+                if mode & stat.S_IFDIR:
+                    parse_tree(self.tree(sha), sha_done)
+                sha_done.add((sha, name))
 
         treesha = c.tree
-        if treesha not in sha_done:
-            t = self.tree(treesha)
-            sha_done.add(treesha)
-            parse_tree(t, sha_done)
+        if c.tree not in sha_done:
+            parse_tree(self.tree(c.tree), sha_done)
+            sha_done.add((c.tree, None))
 
         progress("counting objects: %d\r" % len(sha_done))
     return sha_done
@@ -148,7 +144,7 @@ class Repo(object):
     :return: tuple with number of objects, iterator over objects
     """
     shas = self.find_missing_objects(determine_wants, graph_walker, progress)
-    return (len(shas), (self.get_object(sha) for sha in shas))
+    return (len(shas), ((self.get_object(sha), path) for sha, path in shas))
 
   def object_dir(self):
     return os.path.join(self.controldir(), OBJECTDIR)

dulwich/server.py

@@ -146,8 +146,13 @@ class UploadPackHandler(Handler):
 
         graph_walker = ProtocolGraphWalker(self.proto)
         num_objects, objects_iter = self.backend.fetch_objects(determine_wants, graph_walker, progress)
+
+        # Do they want any objects?
+        if num_objects == 0:
+            return
+
         progress("dul-daemon says what\n")
-        progress("counting objects: %d, done.\n" % len(objects))
+        progress("counting objects: %d, done.\n" % num_objects)
         write_pack_data(ProtocolFile(None, write), objects_iter, num_objects)
         progress("how was that, then?\n")
         # we are done

dulwich/tests/test_pack.py

@@ -190,7 +190,7 @@ class TestPack(PackTests):
 
     def test_copy(self):
         p = self.get_pack(pack1_sha)
-        write_pack("Elch", p.iterobjects(), len(p))
+        write_pack("Elch", [(x, "") for x in p.iterobjects()], len(p))
         self.assertEquals(p, Pack("Elch"))
 
     def test_commit_obj(self):