Imported Upstream version 0.9.7

Makefile

@@ -1,8 +1,9 @@
 PYTHON = python
-PYLINT = pylint
+PYFLAKES = pyflakes
+PEP8 = pep8
 SETUP = $(PYTHON) setup.py
 PYDOCTOR ?= pydoctor
-ifeq ($(shell $(PYTHON) -c "import sys; print sys.version_info >= (2, 7)"),True)
+ifeq ($(shell $(PYTHON) -c "import sys; print(sys.version_info >= (2, 7))"),True)
 TESTRUNNER ?= unittest
 else
 TESTRUNNER ?= unittest2.__main__
@@ -29,7 +30,7 @@ check:: build
 	$(RUNTEST) dulwich.tests.test_suite
 
 check-tutorial:: build
-	$(RUNTEST) dulwich.tests.tutorial_test_suite 
+	$(RUNTEST) dulwich.tests.tutorial_test_suite
 
 check-nocompat:: build
 	$(RUNTEST) dulwich.tests.nocompat_test_suite
@@ -49,5 +50,11 @@ clean::
 	$(SETUP) clean --all
 	rm -f dulwich/*.so
 
-lint::
-	$(PYLINT) --rcfile=.pylintrc --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" dulwich
+flakes:
+	$(PYFLAKES) dulwich
+
+pep8:
+	$(PEP8) dulwich
+
+before-push: check
+	git diff origin/master | $(PEP8) --diff

NEWS

@@ -1,3 +1,58 @@
+0.9.7	2014-06-08
+
+ BUG FIXES
+
+  * Fix tests dependent on hash ordering. (Michael Edgar)
+
+  * Support staging symbolic links in Repo.stage.
+    (Robert Brown)
+
+  * Ensure that all files object are closed when running the test suite.
+    (Gary van der Merwe)
+
+  * When writing OFS_DELTA pack entries, write correct offset.
+    (Augie Fackler)
+
+  * Fix handler of larger copy operations in packs. (Augie Fackler)
+
+  * Various fixes to improve test suite running on Windows.
+    (Gary van der Merwe)
+
+  * Fix logic for extra adds of identical files in rename detector.
+    (Robert Brown)
+
+ IMPROVEMENTS
+
+  * Add porcelain 'status'. (Ryan Faulkner)
+
+  * Add porcelain 'daemon'. (Jelmer Vernooij)
+
+  * Add `dulwich.greenthreads` module which provides support
+    for concurrency of some object store operations.
+    (Fabien Boucher)
+
+  * Various changes to improve compatibility with Python 3.
+    (Gary van der Merwe, Hannu Valtonen, michael-k)
+
+  * Add OpenStack Swift backed repository implementation
+    in dulwich.contrib. See README.swift for details. (Fabien Boucher)
+
+API CHANGES
+
+  * An optional close function can be passed to the Protocol class. This will
+    be called by its close method. (Gary van der Merwe)
+
+  * All classes with close methods are now context managers, so that they can
+    be easily closed using a `with` statement. (Gary van der Merwe)
+
+  * Remove deprecated `num_objects` argument to `write_pack` methods.
+    (Jelmer Vernooij)
+
+ OTHER CHANGES
+
+  * The 'dul-daemon' script has been removed. The same functionality
+    is now available as 'dulwich daemon'. (Jelmer Vernooij)
+
 0.9.6	2014-04-23
 
  IMPROVEMENTS

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: dulwich
-Version: 0.9.6
+Version: 0.9.7
 Summary: Python Git Library
 Home-page: https://samba.org/~jelmer/dulwich
 Author: Jelmer Vernooij

bin/dul-receive-pack

@@ -22,7 +22,7 @@ import os
 import sys
 
 if len(sys.argv) < 2:
-    print >>sys.stderr, "usage: %s <git-dir>" % os.path.basename(sys.argv[0])
+    sys.stderr.write("usage: %s <git-dir>\n" % os.path.basename(sys.argv[0]))
     sys.exit(1)
 
 sys.exit(serve_command(ReceivePackHandler))

bin/dul-upload-pack

@@ -22,7 +22,7 @@ import os
 import sys
 
 if len(sys.argv) < 2:
-    print >>sys.stderr, "usage: %s <git-dir>" % os.path.basename(sys.argv[0])
+    sys.stderr.write("usage: %s <git-dir>\n" % os.path.basename(sys.argv[0]))
     sys.exit(1)
 
 sys.exit(serve_command(UploadPackHandler))

bin/dulwich

@@ -29,6 +29,7 @@ a way to test Dulwich.
 import os
 import sys
 from getopt import getopt
+import optparse
 
 from dulwich import porcelain
 from dulwich.client import get_transport_and_path
@@ -80,9 +81,9 @@ def cmd_fetch(args):
     if "--all" in opts:
         determine_wants = r.object_store.determine_wants_all
     refs = client.fetch(path, r, progress=sys.stdout.write)
-    print "Remote refs:"
+    print("Remote refs:")
     for item in refs.iteritems():
-        print "%s -> %s" % item
+        print("%s -> %s" % item)
 
 
 def cmd_log(args):
@@ -98,7 +99,7 @@ def cmd_diff(args):
     opts, args = getopt(args, "", [])
 
     if args == []:
-        print "Usage: dulwich diff COMMITID"
+        print("Usage: dulwich diff COMMITID")
         sys.exit(1)
 
     r = Repo(".")
@@ -112,37 +113,37 @@ def cmd_dump_pack(args):
     opts, args = getopt(args, "", [])
 
     if args == []:
-        print "Usage: dulwich dump-pack FILENAME"
+        print("Usage: dulwich dump-pack FILENAME")
         sys.exit(1)
 
     basename, _ = os.path.splitext(args[0])
     x = Pack(basename)
-    print "Object names checksum: %s" % x.name()
-    print "Checksum: %s" % sha_to_hex(x.get_stored_checksum())
+    print("Object names checksum: %s" % x.name())
+    print("Checksum: %s" % sha_to_hex(x.get_stored_checksum()))
     if not x.check():
-        print "CHECKSUM DOES NOT MATCH"
-    print "Length: %d" % len(x)
+        print("CHECKSUM DOES NOT MATCH")
+    print("Length: %d" % len(x))
     for name in x:
         try:
-            print "\t%s" % x[name]
+            print("\t%s" % x[name])
         except KeyError, k:
-            print "\t%s: Unable to resolve base %s" % (name, k)
+            print("\t%s: Unable to resolve base %s" % (name, k))
         except ApplyDeltaError, e:
-            print "\t%s: Unable to apply delta: %r" % (name, e)
+            print("\t%s: Unable to apply delta: %r" % (name, e))
 
 
 def cmd_dump_index(args):
     opts, args = getopt(args, "", [])
 
     if args == []:
-        print "Usage: dulwich dump-index FILENAME"
+        print("Usage: dulwich dump-index FILENAME")
         sys.exit(1)
 
     filename = args[0]
     idx = Index(filename)
 
     for o in idx:
-        print o, idx[o]
+        print(o, idx[o])
 
 
 def cmd_init(args):
@@ -162,7 +163,7 @@ def cmd_clone(args):
     opts = dict(opts)
 
     if args == []:
-        print "usage: dulwich clone host:path [PATH]"
+        print("usage: dulwich clone host:path [PATH]")
         sys.exit(1)
 
     source = args.pop(0)
@@ -183,7 +184,7 @@ def cmd_commit(args):
 def cmd_commit_tree(args):
     opts, args = getopt(args, "", ["message"])
     if args == []:
-        print "usage: dulwich commit-tree tree"
+        print("usage: dulwich commit-tree tree")
         sys.exit(1)
     opts = dict(opts)
     porcelain.commit_tree(".", tree=args[0], message=opts["--message"])
@@ -196,7 +197,7 @@ def cmd_update_server_info(args):
 def cmd_symbolic_ref(args):
     opts, args = getopt(args, "", ["ref-name", "force"])
     if not args:
-        print "Usage: dulwich symbolic-ref REF_NAME [--force]"
+        print("Usage: dulwich symbolic-ref REF_NAME [--force]")
         sys.exit(1)
 
     ref_name = args.pop(0)
@@ -211,26 +212,30 @@ def cmd_show(args):
 def cmd_diff_tree(args):
     opts, args = getopt(args, "", [])
     if len(args) < 2:
-        print "Usage: dulwich diff-tree OLD-TREE NEW-TREE"
+        print("Usage: dulwich diff-tree OLD-TREE NEW-TREE")
         sys.exit(1)
     porcelain.diff_tree(".", args[0], args[1])
 
 
 def cmd_rev_list(args):
     opts, args = getopt(args, "", [])
-    if len(args) < 2:
-        print "Usage: dulwich tag NAME"
+    if len(args) < 1:
+        print('Usage: dulwich rev-list COMMITID...')
         sys.exit(1)
-    porcelain.tag(".", args[0])
+    porcelain.rev_list('.', args)
 
 
 def cmd_tag(args):
-    opts, args = getopt(args, "", [])
-    porcelain.tag(".", args[0])
+    opts, args = getopt(args, '', [])
+    if len(args) < 2:
+        print 'Usage: dulwich tag NAME'
+        sys.exit(1)
+    porcelain.tag('.', args[0])
 
 
 def cmd_reset(args):
     opts, args = getopt(args, "", ["hard", "soft", "mixed"])
+    opts = dict(opts)
     mode = ""
     if "--hard" in opts:
         mode = "hard"
@@ -238,7 +243,29 @@ def cmd_reset(args):
         mode = "soft"
     elif "--mixed" in opts:
         mode = "mixed"
-    porcelain.tag(".", mode=mode, *args)
+    porcelain.reset('.', mode=mode, *args)
+
+
+def cmd_daemon(args):
+    from dulwich import log_utils
+    from dulwich.protocol import TCP_GIT_PORT
+    parser = optparse.OptionParser()
+    parser.add_option("-l", "--listen_address", dest="listen_address",
+                      default="localhost",
+                      help="Binding IP address.")
+    parser.add_option("-p", "--port", dest="port", type=int,
+                      default=TCP_GIT_PORT,
+                      help="Binding TCP port.")
+    options, args = parser.parse_args(args)
+
+    log_utils.default_logging_config()
+    if len(args) > 1:
+        gitdir = args[1]
+    else:
+        gitdir = '.'
+    from dulwich import porcelain
+    porcelain.daemon(gitdir, address=options.listen_address,
+                     port=options.port)
 
 
 commands = {
@@ -247,6 +274,7 @@ commands = {
     "clone": cmd_clone,
     "commit": cmd_commit,
     "commit-tree": cmd_commit_tree,
+    "daemon": cmd_daemon,
     "diff": cmd_diff,
     "diff-tree": cmd_diff_tree,
     "dump-pack": cmd_dump_pack,
@@ -265,11 +293,11 @@ commands = {
     }
 
 if len(sys.argv) < 2:
-    print "Usage: %s <%s> [OPTIONS...]" % (sys.argv[0], "|".join(commands.keys()))
+    print("Usage: %s <%s> [OPTIONS...]" % (sys.argv[0], "|".join(commands.keys())))
     sys.exit(1)
 
 cmd = sys.argv[1]
 if not cmd in commands:
-    print "No such subcommand: %s" % cmd
+    print("No such subcommand: %s" % cmd)
     sys.exit(1)
 commands[cmd](sys.argv[2:])

docs/tutorial/repo.txt

@@ -14,7 +14,7 @@ repositories:
   Regular Repositories -- They are the ones you create using ``git init`` and
   you daily use. They contain a ``.git`` folder.
 
-  Bare Repositories -- There is not ".git" folder. The top-level folder
+  Bare Repositories -- There is no ".git" folder. The top-level folder
   contains itself the "branches", "hooks"... folders. These are used for
   published repositories (mirrors). They do not have a working tree.
 

dulwich.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: dulwich
-Version: 0.9.6
+Version: 0.9.7
 Summary: Python Git Library
 Home-page: https://samba.org/~jelmer/dulwich
 Author: Jelmer Vernooij

dulwich.egg-info/SOURCES.txt

@@ -6,7 +6,6 @@ Makefile
 NEWS
 README.md
 setup.py
-bin/dul-daemon
 bin/dul-receive-pack
 bin/dul-upload-pack
 bin/dul-web
@@ -36,6 +35,7 @@ dulwich/diff_tree.py
 dulwich/errors.py
 dulwich/fastexport.py
 dulwich/file.py
+dulwich/greenthreads.py
 dulwich/hooks.py
 dulwich/index.py
 dulwich/log_utils.py
@@ -57,6 +57,10 @@ dulwich.egg-info/PKG-INFO
 dulwich.egg-info/SOURCES.txt
 dulwich.egg-info/dependency_links.txt
 dulwich.egg-info/top_level.txt
+dulwich/contrib/__init__.py
+dulwich/contrib/swift.py
+dulwich/contrib/test_swift.py
+dulwich/contrib/test_swift_smoke.py
 dulwich/tests/__init__.py
 dulwich/tests/test_blackbox.py
 dulwich/tests/test_client.py
@@ -65,6 +69,7 @@ dulwich/tests/test_diff_tree.py
 dulwich/tests/test_fastexport.py
 dulwich/tests/test_file.py
 dulwich/tests/test_grafts.py
+dulwich/tests/test_greenthreads.py
 dulwich/tests/test_hooks.py
 dulwich/tests/test_index.py
 dulwich/tests/test_lru_cache.py
@@ -102,6 +107,7 @@ dulwich/tests/data/commits/60/dacdc733de308bb77bb76ce0fb0f9b44c9769e
 dulwich/tests/data/indexes/index
 dulwich/tests/data/packs/pack-bc63ddad95e7321ee734ea11a7a62d314e0d7481.idx
 dulwich/tests/data/packs/pack-bc63ddad95e7321ee734ea11a7a62d314e0d7481.pack
+dulwich/tests/data/repos/.gitattributes
 dulwich/tests/data/repos/server_new.export
 dulwich/tests/data/repos/server_old.export
 dulwich/tests/data/repos/a.git/HEAD

dulwich/__init__.py

@@ -21,4 +21,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 9, 6)
+__version__ = (0, 9, 7)

dulwich/_diff_tree.c

@@ -437,7 +437,11 @@ init_diff_tree(void)
 	}
 
 	Py_DECREF(diff_tree_mod);
+#if PY_MAJOR_VERSION < 3
 	return;
+#else
+	return NULL;
+#endif
 
 error:
 	Py_XDECREF(objects_mod);
@@ -446,5 +450,9 @@ error:
 	Py_XDECREF(block_size_obj);
 	Py_XDECREF(defaultdict_cls);
 	Py_XDECREF(int_cls);
+#if PY_MAJOR_VERSION < 3
 	return;
+#else
+	return NULL;
+#endif
 }

dulwich/_objects.c

@@ -104,7 +104,7 @@ static PyObject *py_parse_tree(PyObject *self, PyObject *args, PyObject *kw)
 			Py_DECREF(name);
 			return NULL;
 		}
-		item = Py_BuildValue("(NlN)", name, mode, sha); 
+		item = Py_BuildValue("(NlN)", name, mode, sha);
 		if (item == NULL) {
 			Py_DECREF(ret);
 			Py_DECREF(sha);
@@ -254,28 +254,52 @@ init_objects(void)
 	PyObject *m, *objects_mod, *errors_mod;
 
 	m = Py_InitModule3("_objects", py_objects_methods, NULL);
-	if (m == NULL)
-		return;
-
+	if (m == NULL) {
+#if PY_MAJOR_VERSION < 3
+	  return;
+#else
+	  return NULL;
+#endif
+	}
 
 	errors_mod = PyImport_ImportModule("dulwich.errors");
-	if (errors_mod == NULL)
-		return;
+	if (errors_mod == NULL) {
+#if PY_MAJOR_VERSION < 3
+	  return;
+#else
+	  return NULL;
+#endif
+	}
 
 	object_format_exception_cls = PyObject_GetAttrString(
 		errors_mod, "ObjectFormatException");
 	Py_DECREF(errors_mod);
-	if (object_format_exception_cls == NULL)
-		return;
+	if (object_format_exception_cls == NULL) {
+#if PY_MAJOR_VERSION < 3
+	  return;
+#else
+	  return NULL;
+#endif
+	}
 
 	/* This is a circular import but should be safe since this module is
 	 * imported at at the very bottom of objects.py. */
 	objects_mod = PyImport_ImportModule("dulwich.objects");
-	if (objects_mod == NULL)
-		return;
+	if (objects_mod == NULL) {
+#if PY_MAJOR_VERSION < 3
+	  return;
+#else
+	  return NULL;
+#endif
+	}
 
 	tree_entry_cls = PyObject_GetAttrString(objects_mod, "TreeEntry");
 	Py_DECREF(objects_mod);
-	if (tree_entry_cls == NULL)
-		return;
+	if (tree_entry_cls == NULL) {
+#if PY_MAJOR_VERSION < 3
+	  return;
+#else
+	  return NULL;
+#endif
+	}
 }

dulwich/client.py

@@ -74,12 +74,13 @@ def _fileno_can_read(fileno):
     return len(select.select([fileno], [], [], 0)[0]) > 0
 
 COMMON_CAPABILITIES = ['ofs-delta', 'side-band-64k']
-FETCH_CAPABILITIES = ['thin-pack', 'multi_ack', 'multi_ack_detailed'] + COMMON_CAPABILITIES
+FETCH_CAPABILITIES = (['thin-pack', 'multi_ack', 'multi_ack_detailed'] +
+                      COMMON_CAPABILITIES)
 SEND_CAPABILITIES = ['report-status'] + COMMON_CAPABILITIES
 
 
 class ReportStatusParser(object):
-    """Handle status as reported by servers with the 'report-status' capability.
+    """Handle status as reported by servers with 'report-status' capability.
     """
 
     def __init__(self):
@@ -180,8 +181,8 @@ class GitClient(object):
         """Upload a pack to a remote repository.
 
         :param path: Repository path
-        :param generate_pack_contents: Function that can return a sequence of the
-            shas of the objects to upload.
+        :param generate_pack_contents: Function that can return a sequence of
+            the shas of the objects to upload.
         :param progress: Optional progress function
 
         :raises SendPackError: if server rejects the pack data
@@ -204,8 +205,9 @@ class GitClient(object):
             determine_wants = target.object_store.determine_wants_all
         f, commit, abort = target.object_store.add_pack()
         try:
-            result = self.fetch_pack(path, determine_wants,
-                    target.get_graph_walker(), f.write, progress)
+            result = self.fetch_pack(
+                path, determine_wants, target.get_graph_walker(), f.write,
+                progress)
         except:
             abort()
             raise
@@ -282,7 +284,8 @@ class GitClient(object):
                 if cb is not None:
                     cb(pkt)
 
-    def _handle_receive_pack_head(self, proto, capabilities, old_refs, new_refs):
+    def _handle_receive_pack_head(self, proto, capabilities, old_refs,
+                                  new_refs):
         """Handle the head of a 'git-receive-pack' request.
 
         :param proto: Protocol object to read from
@@ -301,12 +304,12 @@ class GitClient(object):
 
             if old_sha1 != new_sha1:
                 if sent_capabilities:
-                    proto.write_pkt_line('%s %s %s' % (old_sha1, new_sha1,
-                                                            refname))
+                    proto.write_pkt_line('%s %s %s' % (
+                        old_sha1, new_sha1, refname))
                 else:
                     proto.write_pkt_line(
-                      '%s %s %s\0%s' % (old_sha1, new_sha1, refname,
-                                        ' '.join(capabilities)))
+                        '%s %s %s\0%s' % (old_sha1, new_sha1, refname,
+                                          ' '.join(capabilities)))
                     sent_capabilities = True
             if new_sha1 not in have and new_sha1 != ZERO_SHA:
                 want.append(new_sha1)
@@ -323,7 +326,7 @@ class GitClient(object):
         if "side-band-64k" in capabilities:
             if progress is None:
                 progress = lambda x: None
-            channel_callbacks = { 2: progress }
+            channel_callbacks = {2: progress}
             if 'report-status' in capabilities:
                 channel_callbacks[1] = PktLineParser(
                     self._report_status_parser.handle_packet).parse
@@ -426,8 +429,8 @@ class TraditionalGitClient(GitClient):
         """Upload a pack to a remote repository.
 
         :param path: Repository path
-        :param generate_pack_contents: Function that can return a sequence of the
-            shas of the objects to upload.
+        :param generate_pack_contents: Function that can return a sequence of
+            the shas of the objects to upload.
         :param progress: Optional callback called with progress updates
 
         :raises SendPackError: if server rejects the pack data
@@ -435,67 +438,68 @@ class TraditionalGitClient(GitClient):
                                  and rejects ref updates
         """
         proto, unused_can_read = self._connect('receive-pack', path)
-        old_refs, server_capabilities = read_pkt_refs(proto)
-        negotiated_capabilities = self._send_capabilities & server_capabilities
-
-        if 'report-status' in negotiated_capabilities:
-            self._report_status_parser = ReportStatusParser()
-        report_status_parser = self._report_status_parser
-
-        try:
-            new_refs = orig_new_refs = determine_wants(dict(old_refs))
-        except:
-            proto.write_pkt_line(None)
-            raise
-
-        if not 'delete-refs' in server_capabilities:
-            # Server does not support deletions. Fail later.
-            def remove_del(pair):
-                if pair[1] == ZERO_SHA:
-                    if 'report-status' in negotiated_capabilities:
-                        report_status_parser._ref_statuses.append(
-                            'ng %s remote does not support deleting refs'
-                            % pair[1])
-                        report_status_parser._ref_status_ok = False
-                    return False
-                else:
-                    return True
-
-            new_refs = dict(
-                filter(
-                    remove_del,
-                    [(ref, sha) for ref, sha in new_refs.iteritems()]))
-
-        if new_refs is None:
-            proto.write_pkt_line(None)
-            return old_refs
+        with proto:
+            old_refs, server_capabilities = read_pkt_refs(proto)
+            negotiated_capabilities = self._send_capabilities & server_capabilities
 
-        if len(new_refs) == 0 and len(orig_new_refs):
-            # NOOP - Original new refs filtered out by policy
-            proto.write_pkt_line(None)
-            if self._report_status_parser is not None:
-                self._report_status_parser.check()
-            return old_refs
+            if 'report-status' in negotiated_capabilities:
+                self._report_status_parser = ReportStatusParser()
+            report_status_parser = self._report_status_parser
 
-        (have, want) = self._handle_receive_pack_head(proto,
-            negotiated_capabilities, old_refs, new_refs)
-        if not want and old_refs == new_refs:
-            return new_refs
-        objects = generate_pack_contents(have, want)
-        if len(objects) > 0:
-            entries, sha = write_pack_objects(proto.write_file(), objects)
-        elif len(set(new_refs.values()) - set([ZERO_SHA])) > 0:
-            # Check for valid create/update refs
-            filtered_new_refs = \
-                dict([(ref, sha) for ref, sha in new_refs.iteritems()
-                     if sha != ZERO_SHA])
-            if len(set(filtered_new_refs.iteritems()) -
-                    set(old_refs.iteritems())) > 0:
+            try:
+                new_refs = orig_new_refs = determine_wants(dict(old_refs))
+            except:
+                proto.write_pkt_line(None)
+                raise
+
+            if not 'delete-refs' in server_capabilities:
+                # Server does not support deletions. Fail later.
+                def remove_del(pair):
+                    if pair[1] == ZERO_SHA:
+                        if 'report-status' in negotiated_capabilities:
+                            report_status_parser._ref_statuses.append(
+                                'ng %s remote does not support deleting refs'
+                                % pair[1])
+                            report_status_parser._ref_status_ok = False
+                        return False
+                    else:
+                        return True
+
+                new_refs = dict(
+                    filter(
+                        remove_del,
+                        [(ref, sha) for ref, sha in new_refs.iteritems()]))
+
+            if new_refs is None:
+                proto.write_pkt_line(None)
+                return old_refs
+
+            if len(new_refs) == 0 and len(orig_new_refs):
+                # NOOP - Original new refs filtered out by policy
+                proto.write_pkt_line(None)
+                if self._report_status_parser is not None:
+                    self._report_status_parser.check()
+                return old_refs
+
+            (have, want) = self._handle_receive_pack_head(
+                proto, negotiated_capabilities, old_refs, new_refs)
+            if not want and old_refs == new_refs:
+                return new_refs
+            objects = generate_pack_contents(have, want)
+            if len(objects) > 0:
                 entries, sha = write_pack_objects(proto.write_file(), objects)
-
-        self._handle_receive_pack_tail(proto, negotiated_capabilities,
-            progress)
-        return new_refs
+            elif len(set(new_refs.values()) - set([ZERO_SHA])) > 0:
+                # Check for valid create/update refs
+                filtered_new_refs = \
+                    dict([(ref, sha) for ref, sha in new_refs.iteritems()
+                         if sha != ZERO_SHA])
+                if len(set(filtered_new_refs.iteritems()) -
+                        set(old_refs.iteritems())) > 0:
+                    entries, sha = write_pack_objects(proto.write_file(), objects)
+
+            self._handle_receive_pack_tail(
+                proto, negotiated_capabilities, progress)
+            return new_refs
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
                    progress=None):
@@ -507,46 +511,51 @@ class TraditionalGitClient(GitClient):
         :param progress: Callback for progress reports (strings)
         """
         proto, can_read = self._connect('upload-pack', path)
-        refs, server_capabilities = read_pkt_refs(proto)
-        negotiated_capabilities = self._fetch_capabilities & server_capabilities
+        with proto:
+            refs, server_capabilities = read_pkt_refs(proto)
+            negotiated_capabilities = (
+                self._fetch_capabilities & server_capabilities)
 
-        if refs is None:
-            proto.write_pkt_line(None)
+            if refs is None:
+                proto.write_pkt_line(None)
+                return refs
+
+            try:
+                wants = determine_wants(refs)
+            except:
+                proto.write_pkt_line(None)
+                raise
+            if wants is not None:
+                wants = [cid for cid in wants if cid != ZERO_SHA]
+            if not wants:
+                proto.write_pkt_line(None)
+                return refs
+            self._handle_upload_pack_head(
+                proto, negotiated_capabilities, graph_walker, wants, can_read)
+            self._handle_upload_pack_tail(
+                proto, negotiated_capabilities, graph_walker, pack_data, progress)
             return refs
 
-        try:
-            wants = determine_wants(refs)
-        except:
-            proto.write_pkt_line(None)
-            raise
-        if wants is not None:
-            wants = [cid for cid in wants if cid != ZERO_SHA]
-        if not wants:
+    def archive(self, path, committish, write_data, progress=None,
+                write_error=None):
+        proto, can_read = self._connect(b'upload-archive', path)
+        with proto:
+            proto.write_pkt_line("argument %s" % committish)
             proto.write_pkt_line(None)
-            return refs
-        self._handle_upload_pack_head(proto, negotiated_capabilities,
-            graph_walker, wants, can_read)
-        self._handle_upload_pack_tail(proto, negotiated_capabilities,
-            graph_walker, pack_data, progress)
-        return refs
-
-    def archive(self, path, committish, write_data, progress=None):
-        proto, can_read = self._connect('upload-archive', path)
-        proto.write_pkt_line("argument %s" % committish)
-        proto.write_pkt_line(None)
-        pkt = proto.read_pkt_line()
-        if pkt == "NACK\n":
-            return
-        elif pkt == "ACK\n":
-            pass
-        elif pkt.startswith("ERR "):
-            raise GitProtocolError(pkt[4:].rstrip("\n"))
-        else:
-            raise AssertionError("invalid response %r" % pkt)
-        ret = proto.read_pkt_line()
-        if ret is not None:
-            raise AssertionError("expected pkt tail")
-        self._read_side_band64k_data(proto, {1: write_data, 2: progress})
+            pkt = proto.read_pkt_line()
+            if pkt == "NACK\n":
+                return
+            elif pkt == "ACK\n":
+                pass
+            elif pkt.startswith("ERR "):
+                raise GitProtocolError(pkt[4:].rstrip("\n"))
+            else:
+                raise AssertionError("invalid response %r" % pkt)
+            ret = proto.read_pkt_line()
+            if ret is not None:
+                raise AssertionError("expected pkt tail")
+            self._read_side_band64k_data(proto, {
+                1: write_data, 2: progress, 3: write_error})
 
 
 class TCPGitClient(TraditionalGitClient):
@@ -560,8 +569,8 @@ class TCPGitClient(TraditionalGitClient):
         TraditionalGitClient.__init__(self, *args, **kwargs)
 
     def _connect(self, cmd, path):
-        sockaddrs = socket.getaddrinfo(self._host, self._port,
-            socket.AF_UNSPEC, socket.SOCK_STREAM)
+        sockaddrs = socket.getaddrinfo(
+            self._host, self._port, socket.AF_UNSPEC, socket.SOCK_STREAM)
         s = None
         err = socket.error("no address found for %s" % self._host)
         for (family, socktype, proto, canonname, sockaddr) in sockaddrs:
@@ -580,7 +589,12 @@ class TCPGitClient(TraditionalGitClient):
         rfile = s.makefile('rb', -1)
         # 0 means unbuffered
         wfile = s.makefile('wb', 0)
-        proto = Protocol(rfile.read, wfile.write,
+        def close():
+            rfile.close()
+            wfile.close()
+            s.close()
+
+        proto = Protocol(rfile.read, wfile.write, close,
                          report_activity=self._report_activity)
         if path.startswith("/~"):
             path = path[1:]
@@ -608,6 +622,8 @@ class SubprocessWrapper(object):
     def close(self):
         self.proc.stdin.close()
         self.proc.stdout.close()
+        if self.proc.stderr:
+            self.proc.stderr.close()
         self.proc.wait()
 
 
@@ -629,7 +645,7 @@ class SubprocessGitClient(TraditionalGitClient):
             subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE,
                              stderr=self._stderr))
-        return Protocol(p.read, p.write,
+        return Protocol(p.read, p.write, p.close,
                         report_activity=self._report_activity), p.can_read
 
 
@@ -652,8 +668,8 @@ class LocalGitClient(GitClient):
         """Upload a pack to a remote repository.
 
         :param path: Repository path
-        :param generate_pack_contents: Function that can return a sequence of the
-            shas of the objects to upload.
+        :param generate_pack_contents: Function that can return a sequence of
+            the shas of the objects to upload.
         :param progress: Optional progress function
 
         :raises SendPackError: if server rejects the pack data
@@ -674,7 +690,8 @@ class LocalGitClient(GitClient):
         """
         from dulwich.repo import Repo
         r = Repo(path)
-        return r.fetch(target, determine_wants=determine_wants, progress=progress)
+        return r.fetch(target, determine_wants=determine_wants,
+                       progress=progress)
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
                    progress=None):
@@ -699,6 +716,7 @@ class LocalGitClient(GitClient):
 # What Git client to use for local access
 default_local_git_client_cls = SubprocessGitClient
 
+
 class SSHVendor(object):
     """A client side SSH implementation."""
 
@@ -764,7 +782,8 @@ else:
 
             # Start
             if self.should_monitor:
-                self.monitor_thread = threading.Thread(target=self.monitor_stderr)
+                self.monitor_thread = threading.Thread(
+                    target=self.monitor_stderr)
                 self.monitor_thread.start()
 
         def monitor_stderr(self):
@@ -821,16 +840,13 @@ else:
             self.channel.close()
             self.stop_monitoring()
 
-        def __del__(self):
-            self.close()
-
     class ParamikoSSHVendor(object):
 
         def __init__(self):
             self.ssh_kwargs = {}
 
         def run_command(self, host, command, username=None, port=None,
-                progress_stderr=None):
+                        progress_stderr=None):
 
             # Paramiko needs an explicit port. None is not valid
             if port is None:
@@ -849,8 +865,8 @@ else:
             # Run commands
             channel.exec_command(*command)
 
-            return ParamikoWrapper(client, channel,
-                    progress_stderr=progress_stderr)
+            return ParamikoWrapper(
+                client, channel, progress_stderr=progress_stderr)
 
 
 # Can be overridden by users
@@ -875,7 +891,8 @@ class SSHGitClient(TraditionalGitClient):
         con = get_ssh_vendor().run_command(
             self.host, ["%s '%s'" % (self._get_cmd_path(cmd), path)],
             port=self.port, username=self.username)
-        return (Protocol(con.read, con.write, report_activity=self._report_activity),
+        return (Protocol(con.read, con.write, con.close, 
+                         report_activity=self._report_activity), 
                 con.can_read)
 
 
@@ -890,7 +907,7 @@ def default_urllib2_opener(config):
         proxy_server = None
     handlers = []
     if proxy_server is not None:
-        handlers.append(urllib2.ProxyHandler({"http" : proxy_server}))
+        handlers.append(urllib2.ProxyHandler({"http": proxy_server}))
     opener = urllib2.build_opener(*handlers)
     if config is not None:
         user_agent = config.get("http", "useragent")
@@ -904,7 +921,8 @@ def default_urllib2_opener(config):
 
 class HttpGitClient(GitClient):
 
-    def __init__(self, base_url, dumb=None, opener=None, config=None, *args, **kwargs):
+    def __init__(self, base_url, dumb=None, opener=None, config=None, *args,
+                 **kwargs):
         self.base_url = base_url.rstrip("/") + "/"
         self.dumb = dumb
         if opener is None:
@@ -931,21 +949,24 @@ class HttpGitClient(GitClient):
         assert url[-1] == "/"
         url = urlparse.urljoin(url, "info/refs")
         headers = {}
-        if self.dumb != False:
+        if self.dumb is not False:
             url += "?service=%s" % service
             headers["Content-Type"] = "application/x-%s-request" % service
         resp = self._http_request(url, headers)
-        self.dumb = (not resp.info().gettype().startswith("application/x-git-"))
-        if not self.dumb:
-            proto = Protocol(resp.read, None)
-            # The first line should mention the service
-            pkts = list(proto.read_pkt_seq())
-            if pkts != [('# service=%s\n' % service)]:
-                raise GitProtocolError(
-                    "unexpected first line %r from smart server" % pkts)
-            return read_pkt_refs(proto)
-        else:
-            return read_info_refs(resp), set()
+        try:
+            self.dumb = (not resp.info().gettype().startswith("application/x-git-"))
+            if not self.dumb:
+                proto = Protocol(resp.read, None)
+                # The first line should mention the service
+                pkts = list(proto.read_pkt_seq())
+                if pkts != [('# service=%s\n' % service)]:
+                    raise GitProtocolError(
+                        "unexpected first line %r from smart server" % pkts)
+                return read_pkt_refs(proto)
+            else:
+                return read_info_refs(resp), set()
+        finally:
+            resp.close()
 
     def _smart_request(self, service, url, data):
         assert url[-1] == "/"
@@ -962,8 +983,8 @@ class HttpGitClient(GitClient):
         """Upload a pack to a remote repository.
 
         :param path: Repository path
-        :param generate_pack_contents: Function that can return a sequence of the
-            shas of the objects to upload.
+        :param generate_pack_contents: Function that can return a sequence of
+            the shas of the objects to upload.
         :param progress: Optional progress function
 
         :raises SendPackError: if server rejects the pack data
@@ -993,11 +1014,15 @@ class HttpGitClient(GitClient):
         if len(objects) > 0:
             entries, sha = write_pack_objects(req_proto.write_file(), objects)
         resp = self._smart_request("git-receive-pack", url,
-            data=req_data.getvalue())
-        resp_proto = Protocol(resp.read, None)
-        self._handle_receive_pack_tail(resp_proto, negotiated_capabilities,
-            progress)
-        return new_refs
+                                   data=req_data.getvalue())
+        try:
+            resp_proto = Protocol(resp.read, None)
+            self._handle_receive_pack_tail(resp_proto, negotiated_capabilities,
+                progress)
+            return new_refs
+        finally:
+            resp.close()
+
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
                    progress=None):
@@ -1022,15 +1047,18 @@ class HttpGitClient(GitClient):
             raise NotImplementedError(self.send_pack)
         req_data = BytesIO()
         req_proto = Protocol(None, req_data.write)
-        self._handle_upload_pack_head(req_proto,
-            negotiated_capabilities, graph_walker, wants,
+        self._handle_upload_pack_head(
+            req_proto, negotiated_capabilities, graph_walker, wants,
             lambda: False)
-        resp = self._smart_request("git-upload-pack", url,
-            data=req_data.getvalue())
-        resp_proto = Protocol(resp.read, None)
-        self._handle_upload_pack_tail(resp_proto, negotiated_capabilities,
-            graph_walker, pack_data, progress)
-        return refs
+        resp = self._smart_request(
+            "git-upload-pack", url, data=req_data.getvalue())
+        try:
+            resp_proto = Protocol(resp.read, None)
+            self._handle_upload_pack_tail(resp_proto, negotiated_capabilities,
+                graph_walker, pack_data, progress)
+            return refs
+        finally:
+            resp.close()
 
 
 def get_transport_and_path_from_url(url, config=None, **kwargs):

dulwich/config.py

@@ -170,7 +170,7 @@ def _parse_string(value):
     value = value.strip()
     ret = []
     block = []
-    in_quotes  = False
+    in_quotes = False
     for c in value:
         if c == "\"":
             in_quotes = (not in_quotes)
@@ -290,7 +290,7 @@ class ConfigFile(ConfigDict):
                 ret._values[section][setting] = value
                 if not continuation:
                     setting = None
-            else: # continuation line
+            else:  # continuation line
                 if line.endswith("\\\n"):
                     line = line[:-2]
                     continuation = True

bin/dul-daemon → dulwich/contrib/__init__.py

@@ -1,11 +1,11 @@
-#!/usr/bin/python
-# dul-daemon - Simple git-daemon-like server
-# Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
+# __init__.py -- Contrib module for Dulwich
+# Copyright (C) 2014 Jelmer Vernooij <jelmer@samba.org>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
-# or (at your option) a later version of the License.
+# of the License or (at your option) any later version of
+# the License.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -17,7 +17,12 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-from dulwich.server import main
 
-if __name__ == '__main__':
-    main()
+def test_suite():
+    import unittest
+    names = [
+        'swift',
+        ]
+    module_names = ['dulwich.contrib.test_' + name for name in names]
+    loader = unittest.TestLoader()
+    return loader.loadTestsFromNames(module_names)

dulwich/contrib/swift.py

@@ -0,0 +1,1033 @@
+# swift.py -- Repo implementation atop OpenStack SWIFT
+# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+#
+# Author: Fabien Boucher <fabien.boucher@enovance.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) any later version of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Repo implementation atop OpenStack SWIFT."""
+
+# TODO: Refactor to share more code with dulwich/repo.py.
+# TODO(fbo): Second attempt to _send() must be notified via real log
+# TODO(fbo): More logs for operations
+
+import os
+import stat
+import zlib
+import tempfile
+import posixpath
+
+from urlparse import urlparse
+from cStringIO import StringIO
+from ConfigParser import ConfigParser
+from geventhttpclient import HTTPClient
+
+from dulwich.greenthreads import (
+    GreenThreadsMissingObjectFinder,
+    GreenThreadsObjectStoreIterator,
+    )
+
+from dulwich.lru_cache import LRUSizeCache
+from dulwich.objects import (
+    Blob,
+    Commit,
+    Tree,
+    Tag,
+    S_ISGITLINK,
+    )
+from dulwich.object_store import (
+    PackBasedObjectStore,
+    PACKDIR,
+    INFODIR,
+    )
+from dulwich.pack import (
+    PackData,
+    Pack,
+    PackIndexer,
+    PackStreamCopier,
+    write_pack_header,
+    compute_file_sha,
+    iter_sha1,
+    write_pack_index_v2,
+    load_pack_index_file,
+    read_pack_header,
+    _compute_object_size,
+    unpack_object,
+    write_pack_object,
+    )
+from dulwich.protocol import TCP_GIT_PORT
+from dulwich.refs import (
+    InfoRefsContainer,
+    read_info_refs,
+    write_info_refs,
+    )
+from dulwich.repo import (
+    BaseRepo,
+    OBJECTDIR,
+    )
+from dulwich.server import (
+    Backend,
+    TCPGitServer,
+    )
+
+try:
+    from simplejson import loads as json_loads
+    from simplejson import dumps as json_dumps
+except ImportError:
+    from json import loads as json_loads
+    from json import dumps as json_dumps
+
+import sys
+
+
+"""
+# Configuration file sample
+[swift]
+# Authentication URL (Keystone or Swift)
+auth_url = http://127.0.0.1:5000/v2.0
+# Authentication version to use
+auth_ver = 2
+# The tenant and username separated by a semicolon
+username = admin;admin
+# The user password
+password = pass
+# The Object storage region to use (auth v2) (Default RegionOne)
+region_name = RegionOne
+# The Object storage endpoint URL to use (auth v2) (Default internalURL)
+endpoint_type = internalURL
+# Concurrency to use for parallel tasks (Default 10)
+concurrency = 10
+# Size of the HTTP pool (Default 10)
+http_pool_length = 10
+# Timeout delay for HTTP connections (Default 20)
+http_timeout = 20
+# Chunk size to read from pack (Bytes) (Default 12228)
+chunk_length = 12228
+# Cache size (MBytes) (Default 20)
+cache_length = 20
+"""
+
+
+class PackInfoObjectStoreIterator(GreenThreadsObjectStoreIterator):
+
+    def __len__(self):
+        while len(self.finder.objects_to_send):
+            for _ in xrange(0, len(self.finder.objects_to_send)):
+                sha = self.finder.next()
+                self._shas.append(sha)
+        return len(self._shas)
+
+
+class PackInfoMissingObjectFinder(GreenThreadsMissingObjectFinder):
+
+    def next(self):
+        while True:
+            if not self.objects_to_send:
+                return None
+            (sha, name, leaf) = self.objects_to_send.pop()
+            if sha not in self.sha_done:
+                break
+        if not leaf:
+            info = self.object_store.pack_info_get(sha)
+            if info[0] == Commit.type_num:
+                self.add_todo([(info[2], "", False)])
+            elif info[0] == Tree.type_num:
+                self.add_todo([tuple(i) for i in info[1]])
+            elif info[0] == Tag.type_num:
+                self.add_todo([(info[1], None, False)])
+            if sha in self._tagged:
+                self.add_todo([(self._tagged[sha], None, True)])
+        self.sha_done.add(sha)
+        self.progress("counting objects: %d\r" % len(self.sha_done))
+        return (sha, name)
+
+
+def load_conf(path=None, file=None):
+    """Load configuration in global var CONF
+
+    :param path: The path to the configuration file
+    :param file: If provided read instead the file like object
+    """
+    conf = ConfigParser(allow_no_value=True)
+    if file:
+        conf.readfp(file)
+        return conf
+    confpath = None
+    if not path:
+        try:
+            confpath = os.environ['DULWICH_SWIFT_CFG']
+        except KeyError:
+            raise Exception("You need to specify a configuration file")
+    else:
+        confpath = path
+    if not os.path.isfile(confpath):
+        raise Exception("Unable to read configuration file %s" % confpath)
+    conf.read(confpath)
+    return conf
+
+
+def swift_load_pack_index(scon, filename):
+    """Read a pack index file from Swift
+
+    :param scon: a `SwiftConnector` instance
+    :param filename: Path to the index file objectise
+    :return: a `PackIndexer` instance
+    """
+    f = scon.get_object(filename)
+    try:
+        return load_pack_index_file(filename, f)
+    finally:
+        f.close()
+
+
+def pack_info_create(pack_data, pack_index):
+    pack = Pack.from_objects(pack_data, pack_index)
+    info = {}
+    for obj in pack.iterobjects():
+        # Commit
+        if obj.type_num == Commit.type_num:
+            info[obj.id] = (obj.type_num, obj.parents, obj.tree)
+        # Tree
+        elif obj.type_num == Tree.type_num:
+            shas = [(s, n, not stat.S_ISDIR(m)) for
+                    n, m, s in obj.iteritems() if not S_ISGITLINK(m)]
+            info[obj.id] = (obj.type_num, shas)
+        # Blob
+        elif obj.type_num == Blob.type_num:
+            info[obj.id] = None
+        # Tag
+        elif obj.type_num == Tag.type_num:
+            info[obj.id] = (obj.type_num, obj._object_sha)
+    return zlib.compress(json_dumps(info))
+
+
+def load_pack_info(filename, scon=None, file=None):
+    if not file:
+        f = scon.get_object(filename)
+    else:
+        f = file
+    if not f:
+        return None
+    try:
+        return json_loads(zlib.decompress(f.read()))
+    finally:
+        f.close()
+
+
+class SwiftException(Exception):
+    pass
+
+
+class SwiftConnector(object):
+    """A Connector to swift that manage authentication and errors catching
+    """
+
+    def __init__(self, root, conf):
+        """ Initialize a SwiftConnector
+
+        :param root: The swift container that will act as Git bare repository
+        :param conf: A ConfigParser Object
+        """
+        self.conf = conf
+        self.auth_ver = self.conf.get("swift", "auth_ver")
+        if self.auth_ver not in ["1", "2"]:
+            raise NotImplementedError(
+                "Wrong authentication version use either 1 or 2")
+        self.auth_url = self.conf.get("swift", "auth_url")
+        self.user = self.conf.get("swift", "username")
+        self.password = self.conf.get("swift", "password")
+        self.concurrency = self.conf.getint('swift', 'concurrency') or 10
+        self.http_timeout = self.conf.getint('swift', 'http_timeout') or 20
+        self.http_pool_length = \
+            self.conf.getint('swift', 'http_pool_length') or 10
+        self.region_name = self.conf.get("swift", "region_name") or "RegionOne"
+        self.endpoint_type = \
+            self.conf.get("swift", "endpoint_type") or "internalURL"
+        self.cache_length = self.conf.getint("swift", "cache_length") or 20
+        self.chunk_length = self.conf.getint("swift", "chunk_length") or 12228
+        self.root = root
+        block_size = 1024 * 12  # 12KB
+        if self.auth_ver == "1":
+            self.storage_url, self.token = self.swift_auth_v1()
+        else:
+            self.storage_url, self.token = self.swift_auth_v2()
+
+        token_header = {'X-Auth-Token': str(self.token)}
+        self.httpclient = \
+            HTTPClient.from_url(str(self.storage_url),
+                                concurrency=self.http_pool_length,
+                                block_size=block_size,
+                                connection_timeout=self.http_timeout,
+                                network_timeout=self.http_timeout,
+                                headers=token_header)
+        self.base_path = str(posixpath.join(urlparse(self.storage_url).path,
+                             self.root))
+
+    def swift_auth_v1(self):
+        self.user = self.user.replace(";", ":")
+        auth_httpclient = HTTPClient.from_url(
+            self.auth_url,
+            connection_timeout=self.http_timeout,
+            network_timeout=self.http_timeout,
+            )
+        headers = {'X-Auth-User': self.user,
+                   'X-Auth-Key': self.password}
+        path = urlparse(self.auth_url).path
+
+        ret = auth_httpclient.request('GET', path, headers=headers)
+
+        # Should do something with redirections (301 in my case)
+
+        if ret.status_code < 200 or ret.status_code >= 300:
+            raise SwiftException('AUTH v1.0 request failed on ' +
+                                 '%s with error code %s (%s)'
+                                 % (str(auth_httpclient.get_base_url()) +
+                                    path, ret.status_code,
+                                    str(ret.items())))
+        storage_url = ret['X-Storage-Url']
+        token = ret['X-Auth-Token']
+        return storage_url, token
+
+    def swift_auth_v2(self):
+        self.tenant, self.user = self.user.split(';')
+        auth_dict = {}
+        auth_dict['auth'] = {'passwordCredentials':
+                             {
+                                 'username': self.user,
+                                 'password': self.password,
+                             },
+                             'tenantName': self.tenant}
+        auth_json = json_dumps(auth_dict)
+        headers = {'Content-Type': 'application/json'}
+        auth_httpclient = HTTPClient.from_url(
+            self.auth_url,
+            connection_timeout=self.http_timeout,
+            network_timeout=self.http_timeout,
+            )
+        path = urlparse(self.auth_url).path
+        if not path.endswith('tokens'):
+            path = posixpath.join(path, 'tokens')
+        ret = auth_httpclient.request('POST', path,
+                                      body=auth_json,
+                                      headers=headers)
+
+        if ret.status_code < 200 or ret.status_code >= 300:
+            raise SwiftException('AUTH v2.0 request failed on ' +
+                                 '%s with error code %s (%s)'
+                                 % (str(auth_httpclient.get_base_url()) +
+                                    path, ret.status_code,
+                                    str(ret.items())))
+        auth_ret_json = json_loads(ret.read())
+        token = auth_ret_json['access']['token']['id']
+        catalogs = auth_ret_json['access']['serviceCatalog']
+        object_store = [o_store for o_store in catalogs if
+                        o_store['type'] == 'object-store'][0]
+        endpoints = object_store['endpoints']
+        endpoint = [endp for endp in endpoints if
+                    endp["region"] == self.region_name][0]
+        return endpoint[self.endpoint_type], token
+
+    def test_root_exists(self):
+        """Check that Swift container exist
+
+        :return: True if exist or None it not
+        """
+        ret = self.httpclient.request('HEAD', self.base_path)
+        if ret.status_code == 404:
+            return None
+        if ret.status_code < 200 or ret.status_code > 300:
+            raise SwiftException('HEAD request failed with error code %s'
+                                 % ret.status_code)
+        return True
+
+    def create_root(self):
+        """Create the Swift container
+
+        :raise: `SwiftException` if unable to create
+        """
+        if not self.test_root_exists():
+            ret = self.httpclient.request('PUT', self.base_path)
+            if ret.status_code < 200 or ret.status_code > 300:
+                raise SwiftException('PUT request failed with error code %s'
+                                     % ret.status_code)
+
+    def get_container_objects(self):
+        """Retrieve objects list in a container
+
+        :return: A list of dict that describe objects
+                 or None if container does not exist
+        """
+        qs = '?format=json'
+        path = self.base_path + qs
+        ret = self.httpclient.request('GET', path)
+        if ret.status_code == 404:
+            return None
+        if ret.status_code < 200 or ret.status_code > 300:
+            raise SwiftException('GET request failed with error code %s'
+                                 % ret.status_code)
+        content = ret.read()
+        return json_loads(content)
+
+    def get_object_stat(self, name):
+        """Retrieve object stat
+
+        :param name: The object name
+        :return: A dict that describe the object
+                 or None if object does not exist
+        """
+        path = self.base_path + '/' + name
+        ret = self.httpclient.request('HEAD', path)
+        if ret.status_code == 404:
+            return None
+        if ret.status_code < 200 or ret.status_code > 300:
+            raise SwiftException('HEAD request failed with error code %s'
+                                 % ret.status_code)
+        resp_headers = {}
+        for header, value in ret.iteritems():
+            resp_headers[header.lower()] = value
+        return resp_headers
+
+    def put_object(self, name, content):
+        """Put an object
+
+        :param name: The object name
+        :param content: A file object
+        :raise: `SwiftException` if unable to create
+        """
+        content.seek(0)
+        data = content.read()
+        path = self.base_path + '/' + name
+        headers = {'Content-Length': str(len(data))}
+
+        def _send():
+            ret = self.httpclient.request('PUT', path,
+                                          body=data,
+                                          headers=headers)
+            return ret
+
+        try:
+            # Sometime got Broken Pipe - Dirty workaround
+            ret = _send()
+        except Exception:
+            # Second attempt work
+            ret = _send()
+
+        if ret.status_code < 200 or ret.status_code > 300:
+            raise SwiftException('PUT request failed with error code %s'
+                                 % ret.status_code)
+
+    def get_object(self, name, range=None):
+        """Retrieve an object
+
+        :param name: The object name
+        :param range: A string range like "0-10" to
+                      retrieve specified bytes in object content
+        :return: A file like instance
+                 or bytestring if range is specified
+        """
+        headers = {}
+        if range:
+            headers['Range'] = 'bytes=%s' % range
+        path = self.base_path + '/' + name
+        ret = self.httpclient.request('GET', path, headers=headers)
+        if ret.status_code == 404:
+            return None
+        if ret.status_code < 200 or ret.status_code > 300:
+            raise SwiftException('GET request failed with error code %s'
+                                 % ret.status_code)
+        content = ret.read()
+
+        if range:
+            return content
+        return StringIO(content)
+
+    def del_object(self, name):
+        """Delete an object
+
+        :param name: The object name
+        :raise: `SwiftException` if unable to delete
+        """
+        path = self.base_path + '/' + name
+        ret = self.httpclient.request('DELETE', path)
+        if ret.status_code < 200 or ret.status_code > 300:
+            raise SwiftException('DELETE request failed with error code %s'
+                                 % ret.status_code)
+
+    def del_root(self):
+        """Delete the root container by removing container content
+
+        :raise: `SwiftException` if unable to delete
+        """
+        for obj in self.get_container_objects():
+            self.del_object(obj['name'])
+        ret = self.httpclient.request('DELETE', self.base_path)
+        if ret.status_code < 200 or ret.status_code > 300:
+            raise SwiftException('DELETE request failed with error code %s'
+                                 % ret.status_code)
+
+
+class SwiftPackReader(object):
+    """A SwiftPackReader that mimic read and sync method
+
+    The reader allows to read a specified amount of bytes from
+    a given offset of a Swift object. A read offset is kept internaly.
+    The reader will read from Swift a specified amount of data to complete
+    its internal buffer. chunk_length specifiy the amount of data
+    to read from Swift.
+    """
+
+    def __init__(self, scon, filename, pack_length):
+        """Initialize a SwiftPackReader
+
+        :param scon: a `SwiftConnector` instance
+        :param filename: the pack filename
+        :param pack_length: The size of the pack object
+        """
+        self.scon = scon
+        self.filename = filename
+        self.pack_length = pack_length
+        self.offset = 0
+        self.base_offset = 0
+        self.buff = ''
+        self.buff_length = self.scon.chunk_length
+
+    def _read(self, more=False):
+        if more:
+            self.buff_length = self.buff_length * 2
+        l = self.base_offset
+        r = min(self.base_offset + self.buff_length, self.pack_length)
+        ret = self.scon.get_object(self.filename, range="%s-%s" % (l, r))
+        self.buff = ret
+
+    def read(self, length):
+        """Read a specified amount of Bytes form the pack object
+
+        :param length: amount of bytes to read
+        :return: bytestring
+        """
+        end = self.offset+length
+        if self.base_offset + end > self.pack_length:
+            data = self.buff[self.offset:]
+            self.offset = end
+            return "".join(data)
+        try:
+            self.buff[end]
+        except IndexError:
+            # Need to read more from swift
+            self._read(more=True)
+            return self.read(length)
+        data = self.buff[self.offset:end]
+        self.offset = end
+        return "".join(data)
+
+    def seek(self, offset):
+        """Seek to a specified offset
+
+        :param offset: the offset to seek to
+        """
+        self.base_offset = offset
+        self._read()
+        self.offset = 0
+
+    def read_checksum(self):
+        """Read the checksum from the pack
+
+        :return: the checksum bytestring
+        """
+        return self.scon.get_object(self.filename, range="-20")
+
+
+class SwiftPackData(PackData):
+    """The data contained in a packfile.
+
+    We use the SwiftPackReader to read bytes from packs stored in Swift
+    using the Range header feature of Swift.
+    """
+
+    def __init__(self, scon, filename):
+        """ Initialize a SwiftPackReader
+
+        :param scon: a `SwiftConnector` instance
+        :param filename: the pack filename
+        """
+        self.scon = scon
+        self._filename = filename
+        self._header_size = 12
+        headers = self.scon.get_object_stat(self._filename)
+        self.pack_length = int(headers['content-length'])
+        pack_reader = SwiftPackReader(self.scon, self._filename,
+                                      self.pack_length)
+        (version, self._num_objects) = read_pack_header(pack_reader.read)
+        self._offset_cache = LRUSizeCache(1024*1024*self.scon.cache_length,
+                                          compute_size=_compute_object_size)
+        self.pack = None
+
+    def get_object_at(self, offset):
+        if offset in self._offset_cache:
+            return self._offset_cache[offset]
+        assert isinstance(offset, long) or isinstance(offset, int),\
+            'offset was %r' % offset
+        assert offset >= self._header_size
+        pack_reader = SwiftPackReader(self.scon, self._filename,
+                                      self.pack_length)
+        pack_reader.seek(offset)
+        unpacked, _ = unpack_object(pack_reader.read)
+        return (unpacked.pack_type_num, unpacked._obj())
+
+    def get_stored_checksum(self):
+        pack_reader = SwiftPackReader(self.scon, self._filename,
+                                      self.pack_length)
+        return pack_reader.read_checksum()
+
+    def close(self):
+        pass
+
+
+class SwiftPack(Pack):
+    """A Git pack object.
+
+    Same implementation as pack.Pack except that _idx_load and
+    _data_load are bounded to Swift version of load_pack_index and
+    PackData.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.scon = kwargs['scon']
+        del kwargs['scon']
+        super(SwiftPack, self).__init__(*args, **kwargs)
+        self._pack_info_path = self._basename + '.info'
+        self._pack_info = None
+        self._pack_info_load = lambda: load_pack_info(self._pack_info_path,
+                                                      self.scon)
+        self._idx_load = lambda: swift_load_pack_index(self.scon,
+                                                       self._idx_path)
+        self._data_load = lambda: SwiftPackData(self.scon, self._data_path)
+
+    @property
+    def pack_info(self):
+        """The pack data object being used."""
+        if self._pack_info is None:
+            self._pack_info = self._pack_info_load()
+        return self._pack_info
+
+
+class SwiftObjectStore(PackBasedObjectStore):
+    """A Swift Object Store
+
+    Allow to manage a bare Git repository from Openstack Swift.
+    This object store only supports pack files and not loose objects.
+    """
+    def __init__(self, scon):
+        """Open a Swift object store.
+
+        :param scon: A `SwiftConnector` instance
+        """
+        super(SwiftObjectStore, self).__init__()
+        self.scon = scon
+        self.root = self.scon.root
+        self.pack_dir = posixpath.join(OBJECTDIR, PACKDIR)
+        self._alternates = None
+
+    @property
+    def packs(self):
+        """List with pack objects."""
+        if not self._pack_cache:
+            self._update_pack_cache()
+        return self._pack_cache.values()
+
+    def _update_pack_cache(self):
+        for pack in self._load_packs():
+            self._pack_cache[pack._basename] = pack
+
+    def _iter_loose_objects(self):
+        """Loose objects are not supported by this repository
+        """
+        return []
+
+    def iter_shas(self, finder):
+        """An iterator over pack's ObjectStore.
+
+        :return: a `ObjectStoreIterator` or `GreenThreadsObjectStoreIterator`
+                 instance if gevent is enabled
+        """
+        shas = iter(finder.next, None)
+        return PackInfoObjectStoreIterator(
+            self, shas, finder, self.scon.concurrency)
+
+    def find_missing_objects(self, *args, **kwargs):
+        kwargs['concurrency'] = self.scon.concurrency
+        return PackInfoMissingObjectFinder(self, *args, **kwargs)
+
+    def _load_packs(self):
+        """Load all packs from Swift
+
+        :return: a list of `SwiftPack` instances
+        """
+        objects = self.scon.get_container_objects()
+        pack_files = [o['name'].replace(".pack", "")
+                      for o in objects if o['name'].endswith(".pack")]
+        return [SwiftPack(pack, scon=self.scon) for pack in pack_files]
+
+    def pack_info_get(self, sha):
+        for pack in self.packs:
+            if sha in pack:
+                return pack.pack_info[sha]
+
+    def _collect_ancestors(self, heads, common=set()):
+        def _find_parents(commit):
+            for pack in self.packs:
+                if commit in pack:
+                    try:
+                        parents = pack.pack_info[commit][1]
+                    except KeyError:
+                        # Seems to have no parents
+                        return []
+                    return parents
+
+        bases = set()
+        commits = set()
+        queue = []
+        queue.extend(heads)
+        while queue:
+            e = queue.pop(0)
+            if e in common:
+                bases.add(e)
+            elif e not in commits:
+                commits.add(e)
+                parents = _find_parents(e)
+                queue.extend(parents)
+        return (commits, bases)
+
+    def add_pack(self):
+        """Add a new pack to this object store.
+
+        :return: Fileobject to write to and a commit function to
+            call when the pack is finished.
+        """
+        f = StringIO()
+
+        def commit():
+            f.seek(0)
+            pack = PackData(file=f, filename="")
+            entries = pack.sorted_entries()
+            if len(entries):
+                basename = posixpath.join(self.pack_dir,
+                                          "pack-%s" %
+                                          iter_sha1(entry[0] for
+                                                    entry in entries))
+                index = StringIO()
+                write_pack_index_v2(index, entries, pack.get_stored_checksum())
+                self.scon.put_object(basename + ".pack", f)
+                f.close()
+                self.scon.put_object(basename + ".idx", index)
+                index.close()
+                final_pack = SwiftPack(basename, scon=self.scon)
+                final_pack.check_length_and_checksum()
+                self._add_known_pack(basename, final_pack)
+                return final_pack
+            else:
+                return None
+
+        def abort():
+            pass
+        return f, commit, abort
+
+    def add_object(self, obj):
+        self.add_objects([(obj, None), ])
+
+    def _pack_cache_stale(self):
+        return False
+
+    def _get_loose_object(self, sha):
+        return None
+
+    def add_thin_pack(self, read_all, read_some):
+        """Read a thin pack
+
+        Read it from a stream and complete it in a temporary file.
+        Then the pack and the corresponding index file are uploaded to Swift.
+        """
+        fd, path = tempfile.mkstemp(prefix='tmp_pack_')
+        f = os.fdopen(fd, 'w+b')
+        try:
+            indexer = PackIndexer(f, resolve_ext_ref=self.get_raw)
+            copier = PackStreamCopier(read_all, read_some, f,
+                                      delta_iter=indexer)
+            copier.verify()
+            return self._complete_thin_pack(f, path, copier, indexer)
+        finally:
+            f.close()
+            os.unlink(path)
+
+    def _complete_thin_pack(self, f, path, copier, indexer):
+        entries = list(indexer)
+
+        # Update the header with the new number of objects.
+        f.seek(0)
+        write_pack_header(f, len(entries) + len(indexer.ext_refs()))
+
+        # Must flush before reading (http://bugs.python.org/issue3207)
+        f.flush()
+
+        # Rescan the rest of the pack, computing the SHA with the new header.
+        new_sha = compute_file_sha(f, end_ofs=-20)
+
+        # Must reposition before writing (http://bugs.python.org/issue3207)
+        f.seek(0, os.SEEK_CUR)
+
+        # Complete the pack.
+        for ext_sha in indexer.ext_refs():
+            assert len(ext_sha) == 20
+            type_num, data = self.get_raw(ext_sha)
+            offset = f.tell()
+            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
+            entries.append((ext_sha, offset, crc32))
+        pack_sha = new_sha.digest()
+        f.write(pack_sha)
+        f.flush()
+
+        # Move the pack in.
+        entries.sort()
+        pack_base_name = posixpath.join(
+            self.pack_dir, 'pack-' + iter_sha1(e[0] for e in entries))
+        self.scon.put_object(pack_base_name + '.pack', f)
+
+        # Write the index.
+        filename = pack_base_name + '.idx'
+        index_file = StringIO()
+        write_pack_index_v2(index_file, entries, pack_sha)
+        self.scon.put_object(filename, index_file)
+
+        # Write pack info.
+        f.seek(0)
+        pack_data = PackData(filename="", file=f)
+        index_file.seek(0)
+        pack_index = load_pack_index_file('', index_file)
+        serialized_pack_info = pack_info_create(pack_data, pack_index)
+        f.close()
+        index_file.close()
+        pack_info_file = StringIO(serialized_pack_info)
+        filename = pack_base_name + '.info'
+        self.scon.put_object(filename, pack_info_file)
+        pack_info_file.close()
+
+        # Add the pack to the store and return it.
+        final_pack = SwiftPack(pack_base_name, scon=self.scon)
+        final_pack.check_length_and_checksum()
+        self._add_known_pack(pack_base_name, final_pack)
+        return final_pack
+
+
+class SwiftInfoRefsContainer(InfoRefsContainer):
+    """Manage references in info/refs object.
+    """
+
+    def __init__(self, scon, store):
+        self.scon = scon
+        self.filename = 'info/refs'
+        self.store = store
+        f = self.scon.get_object(self.filename)
+        if not f:
+            f = StringIO('')
+        super(SwiftInfoRefsContainer, self).__init__(f)
+
+    def _load_check_ref(self, name, old_ref):
+        self._check_refname(name)
+        f = self.scon.get_object(self.filename)
+        if not f:
+            return {}
+        refs = read_info_refs(f)
+        if old_ref is not None:
+            if refs[name] != old_ref:
+                return False
+        return refs
+
+    def _write_refs(self, refs):
+        f = StringIO()
+        f.writelines(write_info_refs(refs, self.store))
+        self.scon.put_object(self.filename, f)
+
+    def set_if_equals(self, name, old_ref, new_ref):
+        """Set a refname to new_ref only if it currently equals old_ref.
+        """
+        if name == 'HEAD':
+            return True
+        refs = self._load_check_ref(name, old_ref)
+        if not isinstance(refs, dict):
+            return False
+        refs[name] = new_ref
+        self._write_refs(refs)
+        self._refs[name] = new_ref
+        return True
+
+    def remove_if_equals(self, name, old_ref):
+        """Remove a refname only if it currently equals old_ref.
+        """
+        if name == 'HEAD':
+            return True
+        refs = self._load_check_ref(name, old_ref)
+        if not isinstance(refs, dict):
+            return False
+        del refs[name]
+        self._write_refs(refs)
+        del self._refs[name]
+        return True
+
+    def allkeys(self):
+        try:
+            self._refs['HEAD'] = self._refs['refs/heads/master']
+        except KeyError:
+            pass
+        return self._refs.keys()
+
+
+class SwiftRepo(BaseRepo):
+
+    def __init__(self, root, conf):
+        """Init a Git bare Repository on top of a Swift container.
+
+        References are managed in info/refs objects by
+        `SwiftInfoRefsContainer`. The root attribute is the Swift
+        container that contain the Git bare repository.
+
+        :param root: The container which contains the bare repo
+        :param conf: A ConfigParser object
+        """
+        self.root = root.lstrip('/')
+        self.conf = conf
+        self.scon = SwiftConnector(self.root, self.conf)
+        objects = self.scon.get_container_objects()
+        if not objects:
+            raise Exception('There is not any GIT repo here : %s' % self.root)
+        objects = [o['name'].split('/')[0] for o in objects]
+        if OBJECTDIR not in objects:
+            raise Exception('This repository (%s) is not bare.' % self.root)
+        self.bare = True
+        self._controldir = self.root
+        object_store = SwiftObjectStore(self.scon)
+        refs = SwiftInfoRefsContainer(self.scon, object_store)
+        BaseRepo.__init__(self, object_store, refs)
+
+    def _put_named_file(self, filename, contents):
+        """Put an object in a Swift container
+
+        :param filename: the path to the object to put on Swift
+        :param contents: the content as bytestring
+        """
+        f = StringIO()
+        f.write(contents)
+        self.scon.put_object(filename, f)
+        f.close()
+
+    @classmethod
+    def init_bare(cls, scon, conf):
+        """Create a new bare repository.
+
+        :param scon: a `SwiftConnector` instance
+        :param conf: a ConfigParser object
+        :return: a `SwiftRepo` instance
+        """
+        scon.create_root()
+        for obj in [posixpath.join(OBJECTDIR, PACKDIR),
+                    posixpath.join(INFODIR, 'refs')]:
+            scon.put_object(obj, StringIO(''))
+        ret = cls(scon.root, conf)
+        ret._init_files(True)
+        return ret
+
+
+class SwiftSystemBackend(Backend):
+
+    def __init__(self, logger, conf):
+        self.conf = conf
+        self.logger = logger
+
+    def open_repository(self, path):
+        self.logger.info('opening repository at %s', path)
+        return SwiftRepo(path, self.conf)
+
+
+def cmd_daemon(args):
+    """Entry point for starting a TCP git server."""
+    import optparse
+    parser = optparse.OptionParser()
+    parser.add_option("-l", "--listen_address", dest="listen_address",
+                      default="127.0.0.1",
+                      help="Binding IP address.")
+    parser.add_option("-p", "--port", dest="port", type=int,
+                      default=TCP_GIT_PORT,
+                      help="Binding TCP port.")
+    parser.add_option("-c", "--swift_config", dest="swift_config",
+                      default="",
+                      help="Path to the configuration file for Swift backend.")
+    options, args = parser.parse_args(args)
+
+    try:
+        import gevent
+        import geventhttpclient
+    except ImportError:
+        print("gevent and geventhttpclient libraries are mandatory "
+              " for use the Swift backend.")
+        sys.exit(1)
+    import gevent.monkey
+    gevent.monkey.patch_socket()
+    from dulwich.swift import load_conf
+    from dulwich import log_utils
+    logger = log_utils.getLogger(__name__)
+    conf = load_conf(options.swift_config)
+    backend = SwiftSystemBackend(logger, conf)
+
+    log_utils.default_logging_config()
+    server = TCPGitServer(backend, options.listen_address,
+                          port=options.port)
+    server.serve_forever()
+
+
+def cmd_init(args):
+    import optparse
+    parser = optparse.OptionParser()
+    parser.add_option("-c", "--swift_config", dest="swift_config",
+                      default="",
+                      help="Path to the configuration file for Swift backend.")
+    options, args = parser.parse_args(args)
+
+    conf = load_conf(options.swift_config)
+    if args == []:
+        parser.error("missing repository name")
+    repo = args[0]
+    scon = SwiftConnector(repo, conf)
+    SwiftRepo.init_bare(scon, conf)
+
+
+def main(argv=sys.argv):
+    commands = {
+        "init": cmd_init,
+        "daemon": cmd_daemon,
+    }
+
+    if len(sys.argv) < 2:
+        print("Usage: %s <%s> [OPTIONS...]" % (sys.argv[0], "|".join(commands.keys())))
+        sys.exit(1)
+
+    cmd = sys.argv[1]
+    if not cmd in commands:
+        print("No such subcommand: %s" % cmd)
+        sys.exit(1)
+    commands[cmd](sys.argv[2:])
+
+if __name__ == '__main__':
+    main()

dulwich/contrib/test_swift.py

@@ -0,0 +1,641 @@
+# test_swift.py -- Unittests for the Swift backend.
+# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+#
+# Author: Fabien Boucher <fabien.boucher@enovance.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) any later version of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Tests for dulwich.contrib.swift."""
+
+import posixpath
+
+from time import time
+from cStringIO import StringIO
+from unittest import skipIf
+
+from dulwich.tests import (
+    TestCase,
+    )
+from dulwich.tests.test_object_store import (
+    ObjectStoreTests,
+    )
+from dulwich.tests.utils import (
+    build_pack,
+    )
+from dulwich.objects import (
+    Blob,
+    Commit,
+    Tree,
+    Tag,
+    parse_timezone,
+    )
+from dulwich.pack import (
+    REF_DELTA,
+    write_pack_index_v2,
+    PackData,
+    load_pack_index_file,
+    )
+
+try:
+    from simplejson import dumps as json_dumps
+except ImportError:
+    from json import dumps as json_dumps
+
+missing_libs = []
+
+try:
+    import gevent
+except ImportError:
+    missing_libs.append("gevent")
+
+try:
+    import geventhttpclient
+except ImportError:
+    missing_libs.append("geventhttpclient")
+
+try:
+    from mock import patch
+except ImportError:
+    missing_libs.append("mock")
+
+skipmsg = "Required libraries are not installed (%r)" % missing_libs
+
+if not missing_libs:
+    from dulwich.contrib import swift
+
+config_file = """[swift]
+auth_url = http://127.0.0.1:8080/auth/%(version_str)s
+auth_ver = %(version_int)s
+username = test;tester
+password = testing
+region_name = %(region_name)s
+endpoint_type = %(endpoint_type)s
+concurrency = %(concurrency)s
+chunk_length = %(chunk_length)s
+cache_length = %(cache_length)s
+http_pool_length = %(http_pool_length)s
+http_timeout = %(http_timeout)s
+"""
+
+def_config_file = {'version_str': 'v1.0',
+                   'version_int': 1,
+                   'concurrency': 1,
+                   'chunk_length': 12228,
+                   'cache_length': 1,
+                   'region_name': 'test',
+                   'endpoint_type': 'internalURL',
+                   'http_pool_length': 1,
+                   'http_timeout': 1}
+
+
+def create_swift_connector(store={}):
+    return lambda root, conf: FakeSwiftConnector(root,
+                                                 conf=conf,
+                                                 store=store)
+
+
+class Response(object):
+
+    def __init__(self, headers={}, status=200, content=None):
+        self.headers = headers
+        self.status_code = status
+        self.content = content
+
+    def __getitem__(self, key):
+        return self.headers[key]
+
+    def items(self):
+        return self.headers
+
+    def iteritems(self):
+        for k, v in self.headers.iteritems():
+            yield k, v
+
+    def read(self):
+        return self.content
+
+
+def fake_auth_request_v1(*args, **kwargs):
+    ret = Response({'X-Storage-Url':
+                    'http://127.0.0.1:8080/v1.0/AUTH_fakeuser',
+                    'X-Auth-Token': '12' * 10},
+                   200)
+    return ret
+
+
+def fake_auth_request_v1_error(*args, **kwargs):
+    ret = Response({},
+                   401)
+    return ret
+
+
+def fake_auth_request_v2(*args, **kwargs):
+    s_url = 'http://127.0.0.1:8080/v1.0/AUTH_fakeuser'
+    resp = {'access': {'token': {'id': '12' * 10},
+                       'serviceCatalog':
+                       [
+                           {'type': 'object-store',
+                            'endpoints': [{'region': 'test',
+                                          'internalURL': s_url,
+                                           },
+                                          ]
+                            },
+                       ]
+                       }
+            }
+    ret = Response(status=200, content=json_dumps(resp))
+    return ret
+
+
+def create_commit(data, marker='Default', blob=None):
+    if not blob:
+        blob = Blob.from_string('The blob content %s' % marker)
+    tree = Tree()
+    tree.add("thefile_%s" % marker, 0o100644, blob.id)
+    cmt = Commit()
+    if data:
+        assert isinstance(data[-1], Commit)
+        cmt.parents = [data[-1].id]
+    cmt.tree = tree.id
+    author = "John Doe %s <john@doe.net>" % marker
+    cmt.author = cmt.committer = author
+    tz = parse_timezone('-0200')[0]
+    cmt.commit_time = cmt.author_time = int(time())
+    cmt.commit_timezone = cmt.author_timezone = tz
+    cmt.encoding = "UTF-8"
+    cmt.message = "The commit message %s" % marker
+    tag = Tag()
+    tag.tagger = "john@doe.net"
+    tag.message = "Annotated tag"
+    tag.tag_timezone = parse_timezone('-0200')[0]
+    tag.tag_time = cmt.author_time
+    tag.object = (Commit, cmt.id)
+    tag.name = "v_%s_0.1" % marker
+    return blob, tree, tag, cmt
+
+
+def create_commits(length=1, marker='Default'):
+    data = []
+    for i in xrange(0, length):
+        _marker = "%s_%s" % (marker, i)
+        blob, tree, tag, cmt = create_commit(data, _marker)
+        data.extend([blob, tree, tag, cmt])
+    return data
+
+@skipIf(missing_libs, skipmsg)
+class FakeSwiftConnector(object):
+
+    def __init__(self, root, conf, store=None):
+        if store:
+            self.store = store
+        else:
+            self.store = {}
+        self.conf = conf
+        self.root = root
+        self.concurrency = 1
+        self.chunk_length = 12228
+        self.cache_length = 1
+
+    def put_object(self, name, content):
+        name = posixpath.join(self.root, name)
+        if hasattr(content, 'seek'):
+            content.seek(0)
+            content = content.read()
+        self.store[name] = content
+
+    def get_object(self, name, range=None):
+        name = posixpath.join(self.root, name)
+        if not range:
+            try:
+                return StringIO(self.store[name])
+            except KeyError:
+                return None
+        else:
+            l, r = range.split('-')
+            try:
+                if not l:
+                    r = -int(r)
+                    return self.store[name][r:]
+                else:
+                    return self.store[name][int(l):int(r)]
+            except KeyError:
+                return None
+
+    def get_container_objects(self):
+        return [{'name': k.replace(self.root + '/', '')}
+                for k in self.store]
+
+    def create_root(self):
+        if self.root in self.store.keys():
+            pass
+        else:
+            self.store[self.root] = ''
+
+    def get_object_stat(self, name):
+        name = posixpath.join(self.root, name)
+        if not name in self.store:
+            return None
+        return {'content-length': len(self.store[name])}
+
+
+@skipIf(missing_libs, skipmsg)
+class TestSwiftObjectStore(TestCase):
+
+    def setUp(self):
+        super(TestSwiftObjectStore, self).setUp()
+        self.conf = swift.load_conf(file=StringIO(config_file %
+                                                  def_config_file))
+        self.fsc = FakeSwiftConnector('fakerepo', conf=self.conf)
+
+    def _put_pack(self, sos, commit_amount=1, marker='Default'):
+        odata = create_commits(length=commit_amount, marker=marker)
+        data = [(d.type_num, d.as_raw_string()) for d in odata]
+        f = StringIO()
+        build_pack(f, data, store=sos)
+        sos.add_thin_pack(f.read, None)
+        return odata
+
+    def test_load_packs(self):
+        store = {'fakerepo/objects/pack/pack-'+'1'*40+'.idx': '',
+                 'fakerepo/objects/pack/pack-'+'1'*40+'.pack': '',
+                 'fakerepo/objects/pack/pack-'+'1'*40+'.info': '',
+                 'fakerepo/objects/pack/pack-'+'2'*40+'.idx': '',
+                 'fakerepo/objects/pack/pack-'+'2'*40+'.pack': '',
+                 'fakerepo/objects/pack/pack-'+'2'*40+'.info': ''}
+        fsc = FakeSwiftConnector('fakerepo', conf=self.conf, store=store)
+        sos = swift.SwiftObjectStore(fsc)
+        packs = sos._load_packs()
+        self.assertEqual(len(packs), 2)
+        for pack in packs:
+            self.assertTrue(isinstance(pack, swift.SwiftPack))
+
+    def test_add_thin_pack(self):
+        sos = swift.SwiftObjectStore(self.fsc)
+        self._put_pack(sos, 1, 'Default')
+        self.assertEqual(len(self.fsc.store), 3)
+
+    def test_find_missing_objects(self):
+        commit_amount = 3
+        sos = swift.SwiftObjectStore(self.fsc)
+        odata = self._put_pack(sos, commit_amount, 'Default')
+        head = odata[-1].id
+        i = sos.iter_shas(sos.find_missing_objects([],
+                                                   [head, ],
+                                                   progress=None,
+                                                   get_tagged=None))
+        self.assertEqual(len(i), commit_amount * 3)
+        shas = [d.id for d in odata]
+        for sha, path in i:
+            self.assertIn(sha.id, shas)
+
+    def test_find_missing_objects_with_tag(self):
+        commit_amount = 3
+        sos = swift.SwiftObjectStore(self.fsc)
+        odata = self._put_pack(sos, commit_amount, 'Default')
+        head = odata[-1].id
+        peeled_sha = dict([(sha.object[1], sha.id)
+                           for sha in odata if isinstance(sha, Tag)])
+        get_tagged = lambda: peeled_sha
+        i = sos.iter_shas(sos.find_missing_objects([],
+                                                   [head, ],
+                                                   progress=None,
+                                                   get_tagged=get_tagged))
+        self.assertEqual(len(i), commit_amount * 4)
+        shas = [d.id for d in odata]
+        for sha, path in i:
+            self.assertIn(sha.id, shas)
+
+    def test_find_missing_objects_with_common(self):
+        commit_amount = 3
+        sos = swift.SwiftObjectStore(self.fsc)
+        odata = self._put_pack(sos, commit_amount, 'Default')
+        head = odata[-1].id
+        have = odata[7].id
+        i = sos.iter_shas(sos.find_missing_objects([have, ],
+                                                   [head, ],
+                                                   progress=None,
+                                                   get_tagged=None))
+        self.assertEqual(len(i), 3)
+
+    def test_find_missing_objects_multiple_packs(self):
+        sos = swift.SwiftObjectStore(self.fsc)
+        commit_amount_a = 3
+        odataa = self._put_pack(sos, commit_amount_a, 'Default1')
+        heada = odataa[-1].id
+        commit_amount_b = 2
+        odatab = self._put_pack(sos, commit_amount_b, 'Default2')
+        headb = odatab[-1].id
+        i = sos.iter_shas(sos.find_missing_objects([],
+                                                   [heada, headb],
+                                                   progress=None,
+                                                   get_tagged=None))
+        self.assertEqual(len(self.fsc.store), 6)
+        self.assertEqual(len(i),
+                         commit_amount_a * 3 +
+                         commit_amount_b * 3)
+        shas = [d.id for d in odataa]
+        shas.extend([d.id for d in odatab])
+        for sha, path in i:
+            self.assertIn(sha.id, shas)
+
+    def test_add_thin_pack_ext_ref(self):
+        sos = swift.SwiftObjectStore(self.fsc)
+        odata = self._put_pack(sos, 1, 'Default1')
+        ref_blob_content = odata[0].as_raw_string()
+        ref_blob_id = odata[0].id
+        new_blob = Blob.from_string(ref_blob_content.replace('blob',
+                                                             'yummy blob'))
+        blob, tree, tag, cmt = \
+            create_commit([], marker='Default2', blob=new_blob)
+        data = [(REF_DELTA, (ref_blob_id, blob.as_raw_string())),
+                (tree.type_num, tree.as_raw_string()),
+                (cmt.type_num, cmt.as_raw_string()),
+                (tag.type_num, tag.as_raw_string())]
+        f = StringIO()
+        build_pack(f, data, store=sos)
+        sos.add_thin_pack(f.read, None)
+        self.assertEqual(len(self.fsc.store), 6)
+
+
+@skipIf(missing_libs, skipmsg)
+class TestSwiftRepo(TestCase):
+
+    def setUp(self):
+        super(TestSwiftRepo, self).setUp()
+        self.conf = swift.load_conf(file=StringIO(config_file %
+                                                  def_config_file))
+
+    def test_init(self):
+        store = {'fakerepo/objects/pack': ''}
+        with patch('dulwich.contrib.swift.SwiftConnector',
+                   new_callable=create_swift_connector,
+                   store=store):
+            swift.SwiftRepo('fakerepo', conf=self.conf)
+
+    def test_init_no_data(self):
+        with patch('dulwich.contrib.swift.SwiftConnector',
+                   new_callable=create_swift_connector):
+            self.assertRaises(Exception, swift.SwiftRepo,
+                              'fakerepo', self.conf)
+
+    def test_init_bad_data(self):
+        store = {'fakerepo/.git/objects/pack': ''}
+        with patch('dulwich.contrib.swift.SwiftConnector',
+                   new_callable=create_swift_connector,
+                   store=store):
+            self.assertRaises(Exception, swift.SwiftRepo,
+                              'fakerepo', self.conf)
+
+    def test_put_named_file(self):
+        store = {'fakerepo/objects/pack': ''}
+        with patch('dulwich.contrib.swift.SwiftConnector',
+                   new_callable=create_swift_connector,
+                   store=store):
+            repo = swift.SwiftRepo('fakerepo', conf=self.conf)
+            desc = 'Fake repo'
+            repo._put_named_file('description', desc)
+        self.assertEqual(repo.scon.store['fakerepo/description'],
+                         desc)
+
+    def test_init_bare(self):
+        fsc = FakeSwiftConnector('fakeroot', conf=self.conf)
+        with patch('dulwich.contrib.swift.SwiftConnector',
+                   new_callable=create_swift_connector,
+                   store=fsc.store):
+            swift.SwiftRepo.init_bare(fsc, conf=self.conf)
+        self.assertIn('fakeroot/objects/pack', fsc.store)
+        self.assertIn('fakeroot/info/refs', fsc.store)
+        self.assertIn('fakeroot/description', fsc.store)
+
+
+@skipIf(missing_libs, skipmsg)
+class TestPackInfoLoadDump(TestCase):
+    def setUp(self):
+        conf = swift.load_conf(file=StringIO(config_file %
+                                             def_config_file))
+        sos = swift.SwiftObjectStore(
+            FakeSwiftConnector('fakerepo', conf=conf))
+        commit_amount = 10
+        self.commits = create_commits(length=commit_amount, marker="m")
+        data = [(d.type_num, d.as_raw_string()) for d in self.commits]
+        f = StringIO()
+        fi = StringIO()
+        expected = build_pack(f, data, store=sos)
+        entries = [(sha, ofs, checksum) for
+                   ofs, _, _, sha, checksum in expected]
+        self.pack_data = PackData.from_file(file=f, size=None)
+        write_pack_index_v2(
+            fi, entries, self.pack_data.calculate_checksum())
+        fi.seek(0)
+        self.pack_index = load_pack_index_file('', fi)
+
+#    def test_pack_info_perf(self):
+#        dump_time = []
+#        load_time = []
+#        for i in xrange(0, 100):
+#            start = time()
+#            dumps = swift.pack_info_create(self.pack_data, self.pack_index)
+#            dump_time.append(time() - start)
+#        for i in xrange(0, 100):
+#            start = time()
+#            pack_infos = swift.load_pack_info('', file=StringIO(dumps))
+#            load_time.append(time() - start)
+#        print sum(dump_time) / float(len(dump_time))
+#        print sum(load_time) / float(len(load_time))
+
+    def test_pack_info(self):
+        dumps = swift.pack_info_create(self.pack_data, self.pack_index)
+        pack_infos = swift.load_pack_info('', file=StringIO(dumps))
+        for obj in self.commits:
+            self.assertIn(obj.id, pack_infos)
+
+
+@skipIf(missing_libs, skipmsg)
+class TestSwiftInfoRefsContainer(TestCase):
+
+    def setUp(self):
+        super(TestSwiftInfoRefsContainer, self).setUp()
+        content = \
+            "22effb216e3a82f97da599b8885a6cadb488b4c5\trefs/heads/master\n" + \
+            "cca703b0e1399008b53a1a236d6b4584737649e4\trefs/heads/dev"
+        self.store = {'fakerepo/info/refs': content}
+        self.conf = swift.load_conf(file=StringIO(config_file %
+                                                  def_config_file))
+        self.fsc = FakeSwiftConnector('fakerepo', conf=self.conf)
+        self.object_store = {}
+
+    def test_init(self):
+        """info/refs does not exists"""
+        irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store)
+        self.assertEqual(len(irc._refs), 0)
+        self.fsc.store = self.store
+        irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store)
+        self.assertIn('refs/heads/dev', irc.allkeys())
+        self.assertIn('refs/heads/master', irc.allkeys())
+
+    def test_set_if_equals(self):
+        self.fsc.store = self.store
+        irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store)
+        irc.set_if_equals('refs/heads/dev',
+                          "cca703b0e1399008b53a1a236d6b4584737649e4", '1'*40)
+        self.assertEqual(irc['refs/heads/dev'], '1'*40)
+
+    def test_remove_if_equals(self):
+        self.fsc.store = self.store
+        irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store)
+        irc.remove_if_equals('refs/heads/dev',
+                             "cca703b0e1399008b53a1a236d6b4584737649e4")
+        self.assertNotIn('refs/heads/dev', irc.allkeys())
+
+
+@skipIf(missing_libs, skipmsg)
+class TestSwiftConnector(TestCase):
+
+    def setUp(self):
+        super(TestSwiftConnector, self).setUp()
+        self.conf = swift.load_conf(file=StringIO(config_file %
+                                                  def_config_file))
+        with patch('geventhttpclient.HTTPClient.request',
+                   fake_auth_request_v1):
+            self.conn = swift.SwiftConnector('fakerepo', conf=self.conf)
+
+    def test_init_connector(self):
+        self.assertEqual(self.conn.auth_ver, '1')
+        self.assertEqual(self.conn.auth_url,
+                         'http://127.0.0.1:8080/auth/v1.0')
+        self.assertEqual(self.conn.user, 'test:tester')
+        self.assertEqual(self.conn.password, 'testing')
+        self.assertEqual(self.conn.root, 'fakerepo')
+        self.assertEqual(self.conn.storage_url,
+                         'http://127.0.0.1:8080/v1.0/AUTH_fakeuser')
+        self.assertEqual(self.conn.token, '12' * 10)
+        self.assertEqual(self.conn.http_timeout, 1)
+        self.assertEqual(self.conn.http_pool_length, 1)
+        self.assertEqual(self.conn.concurrency, 1)
+        self.conf.set('swift', 'auth_ver', '2')
+        self.conf.set('swift', 'auth_url', 'http://127.0.0.1:8080/auth/v2.0')
+        with patch('geventhttpclient.HTTPClient.request',
+                   fake_auth_request_v2):
+            conn = swift.SwiftConnector('fakerepo', conf=self.conf)
+        self.assertEqual(conn.user, 'tester')
+        self.assertEqual(conn.tenant, 'test')
+        self.conf.set('swift', 'auth_ver', '1')
+        self.conf.set('swift', 'auth_url', 'http://127.0.0.1:8080/auth/v1.0')
+        with patch('geventhttpclient.HTTPClient.request',
+                   fake_auth_request_v1_error):
+            self.assertRaises(swift.SwiftException,
+                              lambda: swift.SwiftConnector('fakerepo',
+                                                           conf=self.conf))
+
+    def test_root_exists(self):
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args: Response()):
+            self.assertEqual(self.conn.test_root_exists(), True)
+
+    def test_root_not_exists(self):
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args: Response(status=404)):
+            self.assertEqual(self.conn.test_root_exists(), None)
+
+    def test_create_root(self):
+        with patch('dulwich.contrib.swift.SwiftConnector.test_root_exists',
+                lambda *args: None), \
+             patch('geventhttpclient.HTTPClient.request',
+                lambda *args: Response()):
+            self.assertEqual(self.conn.create_root(), None)
+
+    def test_create_root_fails(self):
+        with patch('dulwich.contrib.swift.SwiftConnector.test_root_exists',
+                   lambda *args: None), \
+             patch('geventhttpclient.HTTPClient.request',
+                   lambda *args: Response(status=404)):
+            self.assertRaises(swift.SwiftException,
+                              lambda: self.conn.create_root())
+
+    def test_get_container_objects(self):
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args: Response(content=json_dumps(
+                       (({'name': 'a'}, {'name': 'b'}))))):
+            self.assertEqual(len(self.conn.get_container_objects()), 2)
+
+    def test_get_container_objects_fails(self):
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args: Response(status=404)):
+            self.assertEqual(self.conn.get_container_objects(), None)
+
+    def test_get_object_stat(self):
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args: Response(headers={'content-length': '10'})):
+            self.assertEqual(self.conn.get_object_stat('a')['content-length'],
+                             '10')
+
+    def test_get_object_stat_fails(self):
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args: Response(status=404)):
+            self.assertEqual(self.conn.get_object_stat('a'), None)
+
+    def test_put_object(self):
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args, **kwargs: Response()):
+            self.assertEqual(self.conn.put_object('a', StringIO('content')),
+                             None)
+
+    def test_put_object_fails(self):
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args, **kwargs: Response(status=400)):
+            self.assertRaises(swift.SwiftException,
+                              lambda: self.conn.put_object(
+                                  'a', StringIO('content')))
+
+    def test_get_object(self):
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args, **kwargs: Response(content='content')):
+            self.assertEqual(self.conn.get_object('a').read(), 'content')
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args, **kwargs: Response(content='content')):
+            self.assertEqual(self.conn.get_object('a', range='0-6'), 'content')
+
+    def test_get_object_fails(self):
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args, **kwargs: Response(status=404)):
+            self.assertEqual(self.conn.get_object('a'), None)
+
+    def test_del_object(self):
+        with patch('geventhttpclient.HTTPClient.request',
+                   lambda *args: Response()):
+            self.assertEqual(self.conn.del_object('a'), None)
+
+    def test_del_root(self):
+        with patch('dulwich.contrib.swift.SwiftConnector.del_object',
+                   lambda *args: None), \
+             patch('dulwich.contrib.swift.SwiftConnector.'
+                   'get_container_objects',
+                   lambda *args: ({'name': 'a'}, {'name': 'b'})), \
+             patch('geventhttpclient.HTTPClient.request',
+                    lambda *args: Response()):
+            self.assertEqual(self.conn.del_root(), None)
+
+
+@skipIf(missing_libs, skipmsg)
+class SwiftObjectStoreTests(ObjectStoreTests, TestCase):
+
+    def setUp(self):
+        TestCase.setUp(self)
+        conf = swift.load_conf(file=StringIO(config_file %
+                               def_config_file))
+        fsc = FakeSwiftConnector('fakerepo', conf=conf)
+        self.store = swift.SwiftObjectStore(fsc)

dulwich/contrib/test_swift_smoke.py

@@ -0,0 +1,314 @@
+# test_smoke.py -- Functional tests for the Swift backend.
+# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+#
+# Author: Fabien Boucher <fabien.boucher@enovance.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) any later version of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Start functional tests
+
+A Swift installation must be available before
+starting those tests. The account and authentication method used
+during this functional tests must be changed in the configuration file
+passed as environment variable.
+The container used to create a fake repository is defined
+in cls.fakerepo and will be deleted after the tests.
+
+DULWICH_SWIFT_CFG=/tmp/conf.cfg PYTHONPATH=. python -m unittest \
+    dulwich.tests_swift.test_smoke
+"""
+
+import os
+import unittest
+import tempfile
+import shutil
+
+import gevent
+from gevent import monkey
+monkey.patch_all()
+
+from dulwich import server
+from dulwich import repo
+from dulwich import index
+from dulwich import client
+from dulwich import objects
+from dulwich.contrib import swift
+
+
+class DulwichServer():
+    """Start the TCPGitServer with Swift backend
+    """
+    def __init__(self, backend, port):
+        self.port = port
+        self.backend = backend
+
+    def run(self):
+        self.server = server.TCPGitServer(self.backend,
+                                          'localhost',
+                                          port=self.port)
+        self.job = gevent.spawn(self.server.serve_forever)
+
+    def stop(self):
+        self.server.shutdown()
+        gevent.joinall((self.job,))
+
+
+class SwiftSystemBackend(server.Backend):
+
+    def open_repository(self, path):
+        return swift.SwiftRepo(path, conf=swift.load_conf())
+
+
+class SwiftRepoSmokeTest(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.backend = SwiftSystemBackend()
+        cls.port = 9148
+        cls.server_address = 'localhost'
+        cls.fakerepo = 'fakerepo'
+        cls.th_server = DulwichServer(cls.backend, cls.port)
+        cls.th_server.run()
+        cls.conf = swift.load_conf()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.th_server.stop()
+
+    def setUp(self):
+        self.scon = swift.SwiftConnector(self.fakerepo, self.conf)
+        if self.scon.test_root_exists():
+            try:
+                self.scon.del_root()
+            except swift.SwiftException:
+                pass
+        self.temp_d = tempfile.mkdtemp()
+        if os.path.isdir(self.temp_d):
+            shutil.rmtree(self.temp_d)
+
+    def tearDown(self):
+        if self.scon.test_root_exists():
+            try:
+                self.scon.del_root()
+            except swift.SwiftException:
+                pass
+        if os.path.isdir(self.temp_d):
+            shutil.rmtree(self.temp_d)
+
+    def test_init_bare(self):
+        swift.SwiftRepo.init_bare(self.scon, self.conf)
+        self.assertTrue(self.scon.test_root_exists())
+        obj = self.scon.get_container_objects()
+        filtered = [o for o in obj if o['name'] == 'info/refs'
+                    or o['name'] == 'objects/pack']
+        self.assertEqual(len(filtered), 2)
+
+    def test_clone_bare(self):
+        local_repo = repo.Repo.init(self.temp_d, mkdir=True)
+        swift.SwiftRepo.init_bare(self.scon, self.conf)
+        tcp_client = client.TCPGitClient(self.server_address,
+                                         port=self.port)
+        remote_refs = tcp_client.fetch(self.fakerepo, local_repo)
+        # The remote repo is empty (no refs retreived)
+        self.assertEqual(remote_refs, None)
+
+    def test_push_commit(self):
+        def determine_wants(*args):
+            return {"refs/heads/master": local_repo.refs["HEAD"]}
+
+        local_repo = repo.Repo.init(self.temp_d, mkdir=True)
+        # Nothing in the staging area
+        local_repo.do_commit('Test commit', 'fbo@localhost')
+        sha = local_repo.refs.read_loose_ref('refs/heads/master')
+        swift.SwiftRepo.init_bare(self.scon, self.conf)
+        tcp_client = client.TCPGitClient(self.server_address,
+                                         port=self.port)
+        tcp_client.send_pack(self.fakerepo,
+                             determine_wants,
+                             local_repo.object_store.generate_pack_contents)
+        swift_repo = swift.SwiftRepo("fakerepo", self.conf)
+        remote_sha = swift_repo.refs.read_loose_ref('refs/heads/master')
+        self.assertEqual(sha, remote_sha)
+
+    def test_push_branch(self):
+        def determine_wants(*args):
+            return {"refs/heads/mybranch":
+                    local_repo.refs["refs/heads/mybranch"]}
+
+        local_repo = repo.Repo.init(self.temp_d, mkdir=True)
+        # Nothing in the staging area
+        local_repo.do_commit('Test commit', 'fbo@localhost',
+                             ref='refs/heads/mybranch')
+        sha = local_repo.refs.read_loose_ref('refs/heads/mybranch')
+        swift.SwiftRepo.init_bare(self.scon, self.conf)
+        tcp_client = client.TCPGitClient(self.server_address,
+                                         port=self.port)
+        tcp_client.send_pack("/fakerepo",
+                             determine_wants,
+                             local_repo.object_store.generate_pack_contents)
+        swift_repo = swift.SwiftRepo(self.fakerepo, self.conf)
+        remote_sha = swift_repo.refs.read_loose_ref('refs/heads/mybranch')
+        self.assertEqual(sha, remote_sha)
+
+    def test_push_multiple_branch(self):
+        def determine_wants(*args):
+            return {"refs/heads/mybranch":
+                    local_repo.refs["refs/heads/mybranch"],
+                    "refs/heads/master":
+                    local_repo.refs["refs/heads/master"],
+                    "refs/heads/pullr-108":
+                    local_repo.refs["refs/heads/pullr-108"]}
+
+        local_repo = repo.Repo.init(self.temp_d, mkdir=True)
+        # Nothing in the staging area
+        local_shas = {}
+        remote_shas = {}
+        for branch in ('master', 'mybranch', 'pullr-108'):
+            local_shas[branch] = local_repo.do_commit(
+                'Test commit %s' % branch, 'fbo@localhost',
+                ref='refs/heads/%s' % branch)
+        swift.SwiftRepo.init_bare(self.scon, self.conf)
+        tcp_client = client.TCPGitClient(self.server_address,
+                                         port=self.port)
+        tcp_client.send_pack(self.fakerepo,
+                             determine_wants,
+                             local_repo.object_store.generate_pack_contents)
+        swift_repo = swift.SwiftRepo("fakerepo", self.conf)
+        for branch in ('master', 'mybranch', 'pullr-108'):
+            remote_shas[branch] = swift_repo.refs.read_loose_ref(
+                'refs/heads/%s' % branch)
+        self.assertDictEqual(local_shas, remote_shas)
+
+    def test_push_data_branch(self):
+        def determine_wants(*args):
+            return {"refs/heads/master": local_repo.refs["HEAD"]}
+        local_repo = repo.Repo.init(self.temp_d, mkdir=True)
+        os.mkdir(os.path.join(self.temp_d, "dir"))
+        files = ('testfile', 'testfile2', 'dir/testfile3')
+        i = 0
+        for f in files:
+            file(os.path.join(self.temp_d, f), 'w').write("DATA %s" % i)
+            i += 1
+        local_repo.stage(files)
+        local_repo.do_commit('Test commit', 'fbo@localhost',
+                             ref='refs/heads/master')
+        swift.SwiftRepo.init_bare(self.scon, self.conf)
+        tcp_client = client.TCPGitClient(self.server_address,
+                                         port=self.port)
+        tcp_client.send_pack(self.fakerepo,
+                             determine_wants,
+                             local_repo.object_store.generate_pack_contents)
+        swift_repo = swift.SwiftRepo("fakerepo", self.conf)
+        commit_sha = swift_repo.refs.read_loose_ref('refs/heads/master')
+        otype, data = swift_repo.object_store.get_raw(commit_sha)
+        commit = objects.ShaFile.from_raw_string(otype, data)
+        otype, data = swift_repo.object_store.get_raw(commit._tree)
+        tree = objects.ShaFile.from_raw_string(otype, data)
+        objs = tree.items()
+        objs_ = []
+        for tree_entry in objs:
+            objs_.append(swift_repo.object_store.get_raw(tree_entry.sha))
+        # Blob
+        self.assertEqual(objs_[1][1], 'DATA 0')
+        self.assertEqual(objs_[2][1], 'DATA 1')
+        # Tree
+        self.assertEqual(objs_[0][0], 2)
+
+    def test_clone_then_push_data(self):
+        self.test_push_data_branch()
+        shutil.rmtree(self.temp_d)
+        local_repo = repo.Repo.init(self.temp_d, mkdir=True)
+        tcp_client = client.TCPGitClient(self.server_address,
+                                         port=self.port)
+        remote_refs = tcp_client.fetch(self.fakerepo, local_repo)
+        files = (os.path.join(self.temp_d, 'testfile'),
+                 os.path.join(self.temp_d, 'testfile2'))
+        local_repo["HEAD"] = remote_refs["refs/heads/master"]
+        indexfile = local_repo.index_path()
+        tree = local_repo["HEAD"].tree
+        index.build_index_from_tree(local_repo.path, indexfile,
+                                    local_repo.object_store, tree)
+        for f in files:
+            self.assertEqual(os.path.isfile(f), True)
+
+        def determine_wants(*args):
+            return {"refs/heads/master": local_repo.refs["HEAD"]}
+        os.mkdir(os.path.join(self.temp_d, "test"))
+        files = ('testfile11', 'testfile22', 'test/testfile33')
+        i = 0
+        for f in files:
+            file(os.path.join(self.temp_d, f), 'w').write("DATA %s" % i)
+            i += 1
+        local_repo.stage(files)
+        local_repo.do_commit('Test commit', 'fbo@localhost',
+                             ref='refs/heads/master')
+        tcp_client.send_pack("/fakerepo",
+                             determine_wants,
+                             local_repo.object_store.generate_pack_contents)
+
+    def test_push_remove_branch(self):
+        def determine_wants(*args):
+            return {"refs/heads/pullr-108": objects.ZERO_SHA,
+                    "refs/heads/master":
+                    local_repo.refs['refs/heads/master'],
+                    "refs/heads/mybranch":
+                    local_repo.refs['refs/heads/mybranch'],
+                    }
+        self.test_push_multiple_branch()
+        local_repo = repo.Repo(self.temp_d)
+        tcp_client = client.TCPGitClient(self.server_address,
+                                         port=self.port)
+        tcp_client.send_pack(self.fakerepo,
+                             determine_wants,
+                             local_repo.object_store.generate_pack_contents)
+        swift_repo = swift.SwiftRepo("fakerepo", self.conf)
+        self.assertNotIn('refs/heads/pullr-108', swift_repo.refs.allkeys())
+
+    def test_push_annotated_tag(self):
+        def determine_wants(*args):
+            return {"refs/heads/master": local_repo.refs["HEAD"],
+                    "refs/tags/v1.0": local_repo.refs["refs/tags/v1.0"]}
+        local_repo = repo.Repo.init(self.temp_d, mkdir=True)
+        # Nothing in the staging area
+        sha = local_repo.do_commit('Test commit', 'fbo@localhost')
+        otype, data = local_repo.object_store.get_raw(sha)
+        commit = objects.ShaFile.from_raw_string(otype, data)
+        tag = objects.Tag()
+        tag.tagger = "fbo@localhost"
+        tag.message = "Annotated tag"
+        tag.tag_timezone = objects.parse_timezone('-0200')[0]
+        tag.tag_time = commit.author_time
+        tag.object = (objects.Commit, commit.id)
+        tag.name = "v0.1"
+        local_repo.object_store.add_object(tag)
+        local_repo.refs['refs/tags/v1.0'] = tag.id
+        swift.SwiftRepo.init_bare(self.scon, self.conf)
+        tcp_client = client.TCPGitClient(self.server_address,
+                                         port=self.port)
+        tcp_client.send_pack(self.fakerepo,
+                             determine_wants,
+                             local_repo.object_store.generate_pack_contents)
+        swift_repo = swift.SwiftRepo(self.fakerepo, self.conf)
+        tag_sha = swift_repo.refs.read_loose_ref('refs/tags/v1.0')
+        otype, data = swift_repo.object_store.get_raw(tag_sha)
+        rtag = objects.ShaFile.from_raw_string(otype, data)
+        self.assertEqual(rtag.object[1], commit.id)
+        self.assertEqual(rtag.id, tag.id)
+
+
+if __name__ == '__main__':
+    unittest.main()

dulwich/diff_tree.py

@@ -24,7 +24,7 @@ from collections import (
     )
 
 from io import BytesIO
-import itertools
+from itertools import chain, izip
 import stat
 
 from dulwich.objects import (
@@ -131,7 +131,8 @@ def walk_trees(store, tree1_id, tree2_id, prune_identical=False):
         to None. If neither entry's path is None, they are guaranteed to
         match.
     """
-    # This could be fairly easily generalized to >2 trees if we find a use case.
+    # This could be fairly easily generalized to >2 trees if we find a use
+    # case.
     mode1 = tree1_id and stat.S_IFDIR or None
     mode2 = tree2_id and stat.S_IFDIR or None
     todo = [(TreeEntry('', mode1, tree1_id), TreeEntry('', mode2, tree2_id))]
@@ -171,8 +172,8 @@ def tree_changes(store, tree1_id, tree2_id, want_unchanged=False,
     if (rename_detector is not None and tree1_id is not None and
         tree2_id is not None):
         for change in rename_detector.changes_with_renames(
-          tree1_id, tree2_id, want_unchanged=want_unchanged):
-            yield change
+            tree1_id, tree2_id, want_unchanged=want_unchanged):
+                yield change
         return
 
     entries = walk_trees(store, tree1_id, tree2_id,
@@ -229,8 +230,8 @@ def tree_changes_for_merge(store, parent_tree_ids, tree_id,
         in the merge.
 
         Each list contains one element per parent, with the TreeChange for that
-        path relative to that parent. An element may be None if it never existed
-        in one parent and was deleted in two others.
+        path relative to that parent. An element may be None if it never
+        existed in one parent and was deleted in two others.
 
         A path is only included in the output if it is a conflict, i.e. its SHA
         in the merge tree is not found in any of the parents, or in the case of
@@ -265,7 +266,8 @@ def tree_changes_for_merge(store, parent_tree_ids, tree_id,
             yield changes
         elif None not in changes:
             # If no change was found relative to one parent, that means the SHA
-            # must have matched the SHA in that parent, so it is not a conflict.
+            # must have matched the SHA in that parent, so it is not a
+            # conflict.
             yield changes
 
 
@@ -290,7 +292,7 @@ def _count_blocks(obj):
     block_truncate = block.truncate
     block_getvalue = block.getvalue
 
-    for c in itertools.chain(*obj.as_raw_chunks()):
+    for c in chain(*obj.as_raw_chunks()):
         block_write(c)
         n += 1
         if c == '\n' or n == _BLOCK_SIZE:
@@ -329,11 +331,11 @@ def _similarity_score(obj1, obj2, block_cache=None):
 
     :param obj1: The first object to score.
     :param obj2: The second object to score.
-    :param block_cache: An optional dict of SHA to block counts to cache results
-        between calls.
-    :return: The similarity score between the two objects, defined as the number
-        of bytes in common between the two objects divided by the maximum size,
-        scaled to the range 0-100.
+    :param block_cache: An optional dict of SHA to block counts to cache
+        results between calls.
+    :return: The similarity score between the two objects, defined as the
+        number of bytes in common between the two objects divided by the
+        maximum size, scaled to the range 0-100.
     """
     if block_cache is None:
         block_cache = {}
@@ -372,8 +374,8 @@ class RenameDetector(object):
         :param store: An ObjectStore for looking up objects.
         :param rename_threshold: The threshold similarity score for considering
             an add/delete pair to be a rename/copy; see _similarity_score.
-        :param max_files: The maximum number of adds and deletes to consider, or
-            None for no limit. The detector is guaranteed to compare no more
+        :param max_files: The maximum number of adds and deletes to consider,
+            or None for no limit. The detector is guaranteed to compare no more
             than max_files ** 2 add/delete pairs. This limit is provided because
             rename detection can be quadratic in the project size. If the limit
             is exceeded, no content rename detection is attempted.
@@ -447,7 +449,7 @@ class RenameDetector(object):
         delete_paths = set()
         for sha, sha_deletes in delete_map.iteritems():
             sha_adds = add_map[sha]
-            for (old, is_delete), new in itertools.izip(sha_deletes, sha_adds):
+            for (old, is_delete), new in izip(sha_deletes, sha_adds):
                 if stat.S_IFMT(old.mode) != stat.S_IFMT(new.mode):
                     continue
                 if is_delete:
@@ -459,7 +461,7 @@ class RenameDetector(object):
             num_extra_adds = len(sha_adds) - len(sha_deletes)
             # TODO(dborowitz): Less arbitrary way of dealing with extra copies.
             old = sha_deletes[0][0]
-            if num_extra_adds:
+            if num_extra_adds > 0:
                 for new in sha_adds[-num_extra_adds:]:
                     add_paths.add(new.path)
                     self._changes.append(TreeChange(CHANGE_COPY, old, new))
@@ -475,7 +477,8 @@ class RenameDetector(object):
             return CHANGE_MODIFY
         elif delete.type != CHANGE_DELETE:
             # If it's in deletes but not marked as a delete, it must have been
-            # added due to find_copies_harder, and needs to be marked as a copy.
+            # added due to find_copies_harder, and needs to be marked as a
+            # copy.
             return CHANGE_COPY
         return CHANGE_RENAME
 
@@ -509,7 +512,8 @@ class RenameDetector(object):
                     candidates.append((-score, rename))
 
     def _choose_content_renames(self):
-        # Sort scores from highest to lowest, but keep names in ascending order.
+        # Sort scores from highest to lowest, but keep names in ascending
+        # order.
         self._candidates.sort()
 
         delete_paths = set()
@@ -541,11 +545,12 @@ class RenameDetector(object):
             path = add.new.path
             delete = delete_map.get(path)
             if (delete is not None and
-              stat.S_IFMT(delete.old.mode) == stat.S_IFMT(add.new.mode)):
+                stat.S_IFMT(delete.old.mode) == stat.S_IFMT(add.new.mode)):
                 modifies[path] = TreeChange(CHANGE_MODIFY, delete.old, add.new)
 
         self._adds = [a for a in self._adds if a.new.path not in modifies]
-        self._deletes = [a for a in self._deletes if a.new.path not in modifies]
+        self._deletes = [a for a in self._deletes if a.new.path not in
+                         modifies]
         self._changes += modifies.values()
 
     def _sorted_changes(self):

dulwich/fastexport.py

@@ -70,7 +70,8 @@ class GitFastExporter(object):
         return marker
 
     def _iter_files(self, base_tree, new_tree):
-        for (old_path, new_path), (old_mode, new_mode), (old_hexsha, new_hexsha) in \
+        for ((old_path, new_path), (old_mode, new_mode),
+            (old_hexsha, new_hexsha)) in \
                 self.store.tree_changes(base_tree, new_tree):
             if new_path is None:
                 yield commands.FileDeleteCommand(old_path)
@@ -81,7 +82,8 @@ class GitFastExporter(object):
             if old_path != new_path and old_path is not None:
                 yield commands.FileRenameCommand(old_path, new_path)
             if old_mode != new_mode or old_hexsha != new_hexsha:
-                yield commands.FileModifyCommand(new_path, new_mode, marker, None)
+                yield commands.FileModifyCommand(new_path, new_mode, marker,
+                    None)
 
     def _export_commit(self, commit, ref, base_tree=None):
         file_cmds = list(self._iter_files(base_tree, commit.tree))
@@ -96,7 +98,8 @@ class GitFastExporter(object):
         committer, committer_email = split_email(commit.committer)
         cmd = commands.CommitCommand(ref, marker,
             (author, author_email, commit.author_time, commit.author_timezone),
-            (committer, committer_email, commit.commit_time, commit.commit_timezone),
+            (committer, committer_email, commit.commit_time,
+                commit.commit_timezone),
             commit.message, from_, merges, file_cmds)
         return (cmd, marker)
 
@@ -143,7 +146,8 @@ class GitImportProcessor(processor.ImportProcessor):
         else:
             author = cmd.committer
         (author_name, author_email, author_timestamp, author_timezone) = author
-        (committer_name, committer_email, commit_timestamp, commit_timezone) = cmd.committer
+        (committer_name, committer_email, commit_timestamp,
+            commit_timezone) = cmd.committer
         commit.author = "%s <%s>" % (author_name, author_email)
         commit.author_timezone = author_timezone
         commit.author_time = int(author_timestamp)
@@ -161,15 +165,18 @@ class GitImportProcessor(processor.ImportProcessor):
                     self.repo.object_store.add(blob)
                     blob_id = blob.id
                 else:
-                    assert filecmd.dataref[0] == ":", "non-marker refs not supported yet"
+                    assert filecmd.dataref[0] == ":", \
+                        "non-marker refs not supported yet"
                     blob_id = self.markers[filecmd.dataref[1:]]
                 self._contents[filecmd.path] = (filecmd.mode, blob_id)
             elif filecmd.name == "filedelete":
                 del self._contents[filecmd.path]
             elif filecmd.name == "filecopy":
-                self._contents[filecmd.dest_path] = self._contents[filecmd.src_path]
+                self._contents[filecmd.dest_path] = self._contents[
+                    filecmd.src_path]
             elif filecmd.name == "filerename":
-                self._contents[filecmd.new_path] = self._contents[filecmd.old_path]
+                self._contents[filecmd.new_path] = self._contents[
+                    filecmd.old_path]
                 del self._contents[filecmd.old_path]
             elif filecmd.name == "filedeleteall":
                 self._contents = {}

dulwich/file.py

@@ -21,6 +21,7 @@
 import errno
 import os
 import tempfile
+import io
 
 def ensure_dir_exists(dirname):
     """Ensure a directory exists, creating if necessary."""
@@ -36,7 +37,7 @@ def fancy_rename(oldname, newname):
     if not os.path.exists(newname):
         try:
             os.rename(oldname, newname)
-        except OSError as e:
+        except OSError:
             raise
         return
 
@@ -45,17 +46,17 @@ def fancy_rename(oldname, newname):
         (fd, tmpfile) = tempfile.mkstemp(".tmp", prefix=oldname+".", dir=".")
         os.close(fd)
         os.remove(tmpfile)
-    except OSError as e:
+    except OSError:
         # either file could not be created (e.g. permission problem)
         # or could not be deleted (e.g. rude virus scanner)
         raise
     try:
         os.rename(newname, tmpfile)
-    except OSError as e:
+    except OSError:
         raise   # no rename occurred
     try:
         os.rename(oldname, newname)
-    except OSError as e:
+    except OSError:
         os.rename(tmpfile, newname)
         raise
     os.remove(tmpfile)
@@ -82,7 +83,7 @@ def GitFile(filename, mode='rb', bufsize=-1):
     if 'w' in mode:
         return _GitFile(filename, mode, bufsize)
     else:
-        return file(filename, mode, bufsize)
+        return io.open(filename, mode, bufsize)
 
 
 class _GitFile(object):
@@ -98,8 +99,8 @@ class _GitFile(object):
 
     PROXY_PROPERTIES = set(['closed', 'encoding', 'errors', 'mode', 'name',
                             'newlines', 'softspace'])
-    PROXY_METHODS = ('__iter__', 'flush', 'fileno', 'isatty', 'next', 'read',
-                     'readline', 'readlines', 'xreadlines', 'seek', 'tell',
+    PROXY_METHODS = ('__iter__', 'flush', 'fileno', 'isatty', 'read',
+                     'readline', 'readlines', 'seek', 'tell',
                      'truncate', 'write', 'writelines')
     def __init__(self, filename, mode, bufsize):
         self._filename = filename
@@ -154,6 +155,12 @@ class _GitFile(object):
         finally:
             self.abort()
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
     def __getattr__(self, name):
         """Proxy property calls to the underlying file."""
         if name in self.PROXY_PROPERTIES:

dulwich/greenthreads.py

@@ -0,0 +1,141 @@
+# greenthreads.py -- Utility module for querying an ObjectStore with gevent
+# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+#
+# Author: Fabien Boucher <fabien.boucher@enovance.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) any later version of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Utility module for querying an ObjectStore with gevent."""
+
+import gevent
+from gevent import pool
+
+from dulwich.objects import (
+    Commit,
+    Tag,
+    )
+from dulwich.object_store import (
+    MissingObjectFinder,
+    _collect_filetree_revs,
+    ObjectStoreIterator,
+    )
+
+
+def _split_commits_and_tags(obj_store, lst,
+                            ignore_unknown=False, pool=None):
+    """Split object id list into two list with commit SHA1s and tag SHA1s.
+
+    Same implementation as object_store._split_commits_and_tags
+    except we use gevent to parallelize object retrieval.
+    """
+    commits = set()
+    tags = set()
+
+    def find_commit_type(sha):
+        try:
+            o = obj_store[sha]
+        except KeyError:
+            if not ignore_unknown:
+                raise
+        else:
+            if isinstance(o, Commit):
+                commits.add(sha)
+            elif isinstance(o, Tag):
+                tags.add(sha)
+                commits.add(o.object[1])
+            else:
+                raise KeyError('Not a commit or a tag: %s' % sha)
+    jobs = [pool.spawn(find_commit_type, s) for s in lst]
+    gevent.joinall(jobs)
+    return (commits, tags)
+
+
+class GreenThreadsMissingObjectFinder(MissingObjectFinder):
+    """Find the objects missing from another object store.
+
+    Same implementation as object_store.MissingObjectFinder
+    except we use gevent to parallelize object retrieval.
+    """
+    def __init__(self, object_store, haves, wants,
+                 progress=None, get_tagged=None,
+                 concurrency=1, get_parents=None):
+
+        def collect_tree_sha(sha):
+            self.sha_done.add(sha)
+            cmt = object_store[sha]
+            _collect_filetree_revs(object_store, cmt.tree, self.sha_done)
+
+        self.object_store = object_store
+        p = pool.Pool(size=concurrency)
+
+        have_commits, have_tags = \
+            _split_commits_and_tags(object_store, haves,
+                                    True, p)
+        want_commits, want_tags = \
+            _split_commits_and_tags(object_store, wants,
+                                    False, p)
+        all_ancestors = object_store._collect_ancestors(have_commits)[0]
+        missing_commits, common_commits = \
+            object_store._collect_ancestors(want_commits, all_ancestors)
+
+        self.sha_done = set()
+        jobs = [p.spawn(collect_tree_sha, c) for c in common_commits]
+        gevent.joinall(jobs)
+        for t in have_tags:
+            self.sha_done.add(t)
+        missing_tags = want_tags.difference(have_tags)
+        wants = missing_commits.union(missing_tags)
+        self.objects_to_send = set([(w, None, False) for w in wants])
+        if progress is None:
+            self.progress = lambda x: None
+        else:
+            self.progress = progress
+        self._tagged = get_tagged and get_tagged() or {}
+
+
+class GreenThreadsObjectStoreIterator(ObjectStoreIterator):
+    """ObjectIterator that works on top of an ObjectStore.
+
+    Same implementation as object_store.ObjectStoreIterator
+    except we use gevent to parallelize object retrieval.
+    """
+    def __init__(self, store, shas, finder, concurrency=1):
+        self.finder = finder
+        self.p = pool.Pool(size=concurrency)
+        super(GreenThreadsObjectStoreIterator, self).__init__(store, shas)
+
+    def retrieve(self, args):
+        sha, path = args
+        return self.store[sha], path
+
+    def __iter__(self):
+        for sha, path in self.p.imap_unordered(self.retrieve,
+                                               self.itershas()):
+            yield sha, path
+
+    def __len__(self):
+        if len(self._shas) > 0:
+            return len(self._shas)
+        while len(self.finder.objects_to_send):
+            jobs = []
+            for _ in xrange(0, len(self.finder.objects_to_send)):
+                jobs.append(self.p.spawn(self.finder.next))
+            gevent.joinall(jobs)
+            for j in jobs:
+                if j.value is not None:
+                    self._shas.append(j.value)
+        return len(self._shas)

dulwich/index.py

@@ -18,6 +18,7 @@
 
 """Parser for the git index file format."""
 
+import collections
 import errno
 import os
 import stat
@@ -25,6 +26,7 @@ import struct
 
 from dulwich.file import GitFile
 from dulwich.objects import (
+    Blob,
     S_IFGITLINK,
     S_ISGITLINK,
     Tree,
@@ -37,6 +39,12 @@ from dulwich.pack import (
     )
 
 
+IndexEntry = collections.namedtuple(
+    'IndexEntry', [
+        'ctime', 'mtime', 'dev', 'ino', 'mode', 'uid', 'gid', 'size', 'sha',
+        'flags'])
+
+
 def pathsplit(path):
     """Split a /-delimited path into a directory part and a basename.
 
@@ -97,7 +105,7 @@ def read_cache_entry(f):
     name = f.read((flags & 0x0fff))
     # Padding:
     real_size = ((f.tell() - beginoffset + 8) & ~7)
-    data = f.read((beginoffset + real_size) - f.tell())
+    f.read((beginoffset + real_size) - f.tell())
     return (name, ctime, mtime, dev, ino, mode, uid, gid, size,
             sha_to_hex(sha), flags & ~0x0fff)
 
@@ -138,7 +146,7 @@ def read_index_dict(f):
     """
     ret = {}
     for x in read_index(f):
-        ret[x[0]] = tuple(x[1:])
+        ret[x[0]] = IndexEntry(*x[1:])
     return ret
 
 
@@ -214,7 +222,7 @@ class Index(object):
         try:
             f = SHA1Reader(f)
             for x in read_index(f):
-                self[x[0]] = tuple(x[1:])
+                self[x[0]] = IndexEntry(*x[1:])
             # FIXME: Additional data?
             f.read(os.path.getsize(self._filename)-f.tell()-20)
             f.check_sha()
@@ -238,17 +246,17 @@ class Index(object):
 
     def get_sha1(self, path):
         """Return the (git object) SHA1 for the object at a path."""
-        return self[path][-2]
+        return self[path].sha
 
     def get_mode(self, path):
         """Return the POSIX file mode for the object at a path."""
-        return self[path][-6]
+        return self[path].mode
 
     def iterblobs(self):
         """Iterate over path, sha, mode tuples for use with commit_tree."""
         for path in self:
             entry = self[path]
-            yield path, entry[-2], cleanup_mode(entry[-6])
+            yield path, entry.sha, cleanup_mode(entry.mode)
 
     def clear(self):
         """Remove all contents from this index."""
@@ -281,7 +289,7 @@ class Index(object):
         """
         def lookup_entry(path):
             entry = self[path]
-            return entry[-2], entry[-6]
+            return entry.sha, entry.mode
         for (name, mode, sha) in changes_from_tree(self._byname.keys(),
                 lookup_entry, object_store, tree,
                 want_unchanged=want_unchanged):
@@ -445,3 +453,37 @@ def build_index_from_tree(prefix, index_path, object_store, tree_id,
         index[entry.path] = index_entry_from_stat(st, entry.sha, 0)
 
     index.write()
+
+
+def blob_from_path_and_stat(path, st):
+    """Create a blob from a path and a stat object.
+
+    :param path: Full path to file
+    :param st: A stat object
+    :return: A `Blob` object
+    """
+    blob = Blob()
+    if not stat.S_ISLNK(st.st_mode):
+        f = open(path, 'rb')
+        try:
+            blob.data = f.read()
+        finally:
+            f.close()
+    else:
+        blob.data = os.readlink(path)
+    return blob
+
+
+def get_unstaged_changes(index, path):
+    """Walk through an index and check for differences against working tree.
+
+    :param index: index to check
+    :param path: path in which to find files
+    :return: iterator over paths with unstaged changes
+    """
+    # For each entry in the index check the sha1 & ensure not staged
+    for name, entry in index.iteritems():
+        fp = os.path.join(path, name)
+        blob = blob_from_path_and_stat(fp, os.lstat(fp))
+        if blob.id != entry.sha:
+            yield name

dulwich/lru_cache.py

@@ -362,6 +362,6 @@ class LRUSizeCache(LRUCache):
     def _update_max_size(self, max_size, after_cleanup_size=None):
         self._max_size = max_size
         if after_cleanup_size is None:
-            self._after_cleanup_size = self._max_size * 8 / 10
+            self._after_cleanup_size = self._max_size * 8 // 10
         else:
             self._after_cleanup_size = min(after_cleanup_size, self._max_size)

dulwich/object_store.py

@@ -23,7 +23,7 @@
 
 from io import BytesIO
 import errno
-import itertools
+from itertools import chain
 import os
 import stat
 import tempfile
@@ -336,7 +336,7 @@ class PackBasedObjectStore(BaseObjectStore):
     def __iter__(self):
         """Iterate over the SHAs that are present in this store."""
         iterables = self.packs + [self._iter_loose_objects()] + [self._iter_alternate_objects()]
-        return itertools.chain(*iterables)
+        return chain(*iterables)
 
     def contains_loose(self, sha):
         """Check if a particular object is present by SHA1 and is loose.
@@ -924,10 +924,10 @@ def _collect_filetree_revs(obj_store, tree_sha, kset):
     """
     filetree = obj_store[tree_sha]
     for name, mode, sha in filetree.iteritems():
-       if not S_ISGITLINK(mode) and sha not in kset:
-           kset.add(sha)
-           if stat.S_ISDIR(mode):
-               _collect_filetree_revs(obj_store, sha, kset)
+        if not S_ISGITLINK(mode) and sha not in kset:
+            kset.add(sha)
+            if stat.S_ISDIR(mode):
+                _collect_filetree_revs(obj_store, sha, kset)
 
 
 def _split_commits_and_tags(obj_store, lst, ignore_unknown=False):
@@ -978,7 +978,7 @@ class MissingObjectFinder(object):
     """
 
     def __init__(self, object_store, haves, wants, progress=None,
-            get_tagged=None, get_parents=lambda commit: commit.parents):
+                 get_tagged=None, get_parents=lambda commit: commit.parents):
         self.object_store = object_store
         self._get_parents = get_parents
         # process Commits and Tags differently
@@ -986,22 +986,19 @@ class MissingObjectFinder(object):
         # and such SHAs would get filtered out by _split_commits_and_tags,
         # wants shall list only known SHAs, and otherwise
         # _split_commits_and_tags fails with KeyError
-        have_commits, have_tags = \
-                _split_commits_and_tags(object_store, haves, True)
-        want_commits, want_tags = \
-                _split_commits_and_tags(object_store, wants, False)
+        have_commits, have_tags = (
+            _split_commits_and_tags(object_store, haves, True))
+        want_commits, want_tags = (
+            _split_commits_and_tags(object_store, wants, False))
         # all_ancestors is a set of commits that shall not be sent
         # (complete repository up to 'haves')
         all_ancestors = object_store._collect_ancestors(
-                have_commits,
-                get_parents=self._get_parents)[0]
+            have_commits, get_parents=self._get_parents)[0]
         # all_missing - complete set of commits between haves and wants
         # common - commits from all_ancestors we hit into while
         # traversing parent hierarchy of wants
         missing_commits, common_commits = object_store._collect_ancestors(
-            want_commits,
-            all_ancestors,
-            get_parents=self._get_parents);
+            want_commits, all_ancestors, get_parents=self._get_parents)
         self.sha_done = set()
         # Now, fill sha_done with commits and revisions of
         # files and directories known to be both locally
@@ -1055,6 +1052,8 @@ class MissingObjectFinder(object):
         self.progress("counting objects: %d\r" % len(self.sha_done))
         return (sha, name)
 
+    __next__ = next
+
 
 class ObjectStoreGraphWalker(object):
     """Graph walker that finds what commits are missing from an object store.
@@ -1106,3 +1105,5 @@ class ObjectStoreGraphWalker(object):
             self.heads.update([p for p in ps if not p in self.parents])
             return ret
         return None
+
+    __next__ = next

dulwich/objects.py

@@ -58,6 +58,7 @@ _TAGGER_HEADER = "tagger"
 
 S_IFGITLINK = 0o160000
 
+
 def S_ISGITLINK(m):
     """Check if a mode indicates a submodule.
 
@@ -398,7 +399,7 @@ class ShaFile(object):
             obj._needs_serialization = True
             obj._file = f
             return obj
-        except (IndexError, ValueError) as e:
+        except (IndexError, ValueError):
             raise ObjectFormatException("invalid object header")
 
     @staticmethod
@@ -727,10 +728,12 @@ class Tag(ShaFile):
     tagger = serializable_property("tagger",
         "Returns the name of the person who created this tag")
     tag_time = serializable_property("tag_time",
-        "The creation timestamp of the tag.  As the number of seconds since the epoch")
+        "The creation timestamp of the tag.  As the number of seconds "
+        "since the epoch")
     tag_timezone = serializable_property("tag_timezone",
         "The timezone that tag_time is in.")
-    message = serializable_property("message", "The message attached to this tag")
+    message = serializable_property(
+        "message", "The message attached to this tag")
 
 
 class TreeEntry(namedtuple('TreeEntry', ['path', 'mode', 'sha'])):
@@ -881,7 +884,8 @@ class Tree(ShaFile):
         """
         if isinstance(name, int) and isinstance(mode, str):
             (name, mode) = (mode, name)
-            warnings.warn("Please use Tree.add(name, mode, hexsha)",
+            warnings.warn(
+                "Please use Tree.add(name, mode, hexsha)",
                 category=DeprecationWarning, stacklevel=2)
         self._ensure_parsed()
         self._entries[name] = mode, hexsha
@@ -890,7 +894,8 @@ class Tree(ShaFile):
     def iteritems(self, name_order=False):
         """Iterate over entries.
 
-        :param name_order: If True, iterate in name order instead of tree order.
+        :param name_order: If True, iterate in name order instead of tree
+            order.
         :return: Iterator over (name, mode, sha) tuples
         """
         self._ensure_parsed()
@@ -909,8 +914,8 @@ class Tree(ShaFile):
             parsed_entries = parse_tree("".join(chunks))
         except ValueError as e:
             raise ObjectFormatException(e)
-        # TODO: list comprehension is for efficiency in the common (small) case;
-        # if memory efficiency in the large case is a concern, use a genexp.
+        # TODO: list comprehension is for efficiency in the common (small)
+        # case; if memory efficiency in the large case is a concern, use a genexp.
         self._entries = dict([(n, (m, s)) for n, m, s in parsed_entries])
 
     def check(self):
@@ -1088,12 +1093,12 @@ class Commit(ShaFile):
 
     def _deserialize(self, chunks):
         (self._tree, self._parents, author_info, commit_info, self._encoding,
-                self._mergetag, self._message, self._extra) = \
-                        parse_commit(chunks)
+                self._mergetag, self._message, self._extra) = (
+                        parse_commit(chunks))
         (self._author, self._author_time, (self._author_timezone,
-            self._author_timezone_neg_utc)) = author_info
+             self._author_timezone_neg_utc)) = author_info
         (self._committer, self._commit_time, (self._commit_timezone,
-            self._commit_timezone_neg_utc)) = commit_info
+             self._commit_timezone_neg_utc)) = commit_info
 
     def check(self):
         """Check this object for internal consistency.
@@ -1137,12 +1142,12 @@ class Commit(ShaFile):
         for p in self._parents:
             chunks.append("%s %s\n" % (_PARENT_HEADER, p))
         chunks.append("%s %s %s %s\n" % (
-          _AUTHOR_HEADER, self._author, str(self._author_time),
-          format_timezone(self._author_timezone,
+            _AUTHOR_HEADER, self._author, str(self._author_time),
+            format_timezone(self._author_timezone,
                           self._author_timezone_neg_utc)))
         chunks.append("%s %s %s %s\n" % (
-          _COMMITTER_HEADER, self._committer, str(self._commit_time),
-          format_timezone(self._commit_timezone,
+            _COMMITTER_HEADER, self._committer, str(self._commit_time),
+            format_timezone(self._commit_timezone,
                           self._commit_timezone_neg_utc)))
         if self.encoding:
             chunks.append("%s %s\n" % (_ENCODING_HEADER, self.encoding))
@@ -1158,13 +1163,15 @@ class Commit(ShaFile):
             chunks[-1] = chunks[-1].rstrip(" \n")
         for k, v in self.extra:
             if "\n" in k or "\n" in v:
-                raise AssertionError("newline in extra data: %r -> %r" % (k, v))
+                raise AssertionError(
+                    "newline in extra data: %r -> %r" % (k, v))
             chunks.append("%s %s\n" % (k, v))
-        chunks.append("\n") # There must be a new line after the headers
+        chunks.append("\n")  # There must be a new line after the headers
         chunks.append(self._message)
         return chunks
 
-    tree = serializable_property("tree", "Tree that is the state of this commit")
+    tree = serializable_property(
+        "tree", "Tree that is the state of this commit")
 
     def _get_parents(self):
         """Return a list of parents of this commit."""
@@ -1192,8 +1199,8 @@ class Commit(ShaFile):
     committer = serializable_property("committer",
         "The name of the committer of the commit")
 
-    message = serializable_property("message",
-        "The commit message")
+    message = serializable_property(
+        "message", "The commit message")
 
     commit_time = serializable_property("commit_time",
         "The timestamp of the commit. As the number of seconds since the epoch.")
@@ -1202,16 +1209,17 @@ class Commit(ShaFile):
         "The zone the commit time is in")
 
     author_time = serializable_property("author_time",
-        "The timestamp the commit was written. as the number of seconds since the epoch.")
+        "The timestamp the commit was written. As the number of "
+        "seconds since the epoch.")
 
-    author_timezone = serializable_property("author_timezone",
-        "Returns the zone the author time is in.")
+    author_timezone = serializable_property(
+        "author_timezone", "Returns the zone the author time is in.")
 
-    encoding = serializable_property("encoding",
-        "Encoding of the commit message.")
+    encoding = serializable_property(
+        "encoding", "Encoding of the commit message.")
 
-    mergetag = serializable_property("mergetag",
-        "Associated signed tag.")
+    mergetag = serializable_property(
+        "mergetag", "Associated signed tag.")
 
 
 OBJECT_CLASSES = (
@@ -1228,7 +1236,6 @@ for cls in OBJECT_CLASSES:
     _TYPE_MAP[cls.type_num] = cls
 
 
-
 # Hold on to the pure-python implementations for testing
 _parse_tree_py = parse_tree
 _sorted_tree_items_py = sorted_tree_items

dulwich/pack.py

@@ -38,11 +38,9 @@ from collections import (
     deque,
     )
 import difflib
-from itertools import (
-    chain,
-    imap,
-    izip,
-    )
+
+from itertools import chain, imap, izip
+
 try:
     import mmap
 except ImportError:
@@ -57,7 +55,6 @@ from os import (
     )
 import struct
 from struct import unpack_from
-import sys
 import warnings
 import zlib
 
@@ -76,9 +73,6 @@ from dulwich.objects import (
     object_header,
     )
 
-supports_mmap_offset = (sys.version_info[0] >= 3 or
-        (sys.version_info[0] == 2 and sys.version_info[1] >= 6))
-
 
 OFS_DELTA = 6
 REF_DELTA = 7
@@ -86,6 +80,9 @@ REF_DELTA = 7
 DELTA_TYPES = (OFS_DELTA, REF_DELTA)
 
 
+DEFAULT_PACK_DELTA_WINDOW_SIZE = 10
+
+
 def take_msb_bytes(read, crc32=None):
     """Read bytes marked with most significant bit.
 
@@ -318,7 +315,7 @@ def bisect_find_sha(start, end, sha, unpack_name):
     """
     assert start <= end
     while start <= end:
-        i = (start + end)/2
+        i = (start + end) // 2
         file_sha = unpack_name(i)
         x = cmp(file_sha, sha)
         if x < 0:
@@ -994,6 +991,12 @@ class PackData(object):
     def close(self):
         self._file.close()
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
     def _get_size(self):
         if self._size is not None:
             return self._size
@@ -1444,21 +1447,20 @@ def write_pack_object(f, type, object, sha=None):
     return crc32 & 0xffffffff
 
 
-def write_pack(filename, objects, num_objects=None):
+def write_pack(filename, objects, deltify=None, delta_window_size=None):
     """Write a new pack data file.
 
     :param filename: Path to the new pack file (without .pack extension)
     :param objects: Iterable of (object, path) tuples to write.
         Should provide __len__
+    :param window_size: Delta window size
+    :param deltify: Whether to deltify pack objects
     :return: Tuple with checksum of pack file and index file
     """
-    if num_objects is not None:
-        warnings.warn('num_objects argument to write_pack is deprecated',
-                      DeprecationWarning)
     f = GitFile(filename + '.pack', 'wb')
     try:
         entries, data_sum = write_pack_objects(f, objects,
-            num_objects=num_objects)
+            delta_window_size=delta_window_size, deltify=deltify)
     finally:
         f.close()
     entries = [(k, v[0], v[1]) for (k, v) in entries.iteritems()]
@@ -1477,14 +1479,16 @@ def write_pack_header(f, num_objects):
     f.write(struct.pack('>L', num_objects))  # Number of objects in pack
 
 
-def deltify_pack_objects(objects, window=10):
+def deltify_pack_objects(objects, window_size=None):
     """Generate deltas for pack objects.
 
-    :param objects: Objects to deltify
-    :param window: Window size
+    :param objects: An iterable of (object, path) tuples to deltify.
+    :param window_size: Window size; None for default
     :return: Iterator over type_num, object id, delta_base, content
         delta_base is None for full text entries
     """
+    if window_size is None:
+        window_size = DEFAULT_PACK_DELTA_WINDOW_SIZE
     # Build a list of objects ordered by the magic Linus heuristic
     # This helps us find good objects to diff against us
     magic = []
@@ -1507,28 +1511,29 @@ def deltify_pack_objects(objects, window=10):
                 winner = delta
         yield type_num, o.sha().digest(), winner_base, winner
         possible_bases.appendleft(o)
-        while len(possible_bases) > window:
+        while len(possible_bases) > window_size:
             possible_bases.pop()
 
 
-def write_pack_objects(f, objects, window=10, num_objects=None):
+def write_pack_objects(f, objects, delta_window_size=None, deltify=False):
     """Write a new pack data file.
 
     :param f: File to write to
     :param objects: Iterable of (object, path) tuples to write.
         Should provide __len__
-    :param window: Sliding window size for searching for deltas; currently
-                   unimplemented
-    :param num_objects: Number of objects (do not use, deprecated)
+    :param window_size: Sliding window size for searching for deltas;
+                        Set to None for default window size.
+    :param deltify: Whether to deltify objects
     :return: Dict mapping id -> (offset, crc32 checksum), pack checksum
     """
-    if num_objects is None:
-        num_objects = len(objects)
-    # FIXME: pack_contents = deltify_pack_objects(objects, window)
-    pack_contents = (
-        (o.type_num, o.sha().digest(), None, o.as_raw_string())
-        for (o, path) in objects)
-    return write_pack_data(f, num_objects, pack_contents)
+    if deltify:
+        pack_contents = deltify_pack_objects(objects, delta_window_size)
+    else:
+        pack_contents = (
+            (o.type_num, o.sha().digest(), None, o.as_raw_string())
+            for (o, path) in objects)
+
+    return write_pack_data(f, len(objects), pack_contents)
 
 
 def write_pack_data(f, num_records, records):
@@ -1544,6 +1549,7 @@ def write_pack_data(f, num_records, records):
     f = SHA1Writer(f)
     write_pack_header(f, num_records)
     for type_num, object_id, delta_base, raw in records:
+        offset = f.offset()
         if delta_base is not None:
             try:
                 base_offset, base_crc32 = entries[delta_base]
@@ -1552,8 +1558,7 @@ def write_pack_data(f, num_records, records):
                 raw = (delta_base, raw)
             else:
                 type_num = OFS_DELTA
-                raw = (base_offset, raw)
-        offset = f.offset()
+                raw = (offset - base_offset, raw)
         crc32 = write_pack_object(f, type_num, raw)
         entries[object_id] = (offset, crc32)
     return entries, f.write_sha()
@@ -1585,6 +1590,36 @@ def write_pack_index_v1(f, entries, pack_checksum):
     return f.write_sha()
 
 
+def _delta_encode_size(size):
+    ret = ''
+    c = size & 0x7f
+    size >>= 7
+    while size:
+        ret += chr(c | 0x80)
+        c = size & 0x7f
+        size >>= 7
+    ret += chr(c)
+    return ret
+
+
+# copy operations in git's delta format can be at most this long -
+# after this you have to decompose the copy into multiple operations.
+_MAX_COPY_LEN = 0xffffff
+
+def _encode_copy_operation(start, length):
+    scratch = ''
+    op = 0x80
+    for i in range(4):
+        if start & 0xff << i*8:
+            scratch += chr((start >> i*8) & 0xff)
+            op |= 1 << i
+    for i in range(3):
+        if length & 0xff << i*8:
+            scratch += chr((length >> i*8) & 0xff)
+            op |= 1 << (4+i)
+    return chr(op) + scratch
+
+
 def create_delta(base_buf, target_buf):
     """Use python difflib to work out how to transform base_buf to target_buf.
 
@@ -1594,19 +1629,9 @@ def create_delta(base_buf, target_buf):
     assert isinstance(base_buf, str)
     assert isinstance(target_buf, str)
     out_buf = ''
-    # write delta header
-    def encode_size(size):
-        ret = ''
-        c = size & 0x7f
-        size >>= 7
-        while size:
-            ret += chr(c | 0x80)
-            c = size & 0x7f
-            size >>= 7
-        ret += chr(c)
-        return ret
-    out_buf += encode_size(len(base_buf))
-    out_buf += encode_size(len(target_buf))
+     # write delta header
+    out_buf += _delta_encode_size(len(base_buf))
+    out_buf += _delta_encode_size(len(target_buf))
     # write out delta opcodes
     seq = difflib.SequenceMatcher(a=base_buf, b=target_buf)
     for opcode, i1, i2, j1, j2 in seq.get_opcodes():
@@ -1616,20 +1641,13 @@ def create_delta(base_buf, target_buf):
         if opcode == 'equal':
             # If they are equal, unpacker will use data from base_buf
             # Write out an opcode that says what range to use
-            scratch = ''
-            op = 0x80
-            o = i1
-            for i in range(4):
-                if o & 0xff << i*8:
-                    scratch += chr((o >> i*8) & 0xff)
-                    op |= 1 << i
-            s = i2 - i1
-            for i in range(2):
-                if s & 0xff << i*8:
-                    scratch += chr((s >> i*8) & 0xff)
-                    op |= 1 << (4+i)
-            out_buf += chr(op)
-            out_buf += scratch
+            copy_start = i1
+            copy_len = i2 - i1
+            while copy_len > 0:
+                to_copy = min(copy_len, _MAX_COPY_LEN)
+                out_buf += _encode_copy_operation(copy_start, to_copy)
+                copy_start += to_copy
+                copy_len -= to_copy
         if opcode == 'replace' or opcode == 'insert':
             # If we are replacing a range or adding one, then we just
             # output it to the stream (prefixed by its size)
@@ -1806,6 +1824,12 @@ class Pack(object):
         if self._idx is not None:
             self._idx.close()
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
     def __eq__(self, other):
         return isinstance(self, type(other)) and self.index == other.index
 

dulwich/porcelain.py

@@ -24,6 +24,7 @@ Currently implemented:
  * clone
  * commit
  * commit-tree
+ * daemon
  * diff-tree
  * init
  * list-tags
@@ -34,6 +35,7 @@ Currently implemented:
  * rev-list
  * tag
  * update-server-info
+ * status
  * symbolic-ref
 
 These functions are meant to behave similarly to the git subcommands.
@@ -42,6 +44,7 @@ Differences in behaviour are considered bugs.
 
 __docformat__ = 'restructuredText'
 
+from collections import namedtuple
 import os
 import sys
 import time
@@ -52,6 +55,7 @@ from dulwich.errors import (
     SendPackError,
     UpdateRefsError,
     )
+from dulwich.index import get_unstaged_changes
 from dulwich.objects import (
     Tag,
     parse_timezone,
@@ -61,6 +65,9 @@ from dulwich.patch import write_tree_diff
 from dulwich.repo import (BaseRepo, Repo)
 from dulwich.server import update_server_info as server_update_server_info
 
+# Module level tuple definition for status output
+GitStatus = namedtuple('GitStatus', 'staged unstaged untracked')
+
 
 def open_repo(path_or_repo):
     """Open an argument that can be a repository or a path for a repository."""
@@ -82,7 +89,10 @@ def archive(location, committish=None, outstream=sys.stdout,
     client, path = get_transport_and_path(location)
     if committish is None:
         committish = "HEAD"
-    client.archive(path, committish, outstream.write, errstream.write)
+    # TODO(jelmer): This invokes C git; this introduces a dependency.
+    # Instead, dulwich should have its own archiver implementation.
+    client.archive(path, committish, outstream.write, errstream.write,
+                   errstream.write)
 
 
 def update_server_info(repo="."):
@@ -222,7 +232,7 @@ def rm(repo=".", paths=None):
     index.write()
 
 
-def print_commit(commit, outstream):
+def print_commit(commit, outstream=sys.stdout):
     """Write a human-readable commit log entry.
 
     :param commit: A `Commit` object
@@ -239,7 +249,7 @@ def print_commit(commit, outstream):
     outstream.write("\n")
 
 
-def print_tag(tag, outstream):
+def print_tag(tag, outstream=sys.stdout):
     """Write a human-readable tag.
 
     :param tag: A `Tag` object
@@ -252,7 +262,7 @@ def print_tag(tag, outstream):
     outstream.write("\n")
 
 
-def show_blob(repo, blob, outstream):
+def show_blob(repo, blob, outstream=sys.stdout):
     """Write a blob to a stream.
 
     :param repo: A `Repo` object
@@ -262,7 +272,7 @@ def show_blob(repo, blob, outstream):
     outstream.write(blob.data)
 
 
-def show_commit(repo, commit, outstream):
+def show_commit(repo, commit, outstream=sys.stdout):
     """Show a commit to a stream.
 
     :param repo: A `Repo` object
@@ -274,7 +284,7 @@ def show_commit(repo, commit, outstream):
     write_tree_diff(outstream, repo.object_store, parent_commit.tree, commit.tree)
 
 
-def show_tree(repo, tree, outstream):
+def show_tree(repo, tree, outstream=sys.stdout):
     """Print a tree to a stream.
 
     :param repo: A `Repo` object
@@ -285,7 +295,7 @@ def show_tree(repo, tree, outstream):
         outstream.write("%s\n" % n)
 
 
-def show_tag(repo, tag, outstream):
+def show_tag(repo, tag, outstream=sys.stdout):
     """Print a tag to a stream.
 
     :param repo: A `Repo` object
@@ -485,3 +495,64 @@ def pull(repo, remote_location, refs_path,
     indexfile = r.index_path()
     tree = r["HEAD"].tree
     index.build_index_from_tree(r.path, indexfile, r.object_store, tree)
+
+
+def status(repo):
+    """Returns staged, unstaged, and untracked changes relative to the HEAD.
+
+    :param repo: Path to repository
+    :return: GitStatus tuple,
+        staged -    list of staged paths (diff index/HEAD)
+        unstaged -  list of unstaged paths (diff index/working-tree)
+        untracked - list of untracked, un-ignored & non-.git paths
+    """
+    # 1. Get status of staged
+    tracked_changes = get_tree_changes(repo)
+    # 2. Get status of unstaged
+    unstaged_changes = list(get_unstaged_changes(repo.open_index(), repo.path))
+    # TODO - Status of untracked - add untracked changes, need gitignore.
+    untracked_changes = []
+    return GitStatus(tracked_changes, unstaged_changes, untracked_changes)
+
+
+def get_tree_changes(repo):
+    """Return add/delete/modify changes to tree by comparing index to HEAD.
+
+    :param repo: repo path or object
+    :return: dict with lists for each type of change
+    """
+    r = open_repo(repo)
+    index = r.open_index()
+
+    # Compares the Index to the HEAD & determines changes
+    # Iterate through the changes and report add/delete/modify
+    tracked_changes = {
+        'add': [],
+        'delete': [],
+        'modify': [],
+    }
+    for change in index.changes_from_tree(r.object_store, r['HEAD'].tree):
+        if not change[0][0]:
+            tracked_changes['add'].append(change[0][1])
+        elif not change[0][1]:
+            tracked_changes['delete'].append(change[0][0])
+        elif change[0][0] == change[0][1]:
+            tracked_changes['modify'].append(change[0][0])
+        else:
+            raise AssertionError('git mv ops not yet supported')
+    return tracked_changes
+
+
+def daemon(path=".", address=None, port=None):
+    """Run a daemon serving Git requests over TCP/IP.
+
+    :param path: Path to the directory to serve.
+    """
+    # TODO(jelmer): Support git-daemon-export-ok and --export-all.
+    from dulwich.server import (
+        FileSystemBackend,
+        TCPGitServer,
+        )
+    backend = FileSystemBackend(path)
+    server = TCPGitServer(backend, address, port)
+    server.serve_forever()

dulwich/protocol.py

@@ -77,12 +77,23 @@ class Protocol(object):
         Documentation/technical/protocol-common.txt
     """
 
-    def __init__(self, read, write, report_activity=None):
+    def __init__(self, read, write, close=None, report_activity=None):
         self.read = read
         self.write = write
+        self._close = close
         self.report_activity = report_activity
         self._readahead = None
 
+    def close(self):
+        if self._close:
+            self._close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
     def read_pkt_line(self):
         """Reads a pkt-line from the remote git process.
 
@@ -108,7 +119,11 @@ class Protocol(object):
                 return None
             if self.report_activity:
                 self.report_activity(size, 'read')
-            return read(size-4)
+            pkt_contents = read(size-4)
+            if len(pkt_contents) + 4 != size:
+                raise AssertionError('Length of pkt read {:04x} does not match length prefix {:04x}.'
+                                     .format(len(pkt_contents) + 4, size))
+            return pkt_contents
         except socket.error as e:
             raise GitProtocolError(e)
 

dulwich/repo.py

@@ -667,10 +667,12 @@ class Repo(BaseRepo):
         self._graftpoints = {}
         graft_file = self.get_named_file(os.path.join("info", "grafts"))
         if graft_file:
-            self._graftpoints.update(parse_graftpoints(graft_file))
+            with graft_file:
+                self._graftpoints.update(parse_graftpoints(graft_file))
         graft_file = self.get_named_file("shallow")
         if graft_file:
-            self._graftpoints.update(parse_graftpoints(graft_file))
+            with graft_file:
+                self._graftpoints.update(parse_graftpoints(graft_file))
 
         self.hooks['pre-commit'] = PreCommitShellHook(self.controldir())
         self.hooks['commit-msg'] = CommitMsgShellHook(self.controldir())
@@ -741,12 +743,15 @@ class Repo(BaseRepo):
         """
         if isinstance(paths, basestring):
             paths = [paths]
-        from dulwich.index import index_entry_from_stat
+        from dulwich.index import (
+            blob_from_path_and_stat,
+            index_entry_from_stat,
+            )
         index = self.open_index()
         for path in paths:
             full_path = os.path.join(self.path, path)
             try:
-                st = os.stat(full_path)
+                st = os.lstat(full_path)
             except OSError:
                 # File no longer exists
                 try:
@@ -754,12 +759,7 @@ class Repo(BaseRepo):
                 except KeyError:
                     pass  # already removed
             else:
-                blob = Blob()
-                f = open(full_path, 'rb')
-                try:
-                    blob.data = f.read()
-                finally:
-                    f.close()
+                blob = blob_from_path_and_stat(full_path, st)
                 self.object_store.add_object(blob)
                 index[path] = index_entry_from_stat(st, blob.id, 0)
         index.write()

dulwich/server.py

@@ -480,6 +480,8 @@ class ProtocolGraphWalker(object):
             return None
         return self._cache[self._cache_index]
 
+    __next__ = next
+
     def read_proto_line(self, allowed):
         """Read a line from the wire.
 
@@ -599,6 +601,8 @@ class SingleAckGraphWalkerImpl(object):
         elif command == 'have':
             return sha
 
+    __next__ = next
+
 
 class MultiAckGraphWalkerImpl(object):
     """Graph walker implementation that speaks the multi-ack protocol."""
@@ -638,6 +642,8 @@ class MultiAckGraphWalkerImpl(object):
                     self.walker.send_ack(sha, 'continue')
                 return sha
 
+    __next__ = next
+
 
 class MultiAckDetailedGraphWalkerImpl(object):
     """Graph walker implementation speaking the multi-ack-detailed protocol."""
@@ -679,6 +685,8 @@ class MultiAckDetailedGraphWalkerImpl(object):
                     self.walker.send_ack(sha, 'ready')
                 return sha
 
+    __next__ = next
+
 
 class ReceivePackHandler(Handler):
     """Protocol handler for downloading a pack from the client."""
@@ -708,7 +716,7 @@ class ReceivePackHandler(Handler):
             # TODO: more informative error messages than just the exception string
             try:
                 recv = getattr(self.proto, "recv", None)
-                p = self.repo.object_store.add_thin_pack(self.proto.read, recv)
+                self.repo.object_store.add_thin_pack(self.proto.read, recv)
                 status.append(('unpack', 'ok'))
             except all_exceptions as e:
                 status.append(('unpack', str(e).replace('\n', '')))
@@ -863,22 +871,22 @@ def main(argv=sys.argv):
     """Entry point for starting a TCP git server."""
     import optparse
     parser = optparse.OptionParser()
-    parser.add_option("-b", "--backend", dest="backend",
-                      help="Select backend to use.",
-                      choices=["file"], default="file")
+    parser.add_option("-l", "--listen_address", dest="listen_address",
+                      default="localhost",
+                      help="Binding IP address.")
+    parser.add_option("-p", "--port", dest="port", type=int,
+                      default=TCP_GIT_PORT,
+                      help="Binding TCP port.")
     options, args = parser.parse_args(argv)
 
     log_utils.default_logging_config()
-    if options.backend == "file":
-        if len(argv) > 1:
-            gitdir = args[1]
-        else:
-            gitdir = '.'
-        backend = DictBackend({'/': Repo(gitdir)})
+    if len(argv) > 1:
+        gitdir = args[1]
     else:
-        raise Exception("No such backend %s." % backend)
-    server = TCPGitServer(backend, 'localhost')
-    server.serve_forever()
+        gitdir = '.'
+    from dulwich import porcelain
+    porcelain.daemon(gitdir, address=options.listen_address,
+                     port=options.port)
 
 
 def serve_command(handler_cls, argv=sys.argv, backend=None, inf=sys.stdin,
@@ -909,7 +917,7 @@ def serve_command(handler_cls, argv=sys.argv, backend=None, inf=sys.stdin,
 def generate_info_refs(repo):
     """Generate an info refs file."""
     refs = repo.get_refs()
-    return write_info_refs(repo.get_refs(), repo.object_store)
+    return write_info_refs(refs, repo.object_store)
 
 
 def generate_objects_info_packs(repo):

dulwich/tests/__init__.py

@@ -27,13 +27,9 @@ import sys
 import tempfile
 
 
-if sys.version_info >= (2, 7):
-    # If Python itself provides an exception, use that
-    import unittest
-    from unittest import SkipTest, TestCase as _TestCase
-else:
-    import unittest2 as unittest
-    from unittest2 import SkipTest, TestCase as _TestCase
+# If Python itself provides an exception, use that
+import unittest
+from unittest import TestCase as _TestCase
 
 
 def get_safe_env(env=None):
@@ -118,6 +114,7 @@ def self_test_suite():
         'fastexport',
         'file',
         'grafts',
+        'greenthreads',
         'hooks',
         'index',
         'lru_cache',
@@ -164,7 +161,9 @@ def tutorial_test_suite():
 def nocompat_test_suite():
     result = unittest.TestSuite()
     result.addTests(self_test_suite())
+    from dulwich.contrib import test_suite as contrib_test_suite
     result.addTests(tutorial_test_suite())
+    result.addTests(contrib_test_suite())
     return result
 
 
@@ -181,4 +180,6 @@ def test_suite():
     result.addTests(tutorial_test_suite())
     from dulwich.tests.compat import test_suite as compat_test_suite
     result.addTests(compat_test_suite())
+    from dulwich.contrib import test_suite as contrib_test_suite
+    result.addTests(contrib_test_suite())
     return result

dulwich/tests/compat/server_utils.py

@@ -21,11 +21,9 @@
 
 import errno
 import os
-import select
 import shutil
 import socket
 import tempfile
-import threading
 
 from dulwich.repo import Repo
 from dulwich.objects import hex_to_sha
@@ -119,7 +117,7 @@ class ServerTests(object):
         run_git_or_fail(['push', self.url(port), ":master"],
                         cwd=self._new_repo.path)
 
-        self.assertEquals(
+        self.assertEqual(
             self._old_repo.get_refs().keys(), ["refs/heads/branch"])
 
     def test_fetch_from_dulwich(self):

dulwich/tests/compat/test_client.py

@@ -32,6 +32,7 @@ import tarfile
 import tempfile
 import threading
 import urllib
+from unittest import SkipTest
 
 from dulwich import (
     client,
@@ -44,7 +45,6 @@ from dulwich import (
     )
 from dulwich.tests import (
     get_safe_env,
-    SkipTest,
     )
 
 from dulwich.tests.compat.utils import (
@@ -152,16 +152,18 @@ class DulwichClientTestBase(object):
     def test_send_pack_multiple_errors(self):
         dest, dummy = self.disable_ff_and_make_dummy_commit()
         # set up for two non-ff errors
-        dest.refs['refs/heads/branch'] = dest.refs['refs/heads/master'] = dummy
+        branch, master = 'refs/heads/branch', 'refs/heads/master'
+        dest.refs[branch] = dest.refs[master] = dummy
         sendrefs, gen_pack = self.compute_send()
         c = self._client()
         try:
             c.send_pack(self._build_path('/dest'), lambda _: sendrefs, gen_pack)
         except errors.UpdateRefsError as e:
-            self.assertEqual('refs/heads/branch, refs/heads/master failed to '
-                             'update', str(e))
-            self.assertEqual({'refs/heads/branch': 'non-fast-forward',
-                              'refs/heads/master': 'non-fast-forward'},
+            self.assertIn(str(e),
+                          ['{0}, {1} failed to update'.format(branch, master),
+                           '{1}, {0} failed to update'.format(branch, master)])
+            self.assertEqual({branch: 'non-fast-forward',
+                              master: 'non-fast-forward'},
                              e.ref_status)
 
     def test_archive(self):
@@ -176,7 +178,8 @@ class DulwichClientTestBase(object):
         c = self._client()
         dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
         refs = c.fetch(self._build_path('/server_new.export'), dest)
-        map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
+        for r in refs.items():
+            dest.refs.set_if_equals(r[0], None, r[1])
         self.assertDestEqualsSrc()
 
     def test_incremental_fetch_pack(self):
@@ -186,7 +189,8 @@ class DulwichClientTestBase(object):
         c = self._client()
         dest = repo.Repo(os.path.join(self.gitroot, 'server_new.export'))
         refs = c.fetch(self._build_path('/dest'), dest)
-        map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
+        for r in refs.items():
+            dest.refs.set_if_equals(r[0], None, r[1])
         self.assertDestEqualsSrc()
 
     def test_fetch_pack_no_side_band_64k(self):
@@ -194,7 +198,8 @@ class DulwichClientTestBase(object):
         c._fetch_capabilities.remove('side-band-64k')
         dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
         refs = c.fetch(self._build_path('/server_new.export'), dest)
-        map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
+        for r in refs.items():
+            dest.refs.set_if_equals(r[0], None, r[1])
         self.assertDestEqualsSrc()
 
     def test_fetch_pack_zero_sha(self):
@@ -204,7 +209,8 @@ class DulwichClientTestBase(object):
         dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
         refs = c.fetch(self._build_path('/server_new.export'), dest,
             lambda refs: [protocol.ZERO_SHA])
-        map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
+        for r in refs.items():
+            dest.refs.set_if_equals(r[0], None, r[1])
 
     def test_send_remove_branch(self):
         dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
@@ -242,7 +248,9 @@ class DulwichTCPClientTest(CompatTestCase, DulwichClientTestBase):
 
     def tearDown(self):
         try:
-            os.kill(int(open(self.pidfile).read().strip()), signal.SIGKILL)
+            with open(self.pidfile) as f:
+                pid = f.read()
+            os.kill(int(pid.strip()), signal.SIGKILL)
             os.unlink(self.pidfile)
         except (OSError, IOError):
             pass
@@ -454,6 +462,8 @@ class DulwichHttpClientTest(CompatTestCase, DulwichClientTestBase):
     def tearDown(self):
         DulwichClientTestBase.tearDown(self)
         CompatTestCase.tearDown(self)
+        self._httpd.shutdown()
+        self._httpd.socket.close()
 
     def _client(self):
         return client.HttpGitClient(self._httpd.get_url())

dulwich/tests/compat/test_pack.py

@@ -22,13 +22,19 @@
 
 import binascii
 import os
+import re
 import shutil
 import tempfile
+from unittest import SkipTest
 
 from dulwich.pack import (
     write_pack,
     )
+from dulwich.objects import (
+    Blob,
+    )
 from dulwich.tests.test_pack import (
+    a_sha,
     pack1_sha,
     PackTests,
     )
@@ -37,6 +43,19 @@ from dulwich.tests.compat.utils import (
     run_git_or_fail,
     )
 
+_NON_DELTA_RE = re.compile('non delta: (?P<non_delta>\d+) objects')
+
+def _git_verify_pack_object_list(output):
+    pack_shas = set()
+    for line in output.splitlines():
+        sha = line[:40]
+        try:
+            binascii.unhexlify(sha)
+        except (TypeError, binascii.Error):
+            continue  # non-sha line
+        pack_shas.add(sha)
+    return pack_shas
+
 
 class TestPack(PackTests):
     """Compatibility tests for reading and writing pack files."""
@@ -48,19 +67,87 @@ class TestPack(PackTests):
         self.addCleanup(shutil.rmtree, self._tempdir)
 
     def test_copy(self):
-        origpack = self.get_pack(pack1_sha)
-        self.assertSucceeds(origpack.index.check)
-        pack_path = os.path.join(self._tempdir, "Elch")
-        write_pack(pack_path, origpack.pack_tuples())
+        with self.get_pack(pack1_sha) as origpack:
+            self.assertSucceeds(origpack.index.check)
+            pack_path = os.path.join(self._tempdir, "Elch")
+            write_pack(pack_path, origpack.pack_tuples())
+            output = run_git_or_fail(['verify-pack', '-v', pack_path])
+            orig_shas = set(o.id for o in origpack.iterobjects())
+            self.assertEqual(orig_shas, _git_verify_pack_object_list(output))
+
+    def test_deltas_work(self):
+        orig_pack = self.get_pack(pack1_sha)
+        orig_blob = orig_pack[a_sha]
+        new_blob = Blob()
+        new_blob.data = orig_blob.data + 'x'
+        all_to_pack = list(orig_pack.pack_tuples()) + [(new_blob, None)]
+        pack_path = os.path.join(self._tempdir, "pack_with_deltas")
+        write_pack(pack_path, all_to_pack, deltify=True)
         output = run_git_or_fail(['verify-pack', '-v', pack_path])
+        self.assertEqual(set(x[0].id for x in all_to_pack),
+                         _git_verify_pack_object_list(output))
+        # We specifically made a new blob that should be a delta
+        # against the blob a_sha, so make sure we really got only 3
+        # non-delta objects:
+        got_non_delta = int(_NON_DELTA_RE.search(output).group('non_delta'))
+        self.assertEqual(
+            3, got_non_delta,
+            'Expected 3 non-delta objects, got %d' % got_non_delta)
 
-        pack_shas = set()
-        for line in output.splitlines():
-            sha = line[:40]
-            try:
-                binascii.unhexlify(sha)
-            except TypeError:
-                continue  # non-sha line
-            pack_shas.add(sha)
-        orig_shas = set(o.id for o in origpack.iterobjects())
-        self.assertEqual(orig_shas, pack_shas)
+    def test_delta_medium_object(self):
+        # This tests an object set that will have a copy operation
+        # 2**20 in size.
+        orig_pack = self.get_pack(pack1_sha)
+        orig_blob = orig_pack[a_sha]
+        new_blob = Blob()
+        new_blob.data = orig_blob.data + ('x' * 2 ** 20)
+        new_blob_2 = Blob()
+        new_blob_2.data = new_blob.data + 'y'
+        all_to_pack = list(orig_pack.pack_tuples()) + [(new_blob, None),
+                                                       (new_blob_2, None)]
+        pack_path = os.path.join(self._tempdir, "pack_with_deltas")
+        write_pack(pack_path, all_to_pack, deltify=True)
+        output = run_git_or_fail(['verify-pack', '-v', pack_path])
+        self.assertEqual(set(x[0].id for x in all_to_pack),
+                         _git_verify_pack_object_list(output))
+        # We specifically made a new blob that should be a delta
+        # against the blob a_sha, so make sure we really got only 3
+        # non-delta objects:
+        got_non_delta = int(_NON_DELTA_RE.search(output).group('non_delta'))
+        self.assertEqual(
+            3, got_non_delta,
+            'Expected 3 non-delta objects, got %d' % got_non_delta)
+        # We expect one object to have a delta chain length of two
+        # (new_blob_2), so let's verify that actually happens:
+        self.assertIn('chain length = 2', output)
+
+    # This test is SUPER slow: over 80 seconds on a 2012-era
+    # laptop. This is because SequenceMatcher is worst-case quadratic
+    # on the input size. It's impractical to produce deltas for
+    # objects this large, but it's still worth doing the right thing
+    # when it happens.
+    def test_delta_large_object(self):
+        # This tests an object set that will have a copy operation
+        # 2**25 in size. This is a copy large enough that it requires
+        # two copy operations in git's binary delta format.
+        raise SkipTest('skipping slow, large test')
+        orig_pack = self.get_pack(pack1_sha)
+        orig_blob = orig_pack[a_sha]
+        new_blob = Blob()
+        new_blob.data = 'big blob' + ('x' * 2 ** 25)
+        new_blob_2 = Blob()
+        new_blob_2.data = new_blob.data + 'y'
+        all_to_pack = list(orig_pack.pack_tuples()) + [(new_blob, None),
+                                                       (new_blob_2, None)]
+        pack_path = os.path.join(self._tempdir, "pack_with_deltas")
+        write_pack(pack_path, all_to_pack, deltify=True)
+        output = run_git_or_fail(['verify-pack', '-v', pack_path])
+        self.assertEqual(set(x[0].id for x in all_to_pack),
+                         _git_verify_pack_object_list(output))
+        # We specifically made a new blob that should be a delta
+        # against the blob a_sha, so make sure we really got only 4
+        # non-delta objects:
+        got_non_delta = int(_NON_DELTA_RE.search(output).group('non_delta'))
+        self.assertEqual(
+            4, got_non_delta,
+            'Expected 4 non-delta objects, got %d' % got_non_delta)

dulwich/tests/compat/test_repository.py

@@ -21,7 +21,7 @@
 
 
 from io import BytesIO
-import itertools
+from itertools import chain
 import os
 
 from dulwich.objects import (
@@ -118,7 +118,7 @@ class ObjectStoreTestCase(CompatTestCase):
     def test_packed_objects(self):
         expected_shas = self._get_all_shas() - self._get_loose_shas()
         self.assertShasMatch(expected_shas,
-                             itertools.chain(*self._repo.object_store.packs))
+                             chain(*self._repo.object_store.packs))
 
     def test_all_objects(self):
         expected_shas = self._get_all_shas()

dulwich/tests/compat/test_server.py

@@ -25,6 +25,7 @@ Warning: these tests should be fairly stable, but when writing/debugging new
 """
 
 import threading
+import os
 
 from dulwich.server import (
     DictBackend,
@@ -36,6 +37,7 @@ from dulwich.tests.compat.server_utils import (
     )
 from dulwich.tests.compat.utils import (
     CompatTestCase,
+    require_git_version,
     )
 
 
@@ -61,6 +63,7 @@ class GitServerTestCase(ServerTests, CompatTestCase):
                                   handlers=self._handlers())
         self._check_server(dul_server)
         self.addCleanup(dul_server.shutdown)
+        self.addCleanup(dul_server.server_close)
         threading.Thread(target=dul_server.serve).start()
         self._server = dul_server
         _, port = self._server.socket.getsockname()
@@ -73,6 +76,15 @@ class GitServerSideBand64kTestCase(GitServerTestCase):
     # side-band-64k in git-receive-pack was introduced in git 1.7.0.2
     min_git_version = (1, 7, 0, 2)
 
+    def setUp(self):
+        super(GitServerSideBand64kTestCase, self).setUp()
+        # side-band-64k is broken in the widows client.
+        # https://github.com/msysgit/git/issues/101
+        # Fix has landed for the 1.9.3 release.
+        if os.name == 'nt':
+            require_git_version((1, 9, 3))
+
+
     def _handlers(self):
         return None  # default handlers include side-band-64k
 

dulwich/tests/compat/test_utils.py

@@ -19,8 +19,9 @@
 
 """Tests for git compatibility utilities."""
 
+from unittest import SkipTest
+
 from dulwich.tests import (
-    SkipTest,
     TestCase,
     )
 from dulwich.tests.compat import utils

dulwich/tests/compat/test_web.py

@@ -25,14 +25,14 @@ warning: these tests should be fairly stable, but when writing/debugging new
 """
 
 import threading
+from unittest import (
+    SkipTest,
+    )
 from wsgiref import simple_server
 
 from dulwich.server import (
     DictBackend,
     )
-from dulwich.tests import (
-    SkipTest,
-    )
 from dulwich.web import (
     make_wsgi_chain,
     HTTPGitApplication,
@@ -65,6 +65,7 @@ class WebTests(ServerTests):
           'localhost', 0, app, server_class=WSGIServerLogger,
           handler_class=WSGIRequestHandlerLogger)
         self.addCleanup(dul_server.shutdown)
+        self.addCleanup(dul_server.server_close)
         threading.Thread(target=dul_server.serve_forever).start()
         self._server = dul_server
         _, port = dul_server.socket.getsockname()

dulwich/tests/compat/utils.py

@@ -25,13 +25,13 @@ import socket
 import subprocess
 import tempfile
 import time
+from unittest import SkipTest
 
 from dulwich.repo import Repo
 from dulwich.protocol import TCP_GIT_PORT
 
 from dulwich.tests import (
     get_safe_env,
-    SkipTest,
     TestCase,
     )
 
@@ -193,13 +193,14 @@ def check_for_daemon(limit=10, delay=0.1, timeout=0.1, port=TCP_GIT_PORT):
         s.settimeout(delay)
         try:
             s.connect(('localhost', port))
-            s.close()
             return True
         except socket.error as e:
             if getattr(e, 'errno', False) and e.errno != errno.ECONNREFUSED:
                 raise
             elif e.args[0] != errno.ECONNREFUSED:
                 raise
+        finally:
+            s.close()
     return False
 
 

dulwich/tests/data/repos/.gitattributes

@@ -0,0 +1 @@
+*.export eol=lf

dulwich/tests/test_client.py

@@ -177,11 +177,14 @@ class GitClientTests(TestCase):
             return {}
 
         self.client.send_pack('/', determine_wants, generate_pack_contents)
-        self.assertEqual(
+        self.assertIn(
             self.rout.getvalue(),
-            '007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
-            '0000000000000000000000000000000000000000 '
-            'refs/heads/master\x00report-status ofs-delta0000')
+            ['007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
+             '0000000000000000000000000000000000000000 '
+             'refs/heads/master\x00report-status ofs-delta0000',
+             '007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
+             '0000000000000000000000000000000000000000 '
+             'refs/heads/master\x00ofs-delta report-status0000'])
 
     def test_send_pack_new_ref_only(self):
         self.rin.write(
@@ -203,14 +206,18 @@ class GitClientTests(TestCase):
             return {}
 
         f = BytesIO()
-        empty_pack = write_pack_objects(f, {})
+        write_pack_objects(f, {})
         self.client.send_pack('/', determine_wants, generate_pack_contents)
-        self.assertEqual(
+        self.assertIn(
             self.rout.getvalue(),
-            '007f0000000000000000000000000000000000000000 '
-            '310ca9477129b8586fa2afc779c1f57cf64bba6c '
-            'refs/heads/blah12\x00report-status ofs-delta0000%s'
-            % f.getvalue())
+            ['007f0000000000000000000000000000000000000000 '
+             '310ca9477129b8586fa2afc779c1f57cf64bba6c '
+             'refs/heads/blah12\x00report-status ofs-delta0000%s'
+             % f.getvalue(),
+             '007f0000000000000000000000000000000000000000 '
+             '310ca9477129b8586fa2afc779c1f57cf64bba6c '
+             'refs/heads/blah12\x00ofs-delta report-status0000%s'
+             % f.getvalue()])
 
     def test_send_pack_new_ref(self):
         self.rin.write(
@@ -241,13 +248,16 @@ class GitClientTests(TestCase):
             return [(commit, None), (tree, ''), ]
 
         f = BytesIO()
-        pack = write_pack_objects(f, generate_pack_contents(None, None))
+        write_pack_objects(f, generate_pack_contents(None, None))
         self.client.send_pack('/', determine_wants, generate_pack_contents)
-        self.assertEqual(
+        self.assertIn(
             self.rout.getvalue(),
-            '007f0000000000000000000000000000000000000000 %s '
-            'refs/heads/blah12\x00report-status ofs-delta0000%s'
-            % (commit.id, f.getvalue()))
+            ['007f0000000000000000000000000000000000000000 %s '
+             'refs/heads/blah12\x00report-status ofs-delta0000%s'
+             % (commit.id, f.getvalue()),
+             '007f0000000000000000000000000000000000000000 %s '
+             'refs/heads/blah12\x00ofs-delta report-status0000%s'
+             % (commit.id, f.getvalue())])
 
     def test_send_pack_no_deleteref_delete_only(self):
         pkts = ['310ca9477129b8586fa2afc779c1f57cf64bba6c refs/heads/master'
@@ -482,6 +492,7 @@ class TestSSHVendor(object):
         class Subprocess: pass
         setattr(Subprocess, 'read', lambda: None)
         setattr(Subprocess, 'write', lambda: None)
+        setattr(Subprocess, 'close', lambda: None)
         setattr(Subprocess, 'can_read', lambda: None)
         return Subprocess()
 
@@ -519,12 +530,12 @@ class SSHGitClientTests(TestCase):
         client.port = 1337
 
         client._connect("command", "/path/to/repo")
-        self.assertEquals("username", server.username)
-        self.assertEquals(1337, server.port)
-        self.assertEquals(["git-command '/path/to/repo'"], server.command)
+        self.assertEqual("username", server.username)
+        self.assertEqual(1337, server.port)
+        self.assertEqual(["git-command '/path/to/repo'"], server.command)
 
         client._connect("relative-command", "/~/path/to/repo")
-        self.assertEquals(["git-relative-command '~/path/to/repo'"],
+        self.assertEqual(["git-relative-command '~/path/to/repo'"],
                           server.command)
 
 
@@ -558,7 +569,7 @@ class LocalGitClientTests(TestCase):
         c = LocalGitClient()
         t = MemoryRepo()
         s = open_repo('a.git')
-        self.assertEquals(s.get_refs(), c.fetch(s.path, t))
+        self.assertEqual(s.get_refs(), c.fetch(s.path, t))
 
     def test_fetch_empty(self):
         c = LocalGitClient()
@@ -567,7 +578,7 @@ class LocalGitClientTests(TestCase):
         walker = {}
         c.fetch_pack(s.path, lambda heads: [], graph_walker=walker,
             pack_data=out.write)
-        self.assertEquals("PACK\x00\x00\x00\x02\x00\x00\x00\x00\x02\x9d\x08"
+        self.assertEqual("PACK\x00\x00\x00\x02\x00\x00\x00\x00\x02\x9d\x08"
             "\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e", out.getvalue())
 
     def test_fetch_pack_none(self):

dulwich/tests/test_config.py

@@ -22,7 +22,6 @@ from io import BytesIO
 from dulwich.config import (
     ConfigDict,
     ConfigFile,
-    OrderedDict,
     StackedConfig,
     _check_section_name,
     _check_variable_name,

dulwich/tests/test_diff_tree.py

@@ -18,10 +18,7 @@
 
 """Tests for file and tree diff utilities."""
 
-from itertools import (
-    permutations,
-    )
-
+from itertools import permutations
 from dulwich.diff_tree import (
     CHANGE_MODIFY,
     CHANGE_RENAME,
@@ -755,6 +752,18 @@ class RenameDetectionTest(DiffTestCase):
            TreeChange.delete(('b', F, blob2.id))],
           self.detect_renames(tree1, tree2))
 
+    def test_content_rename_with_more_deletions(self):
+        blob1 = make_object(Blob, data='')
+        tree1 = self.commit_tree([('a', blob1), ('b', blob1), ('c', blob1), ('d', blob1)])
+        tree2 = self.commit_tree([('e', blob1), ('f', blob1), ('g', blob1)])
+        self.maxDiff = None
+        self.assertEqual(
+          [TreeChange(CHANGE_RENAME, ('a', F, blob1.id), ('e', F, blob1.id)),
+           TreeChange(CHANGE_RENAME, ('b', F, blob1.id), ('f', F, blob1.id)),
+           TreeChange(CHANGE_RENAME, ('c', F, blob1.id), ('g', F, blob1.id)),
+           TreeChange.delete(('d', F, blob1.id))],
+          self.detect_renames(tree1, tree2))
+
     def test_content_rename_gitlink(self):
         blob1 = make_object(Blob, data='blob1')
         blob2 = make_object(Blob, data='blob2')
@@ -872,7 +881,6 @@ class RenameDetectionTest(DiffTestCase):
         blob_c2 = make_object(Blob, data='a\nb\nc\ne\n')
         tree1 = self.commit_tree([('a', blob_a1), ('b', blob_b)])
         tree2 = self.commit_tree([('c', blob_c2), ('b', blob_b)])
-        detector = RenameDetector(self.store)
         self.assertEqual(
           [TreeChange(CHANGE_RENAME, ('a', F, blob_a1.id),
                       ('c', F, blob_c2.id))],

dulwich/tests/test_fastexport.py

@@ -19,6 +19,7 @@
 
 from io import BytesIO
 import stat
+from unittest import SkipTest
 
 
 from dulwich.object_store import (
@@ -33,7 +34,6 @@ from dulwich.repo import (
     MemoryRepo,
     )
 from dulwich.tests import (
-    SkipTest,
     TestCase,
     )
 from dulwich.tests.utils import (
@@ -105,7 +105,7 @@ class GitImportProcessorTests(TestCase):
         [c1] = build_commit_graph(self.repo.object_store, [[1]])
         cmd = commands.ResetCommand("refs/heads/foo", c1.id)
         self.processor.reset_handler(cmd)
-        self.assertEquals(c1.id, self.repo.get_refs()["refs/heads/foo"])
+        self.assertEqual(c1.id, self.repo.get_refs()["refs/heads/foo"])
 
     def test_commit_handler(self):
         from fastimport import commands

dulwich/tests/test_file.py

@@ -17,14 +17,15 @@
 # MA  02110-1301, USA.
 
 import errno
+import io
 import os
 import shutil
 import sys
 import tempfile
+from unittest import SkipTest
 
 from dulwich.file import GitFile, fancy_rename
 from dulwich.tests import (
-    SkipTest,
     TestCase,
     )
 
@@ -58,7 +59,7 @@ class FancyRenameTests(TestCase):
         new_f = open(self.bar, 'rb')
         self.assertEqual('foo contents', new_f.read())
         new_f.close()
-         
+
     def test_dest_exists(self):
         self.create(self.bar, 'bar contents')
         fancy_rename(self.foo, self.bar)
@@ -112,7 +113,7 @@ class GitFileTests(TestCase):
 
     def test_readonly(self):
         f = GitFile(self.path('foo'), 'rb')
-        self.assertTrue(isinstance(f, file))
+        self.assertTrue(isinstance(f, io.IOBase))
         self.assertEqual('foo contents', f.read())
         self.assertEqual('', f.read())
         f.seek(4)
@@ -157,6 +158,8 @@ class GitFileTests(TestCase):
             self.fail()
         except OSError as e:
             self.assertEqual(errno.EEXIST, e.errno)
+        else:
+            f2.close()
         f1.write(' contents')
         f1.close()
 

dulwich/tests/test_greenthreads.py

@@ -0,0 +1,135 @@
+# test_greenthreads.py -- Unittests for eventlet.
+# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+#
+# Author: Fabien Boucher <fabien.boucher@enovance.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) any later version of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+import time
+
+from dulwich.tests import (
+    TestCase,
+    )
+from dulwich.object_store import (
+    MemoryObjectStore,
+    MissingObjectFinder,
+    )
+from dulwich.objects import (
+    Commit,
+    Blob,
+    Tree,
+    parse_timezone,
+    )
+
+from unittest import skipIf
+
+try:
+    import gevent
+    gevent_support = True
+except ImportError:
+    gevent_support = False
+
+if gevent_support:
+    from dulwich.greenthreads import (
+        GreenThreadsObjectStoreIterator,
+        GreenThreadsMissingObjectFinder,
+    )
+
+skipmsg = "Gevent library is not installed"
+
+def create_commit(marker=None):
+    blob = Blob.from_string('The blob content %s' % marker)
+    tree = Tree()
+    tree.add("thefile %s" % marker, 0o100644, blob.id)
+    cmt = Commit()
+    cmt.tree = tree.id
+    cmt.author = cmt.committer = "John Doe <john@doe.net>"
+    cmt.message = "%s" % marker
+    tz = parse_timezone('-0200')[0]
+    cmt.commit_time = cmt.author_time = int(time.time())
+    cmt.commit_timezone = cmt.author_timezone = tz
+    return cmt, tree, blob
+
+
+def init_store(store, count=1):
+    ret = []
+    for i in xrange(0, count):
+        objs = create_commit(marker=i)
+        for obj in objs:
+            ret.append(obj)
+            store.add_object(obj)
+    return ret
+
+
+@skipIf(not gevent_support, skipmsg)
+class TestGreenThreadsObjectStoreIterator(TestCase):
+
+    def setUp(self):
+        super(TestGreenThreadsObjectStoreIterator, self).setUp()
+        self.store = MemoryObjectStore()
+        self.cmt_amount = 10
+        self.objs = init_store(self.store, self.cmt_amount)
+
+    def test_len(self):
+        wants = [sha.id for sha in self.objs if isinstance(sha, Commit)]
+        finder = MissingObjectFinder(self.store, (), wants)
+        iterator = GreenThreadsObjectStoreIterator(self.store,
+                                               iter(finder.next, None),
+                                               finder)
+        # One commit refers one tree and one blob
+        self.assertEqual(len(iterator), self.cmt_amount * 3)
+        haves = wants[0:self.cmt_amount-1]
+        finder = MissingObjectFinder(self.store, haves, wants)
+        iterator = GreenThreadsObjectStoreIterator(self.store,
+                                               iter(finder.next, None),
+                                               finder)
+        self.assertEqual(len(iterator), 3)
+
+    def test_iter(self):
+        wants = [sha.id for sha in self.objs if isinstance(sha, Commit)]
+        finder = MissingObjectFinder(self.store, (), wants)
+        iterator = GreenThreadsObjectStoreIterator(self.store,
+                                               iter(finder.next, None),
+                                               finder)
+        objs = []
+        for sha, path in iterator:
+            self.assertIn(sha, self.objs)
+            objs.append(sha)
+        self.assertEqual(len(objs), len(self.objs))
+
+
+@skipIf(not gevent_support, skipmsg)
+class TestGreenThreadsMissingObjectFinder(TestCase):
+
+    def setUp(self):
+        super(TestGreenThreadsMissingObjectFinder, self).setUp()
+        self.store = MemoryObjectStore()
+        self.cmt_amount = 10
+        self.objs = init_store(self.store, self.cmt_amount)
+
+    def test_finder(self):
+        wants = [sha.id for sha in self.objs if isinstance(sha, Commit)]
+        finder = GreenThreadsMissingObjectFinder(self.store, (), wants)
+        self.assertEqual(len(finder.sha_done), 0)
+        self.assertEqual(len(finder.objects_to_send), self.cmt_amount)
+
+        finder = GreenThreadsMissingObjectFinder(self.store,
+                                             wants[0:self.cmt_amount/2],
+                                             wants)
+        # sha_done will contains commit id and sha of blob refered in tree
+        self.assertEqual(len(finder.sha_done), (self.cmt_amount/2)*2)
+        self.assertEqual(len(finder.objects_to_send), self.cmt_amount/2)

dulwich/tests/test_hooks.py

@@ -21,7 +21,6 @@ import os
 import stat
 import shutil
 import tempfile
-import warnings
 
 from dulwich import errors
 

dulwich/tests/test_index.py

@@ -31,10 +31,13 @@ from dulwich.index import (
     build_index_from_tree,
     cleanup_mode,
     commit_tree,
+    get_unstaged_changes,
     index_entry_from_stat,
     read_index,
+    read_index_dict,
     write_cache_time,
     write_index,
+    write_index_dict,
     )
 from dulwich.object_store import (
     MemoryObjectStore,
@@ -82,6 +85,7 @@ class SimpleIndexTestCase(IndexTestCase):
         self.assertEqual('bla', newname)
         self.assertEqual('e69de29bb2d1d6434b8b29ae775ad8c2e48c5391', newsha)
 
+
 class SimpleIndexWriterTestCase(IndexTestCase):
 
     def setUp(self):
@@ -109,6 +113,33 @@ class SimpleIndexWriterTestCase(IndexTestCase):
             x.close()
 
 
+class ReadIndexDictTests(IndexTestCase):
+
+    def setUp(self):
+        IndexTestCase.setUp(self)
+        self.tempdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        IndexTestCase.tearDown(self)
+        shutil.rmtree(self.tempdir)
+
+    def test_simple_write(self):
+        entries = {'barbla': ((1230680220, 0), (1230680220, 0), 2050, 3761020,
+                    33188, 1000, 1000, 0,
+                    'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391', 0)}
+        filename = os.path.join(self.tempdir, 'test-simple-write-index')
+        x = open(filename, 'w+')
+        try:
+            write_index_dict(x, entries)
+        finally:
+            x.close()
+        x = open(filename, 'r')
+        try:
+            self.assertEqual(entries, read_index_dict(x))
+        finally:
+            x.close()
+
+
 class CommitTreeTests(TestCase):
 
     def setUp(self):
@@ -221,17 +252,17 @@ class IndexEntryFromStatTests(TestCase):
 class BuildIndexTests(TestCase):
 
     def assertReasonableIndexEntry(self, index_entry, mode, filesize, sha):
-        self.assertEquals(index_entry[4], mode)  # mode
-        self.assertEquals(index_entry[7], filesize)  # filesize
-        self.assertEquals(index_entry[8], sha)  # sha
+        self.assertEqual(index_entry[4], mode)  # mode
+        self.assertEqual(index_entry[7], filesize)  # filesize
+        self.assertEqual(index_entry[8], sha)  # sha
 
     def assertFileContents(self, path, contents, symlink=False):
         if symlink:
-            self.assertEquals(os.readlink(path), contents)
+            self.assertEqual(os.readlink(path), contents)
         else:
             f = open(path, 'rb')
             try:
-                self.assertEquals(f.read(), contents)
+                self.assertEqual(f.read(), contents)
             finally:
                 f.close()
 
@@ -248,10 +279,10 @@ class BuildIndexTests(TestCase):
 
         # Verify index entries
         index = repo.open_index()
-        self.assertEquals(len(index), 0)
+        self.assertEqual(len(index), 0)
 
         # Verify no files
-        self.assertEquals(['.git'], os.listdir(repo.path))
+        self.assertEqual(['.git'], os.listdir(repo.path))
 
     def test_nonempty(self):
         if os.name != 'posix':
@@ -281,7 +312,7 @@ class BuildIndexTests(TestCase):
 
         # Verify index entries
         index = repo.open_index()
-        self.assertEquals(len(index), 4)
+        self.assertEqual(len(index), 4)
 
         # filea
         apath = os.path.join(repo.path, 'a')
@@ -300,19 +331,51 @@ class BuildIndexTests(TestCase):
         # filed
         dpath = os.path.join(repo.path, 'c', 'd')
         self.assertTrue(os.path.exists(dpath))
-        self.assertReasonableIndexEntry(index['c/d'], 
+        self.assertReasonableIndexEntry(index['c/d'],
             stat.S_IFREG | 0o644, 6, filed.id)
         self.assertFileContents(dpath, 'file d')
 
         # symlink to d
         epath = os.path.join(repo.path, 'c', 'e')
         self.assertTrue(os.path.exists(epath))
-        self.assertReasonableIndexEntry(index['c/e'], 
+        self.assertReasonableIndexEntry(index['c/e'],
             stat.S_IFLNK, 1, filee.id)
         self.assertFileContents(epath, 'd', symlink=True)
 
         # Verify no extra files
-        self.assertEquals(['.git', 'a', 'b', 'c'],
+        self.assertEqual(['.git', 'a', 'b', 'c'],
             sorted(os.listdir(repo.path)))
-        self.assertEquals(['d', 'e'], 
+        self.assertEqual(['d', 'e'],
             sorted(os.listdir(os.path.join(repo.path, 'c'))))
+
+
+class GetUnstagedChangesTests(TestCase):
+
+    def test_get_unstaged_changes(self):
+        """Unit test for get_unstaged_changes."""
+
+        repo_dir = tempfile.mkdtemp()
+        repo = Repo.init(repo_dir)
+        self.addCleanup(shutil.rmtree, repo_dir)
+
+        # Commit a dummy file then modify it
+        foo1_fullpath = os.path.join(repo_dir, 'foo1')
+        with open(foo1_fullpath, 'w') as f:
+            f.write('origstuff')
+
+        foo2_fullpath = os.path.join(repo_dir, 'foo2')
+        with open(foo2_fullpath, 'w') as f:
+            f.write('origstuff')
+
+        repo.stage(['foo1', 'foo2'])
+        repo.do_commit('test status', author='', committer='')
+
+        with open(foo1_fullpath, 'w') as f:
+            f.write('newstuff')
+
+        # modify access and modify time of path
+        os.utime(foo1_fullpath, (0, 0))
+
+        changes = get_unstaged_changes(repo.open_index(), repo_dir)
+
+        self.assertEqual(list(changes), ['foo1'])

dulwich/tests/test_lru_cache.py

@@ -40,18 +40,18 @@ class TestLRUCache(TestCase):
     def test_missing(self):
         cache = lru_cache.LRUCache(max_cache=10)
 
-        self.failIf('foo' in cache)
+        self.assertFalse('foo' in cache)
         self.assertRaises(KeyError, cache.__getitem__, 'foo')
 
         cache['foo'] = 'bar'
         self.assertEqual('bar', cache['foo'])
-        self.failUnless('foo' in cache)
-        self.failIf('bar' in cache)
+        self.assertTrue('foo' in cache)
+        self.assertFalse('bar' in cache)
 
     def test_map_None(self):
         # Make sure that we can properly map None as a key.
         cache = lru_cache.LRUCache(max_cache=10)
-        self.failIf(None in cache)
+        self.assertFalse(None in cache)
         cache[None] = 1
         self.assertEqual(1, cache[None])
         cache[None] = 2
@@ -77,8 +77,8 @@ class TestLRUCache(TestCase):
         # With a max cache of 1, adding 'baz' should pop out 'foo'
         cache['baz'] = 'biz'
 
-        self.failIf('foo' in cache)
-        self.failUnless('baz' in cache)
+        self.assertFalse('foo' in cache)
+        self.assertTrue('baz' in cache)
 
         self.assertEqual('biz', cache['baz'])
 
@@ -94,7 +94,7 @@ class TestLRUCache(TestCase):
         # This must kick out 'foo' because it was the last accessed
         cache['nub'] = 'in'
 
-        self.failIf('foo' in cache)
+        self.assertFalse('foo' in cache)
 
     def test_cleanup(self):
         """Test that we can use a cleanup function."""
@@ -102,7 +102,7 @@ class TestLRUCache(TestCase):
         def cleanup_func(key, val):
             cleanup_called.append((key, val))
 
-        cache = lru_cache.LRUCache(max_cache=2)
+        cache = lru_cache.LRUCache(max_cache=2, after_cleanup_count=2)
 
         cache.add('baz', '1', cleanup=cleanup_func)
         cache.add('foo', '2', cleanup=cleanup_func)
@@ -188,7 +188,7 @@ class TestLRUCache(TestCase):
         # By default _after_cleanup_size is 80% of the normal size
         self.assertEqual(4, cache._after_cleanup_count)
 
-    def test_cleanup(self):
+    def test_cleanup_2(self):
         cache = lru_cache.LRUCache(max_cache=5, after_cleanup_count=2)
 
         # Add these in order

dulwich/tests/test_missing_obj_finder.py

@@ -45,7 +45,7 @@ class MissingObjectFinderTest(TestCase):
                 "(%s,%s) erroneously reported as missing" % (sha, path))
             expected.remove(sha)
 
-        self.assertEquals(len(expected), 0,
+        self.assertEqual(len(expected), 0,
             "some objects are not reported as missing: %s" % (expected, ))
 
 
@@ -143,7 +143,7 @@ class MOFMergeForkRepoTest(MissingObjectFinderTest):
         self.f2_3_id = f2_3.id
         self.f3_3_id = f3_3.id
 
-        self.assertEquals(f1_2.id, f1_7.id, "[sanity]")
+        self.assertEqual(f1_2.id, f1_7.id, "[sanity]")
 
     def test_have6_want7(self):
         # have 6, want 7. Ideally, shall not report f1_7 as it's the same as

dulwich/tests/test_object_store.py

@@ -243,7 +243,7 @@ class PackBasedObjectStoreTests(ObjectStoreTests):
         self.store.add_object(b2)
         self.assertEqual([], self.store.packs)
         self.assertEqual(2, self.store.pack_loose_objects())
-        self.assertNotEquals([], self.store.packs)
+        self.assertNotEqual([], self.store.packs)
         self.assertEqual(0, self.store.pack_loose_objects())
 
 
@@ -424,9 +424,24 @@ class ObjectStoreGraphWalkerTests(TestCase):
                 "d": ["e"],
                 "e": [],
                 })
-        self.assertEqual("a", next(gw))
-        self.assertEqual("c", next(gw))
-        gw.ack("a")
-        self.assertEqual("b", next(gw))
-        self.assertEqual("d", next(gw))
+        walk = []
+        acked = False
+        walk.append(next(gw))
+        walk.append(next(gw))
+        # A branch (a, c) or (b, d) may be done after 2 steps or 3 depending on
+        # the order walked: 3-step walks include (a, b, c) and (b, a, d), etc.
+        if walk == ["a", "c"] or walk == ["b", "d"]:
+          gw.ack(walk[0])
+          acked = True
+
+        walk.append(next(gw))
+        if not acked and walk[2] == "c":
+          gw.ack("a")
+        elif not acked and walk[2] == "d":
+          gw.ack("b")
+        walk.append(next(gw))
         self.assertIs(None, next(gw))
+
+        self.assertEqual(["a", "b", "c", "d"], sorted(walk))
+        self.assertLess(walk.index("a"), walk.index("c"))
+        self.assertLess(walk.index("b"), walk.index("d"))

dulwich/tests/test_objects.py

@@ -421,7 +421,7 @@ Merge ../b
 
         d = Commit()
         d._deserialize(commit.as_raw_chunks())
-        self.assertEquals(commit, d)
+        self.assertEqual(commit, d)
 
 
 default_committer = 'James Westby <jw+debian@jameswestby.net> 1174773719 +0000'

dulwich/tests/test_objectspec.py

@@ -24,9 +24,6 @@
 
 from dulwich.objects import (
     Blob,
-    Commit,
-    Tag,
-    Tree,
     )
 from dulwich.objectspec import (
     parse_object,
@@ -38,7 +35,6 @@ from dulwich.tests import (
     )
 from dulwich.tests.utils import (
     build_commit_graph,
-    make_object,
     )
 
 
@@ -53,7 +49,7 @@ class ParseObjectTests(TestCase):
         r = MemoryRepo()
         b = Blob.from_string("Blah")
         r.object_store.add_object(b)
-        self.assertEquals(b, parse_object(r, b.id))
+        self.assertEqual(b, parse_object(r, b.id))
 
 
 class ParseCommitRangeTests(TestCase):
@@ -67,4 +63,4 @@ class ParseCommitRangeTests(TestCase):
         r = MemoryRepo()
         c1, c2, c3 = build_commit_graph(r.object_store, [[1], [2, 1],
             [3, 1, 2]])
-        self.assertEquals([c1], list(parse_commit_range(r, c1.id)))
+        self.assertEqual([c1], list(parse_commit_range(r, c1.id)))

dulwich/tests/test_pack.py

@@ -37,7 +37,6 @@ from dulwich.object_store import (
     MemoryObjectStore,
     )
 from dulwich.objects import (
-    Blob,
     hex_to_sha,
     sha_to_hex,
     Commit,
@@ -47,7 +46,6 @@ from dulwich.objects import (
 from dulwich.pack import (
     OFS_DELTA,
     REF_DELTA,
-    DELTA_TYPES,
     MemoryPackIndex,
     Pack,
     PackData,
@@ -60,13 +58,14 @@ from dulwich.pack import (
     write_pack_header,
     write_pack_index_v1,
     write_pack_index_v2,
-    SHA1Writer,
     write_pack_object,
     write_pack,
     unpack_object,
     compute_file_sha,
     PackStreamReader,
     DeltaChainIterator,
+    _delta_encode_size,
+    _encode_copy_operation,
     )
 from dulwich.tests import (
     TestCase,
@@ -187,64 +186,64 @@ class TestPackData(PackTests):
     """Tests getting the data from the packfile."""
 
     def test_create_pack(self):
-        p = self.get_pack_data(pack1_sha)
+        self.get_pack_data(pack1_sha).close()
 
     def test_from_file(self):
         path = os.path.join(self.datadir, 'pack-%s.pack' % pack1_sha)
         PackData.from_file(open(path), os.path.getsize(path))
 
     def test_pack_len(self):
-        p = self.get_pack_data(pack1_sha)
-        self.assertEqual(3, len(p))
+        with self.get_pack_data(pack1_sha) as p:
+            self.assertEqual(3, len(p))
 
     def test_index_check(self):
-        p = self.get_pack_data(pack1_sha)
-        self.assertSucceeds(p.check)
+        with self.get_pack_data(pack1_sha) as p:
+            self.assertSucceeds(p.check)
 
     def test_iterobjects(self):
-        p = self.get_pack_data(pack1_sha)
-        commit_data = ('tree b2a2766a2879c209ab1176e7e778b81ae422eeaa\n'
-                       'author James Westby <jw+debian@jameswestby.net> '
-                       '1174945067 +0100\n'
-                       'committer James Westby <jw+debian@jameswestby.net> '
-                       '1174945067 +0100\n'
-                       '\n'
-                       'Test commit\n')
-        blob_sha = '6f670c0fb53f9463760b7295fbb814e965fb20c8'
-        tree_data = '100644 a\0%s' % hex_to_sha(blob_sha)
-        actual = []
-        for offset, type_num, chunks, crc32 in p.iterobjects():
-            actual.append((offset, type_num, ''.join(chunks), crc32))
-        self.assertEqual([
-          (12, 1, commit_data, 3775879613),
-          (138, 2, tree_data, 912998690),
-          (178, 3, 'test 1\n', 1373561701)
-          ], actual)
+        with self.get_pack_data(pack1_sha) as p:
+            commit_data = ('tree b2a2766a2879c209ab1176e7e778b81ae422eeaa\n'
+                           'author James Westby <jw+debian@jameswestby.net> '
+                           '1174945067 +0100\n'
+                           'committer James Westby <jw+debian@jameswestby.net> '
+                           '1174945067 +0100\n'
+                           '\n'
+                           'Test commit\n')
+            blob_sha = '6f670c0fb53f9463760b7295fbb814e965fb20c8'
+            tree_data = '100644 a\0%s' % hex_to_sha(blob_sha)
+            actual = []
+            for offset, type_num, chunks, crc32 in p.iterobjects():
+                actual.append((offset, type_num, ''.join(chunks), crc32))
+            self.assertEqual([
+              (12, 1, commit_data, 3775879613),
+              (138, 2, tree_data, 912998690),
+              (178, 3, 'test 1\n', 1373561701)
+              ], actual)
 
     def test_iterentries(self):
-        p = self.get_pack_data(pack1_sha)
-        entries = set((sha_to_hex(s), o, c) for s, o, c in p.iterentries())
-        self.assertEqual(set([
-          ('6f670c0fb53f9463760b7295fbb814e965fb20c8', 178, 1373561701),
-          ('b2a2766a2879c209ab1176e7e778b81ae422eeaa', 138, 912998690),
-          ('f18faa16531ac570a3fdc8c7ca16682548dafd12', 12, 3775879613),
-          ]), entries)
+        with self.get_pack_data(pack1_sha) as p:
+            entries = set((sha_to_hex(s), o, c) for s, o, c in p.iterentries())
+            self.assertEqual(set([
+              ('6f670c0fb53f9463760b7295fbb814e965fb20c8', 178, 1373561701),
+              ('b2a2766a2879c209ab1176e7e778b81ae422eeaa', 138, 912998690),
+              ('f18faa16531ac570a3fdc8c7ca16682548dafd12', 12, 3775879613),
+              ]), entries)
 
     def test_create_index_v1(self):
-        p = self.get_pack_data(pack1_sha)
-        filename = os.path.join(self.tempdir, 'v1test.idx')
-        p.create_index_v1(filename)
-        idx1 = load_pack_index(filename)
-        idx2 = self.get_pack_index(pack1_sha)
-        self.assertEqual(idx1, idx2)
+        with self.get_pack_data(pack1_sha) as p:
+            filename = os.path.join(self.tempdir, 'v1test.idx')
+            p.create_index_v1(filename)
+            idx1 = load_pack_index(filename)
+            idx2 = self.get_pack_index(pack1_sha)
+            self.assertEqual(idx1, idx2)
 
     def test_create_index_v2(self):
-        p = self.get_pack_data(pack1_sha)
-        filename = os.path.join(self.tempdir, 'v2test.idx')
-        p.create_index_v2(filename)
-        idx1 = load_pack_index(filename)
-        idx2 = self.get_pack_index(pack1_sha)
-        self.assertEqual(idx1, idx2)
+        with self.get_pack_data(pack1_sha) as p:
+            filename = os.path.join(self.tempdir, 'v2test.idx')
+            p.create_index_v2(filename)
+            idx1 = load_pack_index(filename)
+            idx2 = self.get_pack_index(pack1_sha)
+            self.assertEqual(idx1, idx2)
 
     def test_compute_file_sha(self):
         f = BytesIO('abcd1234wxyz')
@@ -264,46 +263,46 @@ class TestPackData(PackTests):
 class TestPack(PackTests):
 
     def test_len(self):
-        p = self.get_pack(pack1_sha)
-        self.assertEqual(3, len(p))
+        with self.get_pack(pack1_sha) as p:
+            self.assertEqual(3, len(p))
 
     def test_contains(self):
-        p = self.get_pack(pack1_sha)
-        self.assertTrue(tree_sha in p)
+        with self.get_pack(pack1_sha) as p:
+            self.assertTrue(tree_sha in p)
 
     def test_get(self):
-        p = self.get_pack(pack1_sha)
-        self.assertEqual(type(p[tree_sha]), Tree)
+        with self.get_pack(pack1_sha) as p:
+            self.assertEqual(type(p[tree_sha]), Tree)
 
     def test_iter(self):
-        p = self.get_pack(pack1_sha)
-        self.assertEqual(set([tree_sha, commit_sha, a_sha]), set(p))
+        with self.get_pack(pack1_sha) as p:
+            self.assertEqual(set([tree_sha, commit_sha, a_sha]), set(p))
 
     def test_iterobjects(self):
-        p = self.get_pack(pack1_sha)
-        expected = set([p[s] for s in [commit_sha, tree_sha, a_sha]])
-        self.assertEqual(expected, set(list(p.iterobjects())))
+        with self.get_pack(pack1_sha) as p:
+            expected = set([p[s] for s in [commit_sha, tree_sha, a_sha]])
+            self.assertEqual(expected, set(list(p.iterobjects())))
 
     def test_pack_tuples(self):
-        p = self.get_pack(pack1_sha)
-        tuples = p.pack_tuples()
-        expected = set([(p[s], None) for s in [commit_sha, tree_sha, a_sha]])
-        self.assertEqual(expected, set(list(tuples)))
-        self.assertEqual(expected, set(list(tuples)))
-        self.assertEqual(3, len(tuples))
+        with self.get_pack(pack1_sha) as p:
+            tuples = p.pack_tuples()
+            expected = set([(p[s], None) for s in [commit_sha, tree_sha, a_sha]])
+            self.assertEqual(expected, set(list(tuples)))
+            self.assertEqual(expected, set(list(tuples)))
+            self.assertEqual(3, len(tuples))
 
     def test_get_object_at(self):
         """Tests random access for non-delta objects"""
-        p = self.get_pack(pack1_sha)
-        obj = p[a_sha]
-        self.assertEqual(obj.type_name, 'blob')
-        self.assertEqual(obj.sha().hexdigest(), a_sha)
-        obj = p[tree_sha]
-        self.assertEqual(obj.type_name, 'tree')
-        self.assertEqual(obj.sha().hexdigest(), tree_sha)
-        obj = p[commit_sha]
-        self.assertEqual(obj.type_name, 'commit')
-        self.assertEqual(obj.sha().hexdigest(), commit_sha)
+        with self.get_pack(pack1_sha) as p:
+            obj = p[a_sha]
+            self.assertEqual(obj.type_name, 'blob')
+            self.assertEqual(obj.sha().hexdigest(), a_sha)
+            obj = p[tree_sha]
+            self.assertEqual(obj.type_name, 'tree')
+            self.assertEqual(obj.sha().hexdigest(), tree_sha)
+            obj = p[commit_sha]
+            self.assertEqual(obj.type_name, 'commit')
+            self.assertEqual(obj.sha().hexdigest(), commit_sha)
 
     def test_copy(self):
         origpack = self.get_pack(pack1_sha)
@@ -331,11 +330,11 @@ class TestPack(PackTests):
             origpack.close()
 
     def test_commit_obj(self):
-        p = self.get_pack(pack1_sha)
-        commit = p[commit_sha]
-        self.assertEqual('James Westby <jw+debian@jameswestby.net>',
-                          commit.author)
-        self.assertEqual([], commit.parents)
+        with self.get_pack(pack1_sha) as p:
+            commit = p[commit_sha]
+            self.assertEqual('James Westby <jw+debian@jameswestby.net>',
+                             commit.author)
+            self.assertEqual([], commit.parents)
 
     def _copy_pack(self, origpack):
         basename = os.path.join(self.tempdir, 'somepack')
@@ -343,10 +342,12 @@ class TestPack(PackTests):
         return Pack(basename)
 
     def test_keep_no_message(self):
-        p = self.get_pack(pack1_sha)
-        p = self._copy_pack(p)
+        with self.get_pack(pack1_sha) as p:
+            p = self._copy_pack(p)
+
+        with p:
+            keepfile_name = p.keep()
 
-        keepfile_name = p.keep()
         # file should exist
         self.assertTrue(os.path.exists(keepfile_name))
 
@@ -358,11 +359,12 @@ class TestPack(PackTests):
             f.close()
 
     def test_keep_message(self):
-        p = self.get_pack(pack1_sha)
-        p = self._copy_pack(p)
+        with self.get_pack(pack1_sha) as p:
+            p = self._copy_pack(p)
 
         msg = 'some message'
-        keepfile_name = p.keep(msg)
+        with p:
+            keepfile_name = p.keep(msg)
 
         # file should exist
         self.assertTrue(os.path.exists(keepfile_name))
@@ -376,46 +378,46 @@ class TestPack(PackTests):
             f.close()
 
     def test_name(self):
-        p = self.get_pack(pack1_sha)
-        self.assertEqual(pack1_sha, p.name())
+        with self.get_pack(pack1_sha) as p:
+            self.assertEqual(pack1_sha, p.name())
 
     def test_length_mismatch(self):
-        data = self.get_pack_data(pack1_sha)
-        index = self.get_pack_index(pack1_sha)
-        Pack.from_objects(data, index).check_length_and_checksum()
-
-        data._file.seek(12)
-        bad_file = BytesIO()
-        write_pack_header(bad_file, 9999)
-        bad_file.write(data._file.read())
-        bad_file = BytesIO(bad_file.getvalue())
-        bad_data = PackData('', file=bad_file)
-        bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
-        self.assertRaises(AssertionError, lambda: bad_pack.data)
-        self.assertRaises(AssertionError,
-                          lambda: bad_pack.check_length_and_checksum())
+        with self.get_pack_data(pack1_sha) as data:
+            index = self.get_pack_index(pack1_sha)
+            Pack.from_objects(data, index).check_length_and_checksum()
+
+            data._file.seek(12)
+            bad_file = BytesIO()
+            write_pack_header(bad_file, 9999)
+            bad_file.write(data._file.read())
+            bad_file = BytesIO(bad_file.getvalue())
+            bad_data = PackData('', file=bad_file)
+            bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
+            self.assertRaises(AssertionError, lambda: bad_pack.data)
+            self.assertRaises(AssertionError,
+                              lambda: bad_pack.check_length_and_checksum())
 
     def test_checksum_mismatch(self):
-        data = self.get_pack_data(pack1_sha)
-        index = self.get_pack_index(pack1_sha)
-        Pack.from_objects(data, index).check_length_and_checksum()
-
-        data._file.seek(0)
-        bad_file = BytesIO(data._file.read()[:-20] + ('\xff' * 20))
-        bad_data = PackData('', file=bad_file)
-        bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
-        self.assertRaises(ChecksumMismatch, lambda: bad_pack.data)
-        self.assertRaises(ChecksumMismatch, lambda:
-                          bad_pack.check_length_and_checksum())
-
-    def test_iterobjects(self):
-        p = self.get_pack(pack1_sha)
-        objs = dict((o.id, o) for o in p.iterobjects())
-        self.assertEqual(3, len(objs))
-        self.assertEqual(sorted(objs), sorted(p.index))
-        self.assertTrue(isinstance(objs[a_sha], Blob))
-        self.assertTrue(isinstance(objs[tree_sha], Tree))
-        self.assertTrue(isinstance(objs[commit_sha], Commit))
+        with self.get_pack_data(pack1_sha) as data:
+            index = self.get_pack_index(pack1_sha)
+            Pack.from_objects(data, index).check_length_and_checksum()
+
+            data._file.seek(0)
+            bad_file = BytesIO(data._file.read()[:-20] + ('\xff' * 20))
+            bad_data = PackData('', file=bad_file)
+            bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
+            self.assertRaises(ChecksumMismatch, lambda: bad_pack.data)
+            self.assertRaises(ChecksumMismatch, lambda:
+                              bad_pack.check_length_and_checksum())
+
+    def test_iterobjects_2(self):
+        with self.get_pack(pack1_sha) as p:
+            objs = dict((o.id, o) for o in p.iterobjects())
+            self.assertEqual(3, len(objs))
+            self.assertEqual(sorted(objs), sorted(p.index))
+            self.assertTrue(isinstance(objs[a_sha], Blob))
+            self.assertTrue(isinstance(objs[tree_sha], Tree))
+            self.assertTrue(isinstance(objs[commit_sha], Commit))
 
 
 class TestThinPack(PackTests):
@@ -446,10 +448,10 @@ class TestThinPack(PackTests):
             f.close()
 
         # Index the new pack.
-        pack = self.make_pack(True)
-        data = PackData(pack._data_path)
-        data.pack = pack
-        data.create_index(self.pack_prefix + '.idx')
+        with self.make_pack(True) as pack:
+            with PackData(pack._data_path) as data:
+                data.pack = pack
+                data.create_index(self.pack_prefix + '.idx')
 
         del self.store[self.blobs['bar'].id]
 
@@ -459,18 +461,22 @@ class TestThinPack(PackTests):
             resolve_ext_ref=self.store.get_raw if resolve_ext_ref else None)
 
     def test_get_raw(self):
-        self.assertRaises(
-            KeyError, self.make_pack(False).get_raw, self.blobs['foo1234'].id)
-        self.assertEqual(
-            (3, 'foo1234'),
-            self.make_pack(True).get_raw(self.blobs['foo1234'].id))
+        with self.make_pack(False) as p:
+            self.assertRaises(
+                KeyError, p.get_raw, self.blobs['foo1234'].id)
+        with self.make_pack(True) as p:
+            self.assertEqual(
+                (3, 'foo1234'),
+                p.get_raw(self.blobs['foo1234'].id))
 
     def test_iterobjects(self):
-        self.assertRaises(KeyError, list, self.make_pack(False).iterobjects())
-        self.assertEqual(
-            sorted([self.blobs['foo1234'].id, self.blobs['bar'].id,
-                    self.blobs['bar2468'].id]),
-            sorted(o.id for o in self.make_pack(True).iterobjects()))
+        with self.make_pack(False) as p:
+            self.assertRaises(KeyError, list, p.iterobjects())
+        with self.make_pack(True) as p:
+            self.assertEqual(
+                sorted([self.blobs['foo1234'].id, self.blobs[b'bar'].id,
+                        self.blobs['bar2468'].id]),
+                sorted(o.id for o in p.iterobjects()))
 
 
 class WritePackTests(TestCase):
@@ -490,7 +496,6 @@ class WritePackTests(TestCase):
 
         f.write('x')  # unpack_object needs extra trailing data.
         f.seek(offset)
-        comp_len = len(f.getvalue()) - offset - 1
         unpacked, unused = unpack_object(f.read, compute_crc32=True)
         self.assertEqual(Blob.type_num, unpacked.pack_type_num)
         self.assertEqual(Blob.type_num, unpacked.obj_type_num)
@@ -644,12 +649,12 @@ class TestPackIndexWritingv2(TestCase, BaseTestFilePackIndexWriting):
 class ReadZlibTests(TestCase):
 
     decomp = (
-      'tree 4ada885c9196b6b6fa08744b5862bf92896fc002\n'
-      'parent None\n'
-      'author Jelmer Vernooij <jelmer@samba.org> 1228980214 +0000\n'
-      'committer Jelmer Vernooij <jelmer@samba.org> 1228980214 +0000\n'
-      '\n'
-      "Provide replacement for mmap()'s offset argument.")
+      b'tree 4ada885c9196b6b6fa08744b5862bf92896fc002\n'
+      b'parent None\n'
+      b'author Jelmer Vernooij <jelmer@samba.org> 1228980214 +0000\n'
+      b'committer Jelmer Vernooij <jelmer@samba.org> 1228980214 +0000\n'
+      b'\n'
+      b"Provide replacement for mmap()'s offset argument.")
     comp = zlib.compress(decomp)
     extra = 'nextobject'
 
@@ -683,7 +688,7 @@ class ReadZlibTests(TestCase):
         read = BytesIO(comp + self.extra).read
         unused = read_zlib_chunks(read, unpacked)
         self.assertEqual('', ''.join(unpacked.decomp_chunks))
-        self.assertNotEquals('', unused)
+        self.assertNotEqual('', unused)
         self.assertEqual(self.extra, unused + read())
 
     def test_decompress_no_crc32(self):
@@ -696,7 +701,7 @@ class ReadZlibTests(TestCase):
                                   buffer_size=buffer_size, **kwargs)
         self.assertEqual(self.decomp, ''.join(self.unpacked.decomp_chunks))
         self.assertEqual(zlib.crc32(self.comp), self.unpacked.crc32)
-        self.assertNotEquals('', unused)
+        self.assertNotEqual('', unused)
         self.assertEqual(self.extra, unused + self.read())
 
     def test_simple_decompress(self):
@@ -998,7 +1003,7 @@ class DeltaChainIteratorTests(TestCase):
     def test_bad_ext_ref_non_thin_pack(self):
         blob, = self.store_blobs(['blob'])
         f = BytesIO()
-        entries = build_pack(f, [(REF_DELTA, (blob.id, 'blob1'))],
+        build_pack(f, [(REF_DELTA, (blob.id, 'blob1'))],
                              store=self.store)
         pack_iter = self.make_pack_iter(f, thin=False)
         try:
@@ -1010,7 +1015,7 @@ class DeltaChainIteratorTests(TestCase):
     def test_bad_ext_ref_thin_pack(self):
         b1, b2, b3 = self.store_blobs(['foo', 'bar', 'baz'])
         f = BytesIO()
-        entries = build_pack(f, [
+        build_pack(f, [
           (REF_DELTA, (1, 'foo99')),
           (REF_DELTA, (b1.id, 'foo1')),
           (REF_DELTA, (b2.id, 'bar2')),
@@ -1023,4 +1028,23 @@ class DeltaChainIteratorTests(TestCase):
             list(pack_iter._walk_all_chains())
             self.fail()
         except KeyError as e:
-            self.assertEqual((sorted([b2.id, b3.id]),), e.args)
+            self.assertEqual((sorted([b2.id, b3.id]),), (sorted(e.args[0]),))
+
+
+class DeltaEncodeSizeTests(TestCase):
+
+    def test_basic(self):
+        self.assertEquals('\x00', _delta_encode_size(0))
+        self.assertEquals('\x01', _delta_encode_size(1))
+        self.assertEquals('\xfa\x01', _delta_encode_size(250))
+        self.assertEquals('\xe8\x07', _delta_encode_size(1000))
+        self.assertEquals('\xa0\x8d\x06', _delta_encode_size(100000))
+
+
+class EncodeCopyOperationTests(TestCase):
+
+    def test_basic(self):
+        self.assertEquals('\x80', _encode_copy_operation(0, 0))
+        self.assertEquals('\x91\x01\x0a', _encode_copy_operation(1, 10))
+        self.assertEquals('\xb1\x64\xe8\x03', _encode_copy_operation(100, 1000))
+        self.assertEquals('\x93\xe8\x03\x01', _encode_copy_operation(1000, 1))

dulwich/tests/test_patch.py

@@ -19,6 +19,7 @@
 """Tests for patch.py."""
 
 from io import BytesIO
+from unittest import SkipTest
 
 from dulwich.objects import (
     Blob,
@@ -37,7 +38,6 @@ from dulwich.patch import (
     write_tree_diff,
     )
 from dulwich.tests import (
-    SkipTest,
     TestCase,
     )
 

dulwich/tests/test_porcelain.py

@@ -35,6 +35,7 @@ from dulwich.repo import Repo
 from dulwich.tests import (
     TestCase,
     )
+from dulwich.tests.compat.utils import require_git_version
 from dulwich.tests.utils import (
     build_commit_graph,
     make_object,
@@ -54,16 +55,18 @@ class ArchiveTests(PorcelainTestCase):
     """Tests for the archive command."""
 
     def test_simple(self):
+        # TODO(jelmer): Remove this once dulwich has its own implementation of archive.
+        require_git_version((1, 5, 0))
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1], [3, 1, 2]])
         self.repo.refs["refs/heads/master"] = c3.id
         out = BytesIO()
         err = BytesIO()
         porcelain.archive(self.repo.path, "refs/heads/master", outstream=out,
             errstream=err)
-        self.assertEquals("", err.getvalue())
+        self.assertEqual("", err.getvalue())
         tf = tarfile.TarFile(fileobj=out)
         self.addCleanup(tf.close)
-        self.assertEquals([], tf.getnames())
+        self.assertEqual([], tf.getnames())
 
 
 class UpdateServerInfoTests(PorcelainTestCase):
@@ -86,7 +89,7 @@ class CommitTests(PorcelainTestCase):
         sha = porcelain.commit(self.repo.path, message="Some message",
                 author="Joe <joe@example.com>", committer="Bob <bob@example.com>")
         self.assertTrue(isinstance(sha, str))
-        self.assertEquals(len(sha), 40)
+        self.assertEqual(len(sha), 40)
 
 
 class CloneTests(PorcelainTestCase):
@@ -106,8 +109,8 @@ class CloneTests(PorcelainTestCase):
         self.addCleanup(shutil.rmtree, target_path)
         r = porcelain.clone(self.repo.path, target_path,
                             checkout=False, outstream=outstream)
-        self.assertEquals(r.path, target_path)
-        self.assertEquals(Repo(target_path).head(), c3.id)
+        self.assertEqual(r.path, target_path)
+        self.assertEqual(Repo(target_path).head(), c3.id)
         self.assertTrue('f1' not in os.listdir(target_path))
         self.assertTrue('f2' not in os.listdir(target_path))
 
@@ -126,8 +129,8 @@ class CloneTests(PorcelainTestCase):
         self.addCleanup(shutil.rmtree, target_path)
         r = porcelain.clone(self.repo.path, target_path,
                             checkout=True, outstream=outstream)
-        self.assertEquals(r.path, target_path)
-        self.assertEquals(Repo(target_path).head(), c3.id)
+        self.assertEqual(r.path, target_path)
+        self.assertEqual(Repo(target_path).head(), c3.id)
         self.assertTrue('f1' in os.listdir(target_path))
         self.assertTrue('f2' in os.listdir(target_path))
 
@@ -146,8 +149,8 @@ class CloneTests(PorcelainTestCase):
         self.addCleanup(shutil.rmtree, target_path)
         r = porcelain.clone(self.repo.path, target_path,
                             bare=True, outstream=outstream)
-        self.assertEquals(r.path, target_path)
-        self.assertEquals(Repo(target_path).head(), c3.id)
+        self.assertEqual(r.path, target_path)
+        self.assertEqual(Repo(target_path).head(), c3.id)
         self.assertFalse('f1' in os.listdir(target_path))
         self.assertFalse('f2' in os.listdir(target_path))
 
@@ -199,7 +202,7 @@ class AddTests(PorcelainTestCase):
 
         # Check that foo was added and nothing in .git was modified
         index = self.repo.open_index()
-        self.assertEquals(list(index), ['blah', 'foo', 'adir/afile'])
+        self.assertEqual(list(index), ['blah', 'foo', 'adir/afile'])
 
     def test_add_file(self):
         with open(os.path.join(self.repo.path, 'foo'), 'w') as f:
@@ -227,7 +230,7 @@ class LogTests(PorcelainTestCase):
         self.repo.refs["HEAD"] = c3.id
         outstream = BytesIO()
         porcelain.log(self.repo.path, outstream=outstream)
-        self.assertEquals(3, outstream.getvalue().count("-" * 50))
+        self.assertEqual(3, outstream.getvalue().count("-" * 50))
 
     def test_max_entries(self):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
@@ -235,7 +238,7 @@ class LogTests(PorcelainTestCase):
         self.repo.refs["HEAD"] = c3.id
         outstream = BytesIO()
         porcelain.log(self.repo.path, outstream=outstream, max_entries=1)
-        self.assertEquals(1, outstream.getvalue().count("-" * 50))
+        self.assertEqual(1, outstream.getvalue().count("-" * 50))
 
 
 class ShowTests(PorcelainTestCase):
@@ -261,7 +264,7 @@ class ShowTests(PorcelainTestCase):
         self.repo.object_store.add_object(b)
         outstream = BytesIO()
         porcelain.show(self.repo.path, objects=[b.id], outstream=outstream)
-        self.assertEquals(outstream.getvalue(), "The Foo\n")
+        self.assertEqual(outstream.getvalue(), "The Foo\n")
 
 
 class SymbolicRefTests(PorcelainTestCase):
@@ -271,7 +274,6 @@ class SymbolicRefTests(PorcelainTestCase):
             [3, 1, 2]])
         self.repo.refs["HEAD"] = c3.id
 
-        outstream = BytesIO()
         self.assertRaises(ValueError, porcelain.symbolic_ref, self.repo.path, 'foobar')
 
     def test_set_force_wrong_symbolic_ref(self):
@@ -282,8 +284,9 @@ class SymbolicRefTests(PorcelainTestCase):
         porcelain.symbolic_ref(self.repo.path, 'force_foobar', force=True)
 
         #test if we actually changed the file
-        new_ref = self.repo.get_named_file('HEAD').read()
-        self.assertEqual(new_ref, 'ref: refs/heads/force_foobar\n')
+        with self.repo.get_named_file('HEAD') as f:
+            new_ref = f.read()
+        self.assertEqual(new_ref, b'ref: refs/heads/force_foobar\n')
 
     def test_set_symbolic_ref(self):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
@@ -301,8 +304,9 @@ class SymbolicRefTests(PorcelainTestCase):
         porcelain.symbolic_ref(self.repo.path, 'develop')
 
         #test if we actually changed the file
-        new_ref = self.repo.get_named_file('HEAD').read()
-        self.assertEqual(new_ref, 'ref: refs/heads/develop\n')
+        with self.repo.get_named_file('HEAD') as f:
+            new_ref = f.read()
+        self.assertEqual(new_ref, b'ref: refs/heads/develop\n')
 
 
 class DiffTreeTests(PorcelainTestCase):
@@ -313,7 +317,7 @@ class DiffTreeTests(PorcelainTestCase):
         self.repo.refs["HEAD"] = c3.id
         outstream = BytesIO()
         porcelain.diff_tree(self.repo.path, c2.tree, c3.tree, outstream=outstream)
-        self.assertEquals(outstream.getvalue(), "")
+        self.assertEqual(outstream.getvalue(), "")
 
 
 class CommitTreeTests(PorcelainTestCase):
@@ -332,7 +336,7 @@ class CommitTreeTests(PorcelainTestCase):
             author="Joe <joe@example.com>",
             committer="Jane <jane@example.com>")
         self.assertTrue(isinstance(sha, str))
-        self.assertEquals(len(sha), 40)
+        self.assertEqual(len(sha), 40)
 
 
 class RevListTests(PorcelainTestCase):
@@ -343,7 +347,7 @@ class RevListTests(PorcelainTestCase):
         outstream = BytesIO()
         porcelain.rev_list(
             self.repo.path, [c3.id], outstream=outstream)
-        self.assertEquals(
+        self.assertEqual(
             "%s\n%s\n%s\n" % (c3.id, c2.id, c1.id),
             outstream.getvalue())
 
@@ -359,11 +363,11 @@ class TagTests(PorcelainTestCase):
                 annotated=True)
 
         tags = self.repo.refs.as_dict("refs/tags")
-        self.assertEquals(tags.keys(), ["tryme"])
+        self.assertEqual(tags.keys(), ["tryme"])
         tag = self.repo['refs/tags/tryme']
         self.assertTrue(isinstance(tag, Tag))
-        self.assertEquals("foo <foo@bar.com>", tag.tagger)
-        self.assertEquals("bar", tag.message)
+        self.assertEqual("foo <foo@bar.com>", tag.tagger)
+        self.assertEqual("bar", tag.message)
 
     def test_unannotated(self):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
@@ -373,23 +377,23 @@ class TagTests(PorcelainTestCase):
         porcelain.tag(self.repo.path, "tryme", annotated=False)
 
         tags = self.repo.refs.as_dict("refs/tags")
-        self.assertEquals(tags.keys(), ["tryme"])
-        tag = self.repo['refs/tags/tryme']
-        self.assertEquals(tags.values(), [self.repo.head()])
+        self.assertEqual(tags.keys(), ["tryme"])
+        self.repo['refs/tags/tryme']
+        self.assertEqual(tags.values(), [self.repo.head()])
 
 
 class ListTagsTests(PorcelainTestCase):
 
     def test_empty(self):
         tags = porcelain.list_tags(self.repo.path)
-        self.assertEquals([], tags)
+        self.assertEqual([], tags)
 
     def test_simple(self):
         self.repo.refs["refs/tags/foo"] = "aa" * 20
         self.repo.refs["refs/tags/bar/bla"] = "bb" * 20
         tags = porcelain.list_tags(self.repo.path)
 
-        self.assertEquals(["bar/bla", "foo"], tags)
+        self.assertEqual(["bar/bla", "foo"], tags)
 
 
 class ResetTests(PorcelainTestCase):
@@ -418,7 +422,7 @@ class ResetTests(PorcelainTestCase):
                        index.commit(self.repo.object_store),
                        self.repo['HEAD'].tree))
 
-        self.assertEquals([], changes)
+        self.assertEqual([], changes)
 
 
 class PushTests(PorcelainTestCase):
@@ -461,8 +465,8 @@ class PushTests(PorcelainTestCase):
         change = list(tree_changes(self.repo, self.repo['HEAD'].tree,
                                    self.repo['refs/heads/foo'].tree))[0]
 
-        self.assertEquals(r_clone['HEAD'].id, self.repo[refs_path].id)
-        self.assertEquals(os.path.basename(fullpath), change.new.path)
+        self.assertEqual(r_clone['HEAD'].id, self.repo[refs_path].id)
+        self.assertEqual(os.path.basename(fullpath), change.new.path)
 
 
 class PullTests(PorcelainTestCase):
@@ -495,4 +499,101 @@ class PullTests(PorcelainTestCase):
 
         # Check the target repo for pushed changes
         r = Repo(target_path)
-        self.assertEquals(r['HEAD'].id, self.repo['HEAD'].id)
+        self.assertEqual(r['HEAD'].id, self.repo['HEAD'].id)
+
+
+class StatusTests(PorcelainTestCase):
+
+    def test_status(self):
+        """Integration test for `status` functionality."""
+
+        # Commit a dummy file then modify it
+        fullpath = os.path.join(self.repo.path, 'foo')
+        with open(fullpath, 'w') as f:
+            f.write('origstuff')
+
+        porcelain.add(repo=self.repo.path, paths=['foo'])
+        porcelain.commit(repo=self.repo.path, message='test status',
+            author='', committer='')
+
+        # modify access and modify time of path
+        os.utime(fullpath, (0, 0))
+
+        with open(fullpath, 'w') as f:
+            f.write('stuff')
+
+        # Make a dummy file and stage it
+        filename_add = 'bar'
+        fullpath = os.path.join(self.repo.path, filename_add)
+        with open(fullpath, 'w') as f:
+            f.write('stuff')
+        porcelain.add(repo=self.repo.path, paths=filename_add)
+
+        results = porcelain.status(self.repo)
+
+        self.assertEqual(results.staged['add'][0], filename_add)
+        self.assertEqual(results.unstaged, ['foo'])
+
+    def test_get_tree_changes_add(self):
+        """Unit test for get_tree_changes add."""
+
+        # Make a dummy file, stage
+        filename = 'bar'
+        with open(os.path.join(self.repo.path, filename), 'w') as f:
+            f.write('stuff')
+        porcelain.add(repo=self.repo.path, paths=filename)
+        porcelain.commit(repo=self.repo.path, message='test status',
+            author='', committer='')
+
+        filename = 'foo'
+        with open(os.path.join(self.repo.path, filename), 'w') as f:
+            f.write('stuff')
+        porcelain.add(repo=self.repo.path, paths=filename)
+        changes = porcelain.get_tree_changes(self.repo.path)
+
+        self.assertEqual(changes['add'][0], filename)
+        self.assertEqual(len(changes['add']), 1)
+        self.assertEqual(len(changes['modify']), 0)
+        self.assertEqual(len(changes['delete']), 0)
+
+    def test_get_tree_changes_modify(self):
+        """Unit test for get_tree_changes modify."""
+
+        # Make a dummy file, stage, commit, modify
+        filename = 'foo'
+        fullpath = os.path.join(self.repo.path, filename)
+        with open(fullpath, 'w') as f:
+            f.write('stuff')
+        porcelain.add(repo=self.repo.path, paths=filename)
+        porcelain.commit(repo=self.repo.path, message='test status',
+            author='', committer='')
+        with open(fullpath, 'w') as f:
+            f.write('otherstuff')
+        porcelain.add(repo=self.repo.path, paths=filename)
+        changes = porcelain.get_tree_changes(self.repo.path)
+
+        self.assertEqual(changes['modify'][0], filename)
+        self.assertEqual(len(changes['add']), 0)
+        self.assertEqual(len(changes['modify']), 1)
+        self.assertEqual(len(changes['delete']), 0)
+
+    def test_get_tree_changes_delete(self):
+        """Unit test for get_tree_changes delete."""
+
+        # Make a dummy file, stage, commit, remove
+        filename = 'foo'
+        with open(os.path.join(self.repo.path, filename), 'w') as f:
+            f.write('stuff')
+        porcelain.add(repo=self.repo.path, paths=filename)
+        porcelain.commit(repo=self.repo.path, message='test status',
+            author='', committer='')
+        porcelain.rm(repo=self.repo.path, paths=[filename])
+        changes = porcelain.get_tree_changes(self.repo.path)
+
+        self.assertEqual(changes['delete'][0], filename)
+        self.assertEqual(len(changes['add']), 0)
+        self.assertEqual(len(changes['modify']), 0)
+        self.assertEqual(len(changes['delete']), 1)
+
+
+# TODO(jelmer): Add test for dulwich.porcelain.daemon

dulwich/tests/test_protocol.py

@@ -82,6 +82,11 @@ class BaseProtocolTests(object):
         self.rin.seek(0)
         self.assertEqual(None, self.proto.read_pkt_line())
 
+    def test_read_pkt_line_wrong_size(self):
+        self.rin.write('0100too short')
+        self.rin.seek(0)
+        self.assertRaises(AssertionError, self.proto.read_pkt_line)
+
     def test_write_sideband(self):
         self.proto.write_sideband(3, 'bloe')
         self.assertEqual(self.rout.getvalue(), '0009\x03bloe')

dulwich/tests/test_repository.py

@@ -178,13 +178,13 @@ class RepositoryTests(TestCase):
             f.write("Some description")
         finally:
             f.close()
-        self.assertEquals("Some description", r.get_description())
+        self.assertEqual("Some description", r.get_description())
 
     def test_set_description(self):
         r = self._repo = open_repo('a.git')
         description = "Some description"
         r.set_description(description)
-        self.assertEquals(description, r.get_description())
+        self.assertEqual(description, r.get_description())
 
     def test_contains_missing(self):
         r = self._repo = open_repo('a.git')
@@ -339,13 +339,13 @@ class RepositoryTests(TestCase):
 
         try:
             r1 = Repo.init_bare(r1_dir)
-            map(lambda c: r1.object_store.add_object(r_base.get_object(c)), \
-                r1_commits)
+            for c in r1_commits:
+                r1.object_store.add_object(r_base.get_object(c))
             r1.refs['HEAD'] = r1_commits[0]
 
             r2 = Repo.init_bare(r2_dir)
-            map(lambda c: r2.object_store.add_object(r_base.get_object(c)), \
-                r2_commits)
+            for c in r2_commits:
+                r2.object_store.add_object(r_base.get_object(c))
             r2.refs['HEAD'] = r2_commits[0]
 
             # Finally, the 'real' testing!
@@ -502,7 +502,8 @@ exit 1
 
         warnings.simplefilter("always", UserWarning)
         self.addCleanup(warnings.resetwarnings)
-        warnings_list = setup_warning_catcher()
+        warnings_list, restore_warnings = setup_warning_catcher()
+        self.addCleanup(restore_warnings)
 
         commit_sha2 = r.do_commit(
             'empty commit',
@@ -525,9 +526,9 @@ class BuildRepoTests(TestCase):
 
     def setUp(self):
         super(BuildRepoTests, self).setUp()
-        repo_dir = os.path.join(tempfile.mkdtemp(), 'test')
-        os.makedirs(repo_dir)
-        r = self._repo = Repo.init(repo_dir)
+        self._repo_dir = os.path.join(tempfile.mkdtemp(), 'test')
+        os.makedirs(self._repo_dir)
+        r = self._repo = Repo.init(self._repo_dir)
         self.assertFalse(r.bare)
         self.assertEqual('ref: refs/heads/master', r.refs.read_ref('HEAD'))
         self.assertRaises(KeyError, lambda: r.refs['refs/heads/master'])
@@ -566,15 +567,20 @@ class BuildRepoTests(TestCase):
             f.write('new contents')
         finally:
             f.close()
-        r.stage(['a'])
+        os.symlink('a', os.path.join(self._repo_dir, 'b'))
+        r.stage(['a', 'b'])
         commit_sha = r.do_commit('modified a',
                                  committer='Test Committer <test@nodomain.com>',
                                  author='Test Author <test@nodomain.com>',
                                  commit_timestamp=12395, commit_timezone=0,
                                  author_timestamp=12395, author_timezone=0)
         self.assertEqual([self._root_commit], r[commit_sha].parents)
-        _, blob_id = tree_lookup_path(r.get_object, r[commit_sha].tree, 'a')
-        self.assertEqual('new contents', r[blob_id].data)
+        a_mode, a_id = tree_lookup_path(r.get_object, r[commit_sha].tree, 'a')
+        self.assertEqual(stat.S_IFREG | 0o644, a_mode)
+        self.assertEqual('new contents', r[a_id].data)
+        b_mode, b_id = tree_lookup_path(r.get_object, r[commit_sha].tree, 'b')
+        self.assertTrue(stat.S_ISLNK(b_mode))
+        self.assertEqual('a', r[b_id].data)
 
     def test_commit_deleted(self):
         r = self._repo

dulwich/tests/test_server.py

@@ -472,12 +472,12 @@ class ProtocolGraphWalkerTestCase(TestCase):
         self._walker._handle_shallow_request(heads)
 
     def assertReceived(self, expected):
-        self.assertEquals(
+        self.assertEqual(
           expected, list(iter(self._walker.proto.get_received_line, None)))
 
     def test_handle_shallow_request_no_client_shallows(self):
         self._handle_shallow_request(['deepen 1\n'], [FOUR, FIVE])
-        self.assertEquals(set([TWO, THREE]), self._walker.shallow)
+        self.assertEqual(set([TWO, THREE]), self._walker.shallow)
         self.assertReceived([
           'shallow %s' % TWO,
           'shallow %s' % THREE,
@@ -490,7 +490,7 @@ class ProtocolGraphWalkerTestCase(TestCase):
           'deepen 1\n',
           ]
         self._handle_shallow_request(lines, [FOUR, FIVE])
-        self.assertEquals(set([TWO, THREE]), self._walker.shallow)
+        self.assertEqual(set([TWO, THREE]), self._walker.shallow)
         self.assertReceived([])
 
     def test_handle_shallow_request_unshallows(self):
@@ -499,7 +499,7 @@ class ProtocolGraphWalkerTestCase(TestCase):
           'deepen 2\n',
           ]
         self._handle_shallow_request(lines, [FOUR, FIVE])
-        self.assertEquals(set([ONE]), self._walker.shallow)
+        self.assertEqual(set([ONE]), self._walker.shallow)
         self.assertReceived([
           'shallow %s' % ONE,
           'unshallow %s' % TWO,
@@ -845,6 +845,22 @@ class FileSystemBackendTests(TestCase):
         self.assertRaises(NotGitRepository,
             self.backend.open_repository, os.path.join(self.path, "foo"))
 
+    def test_bad_repo_path(self):
+        backend = FileSystemBackend()
+
+        self.assertRaises(NotGitRepository,
+                          lambda: backend.open_repository('/ups'))
+
+
+class DictBackendTests(TestCase):
+    """Tests for DictBackend."""
+
+    def test_nonexistant(self):
+        repo = MemoryRepo.init_bare([], {})
+        backend = DictBackend({'/': repo})
+        self.assertRaises(NotGitRepository,
+            backend.open_repository, "/does/not/exist/unless/foo")
+
     def test_bad_repo_path(self):
         repo = MemoryRepo.init_bare([], {})
         backend = DictBackend({'/': repo})
@@ -888,10 +904,10 @@ class UpdateServerInfoTests(TestCase):
 
     def test_empty(self):
         update_server_info(self.repo)
-        self.assertEqual("",
-            open(os.path.join(self.path, ".git", "info", "refs"), 'r').read())
-        self.assertEqual("",
-            open(os.path.join(self.path, ".git", "objects", "info", "packs"), 'r').read())
+        with open(os.path.join(self.path, ".git", "info", "refs"), 'rb') as f:
+            self.assertEqual(b'', f.read())
+        with open(os.path.join(self.path, ".git", "objects", "info", "packs"), 'rb') as f:
+            self.assertEqual(b'', f.read())
 
     def test_simple(self):
         commit_id = self.repo.do_commit(
@@ -899,7 +915,7 @@ class UpdateServerInfoTests(TestCase):
             committer="Joe Example <joe@example.com>",
             ref="refs/heads/foo")
         update_server_info(self.repo)
-        ref_text = open(os.path.join(self.path, ".git", "info", "refs"), 'r').read()
-        self.assertEqual(ref_text, "%s\trefs/heads/foo\n" % commit_id)
-        packs_text = open(os.path.join(self.path, ".git", "objects", "info", "packs"), 'r').read()
-        self.assertEqual(packs_text, "")
+        with open(os.path.join(self.path, ".git", "info", "refs"), 'rb') as f:
+            self.assertEqual(f.read(), commit_id + b'\trefs/heads/foo\n')
+        with open(os.path.join(self.path, ".git", "objects", "info", "packs"), 'rb') as f:
+            self.assertEqual(f.read(), b'')

dulwich/tests/test_walk.py

@@ -23,10 +23,8 @@ from itertools import (
     )
 
 from dulwich.diff_tree import (
-    CHANGE_ADD,
     CHANGE_MODIFY,
     CHANGE_RENAME,
-    CHANGE_COPY,
     TreeChange,
     RenameDetector,
     )
@@ -404,7 +402,7 @@ class WalkerTest(TestCase):
         #   \          /
         #    \-y3--y4-/--y5
         # Due to skew, y5 is the oldest commit.
-        c1, x2, y3, y4, y5, m6 = cs = self.make_commits(
+        c1, x2, y3, y4, y5, m6 = self.make_commits(
           [[1], [2, 1], [3, 1], [4, 3], [5, 4], [6, 2, 4]],
           times=[2, 3, 4, 5, 1, 6])
         self.assertWalkYields([m6, y4, y3, x2, c1], [m6.id])

dulwich/tests/test_web.py

@@ -306,8 +306,9 @@ class SmartHandlersTestCase(WebTestCase):
 
     def test_handle_service_request_unknown(self):
         mat = re.search('.*', '/git-evil-handler')
-        list(handle_service_request(self._req, 'backend', mat))
+        content = list(handle_service_request(self._req, 'backend', mat))
         self.assertEqual(HTTP_FORBIDDEN, self._status)
+        self.assertFalse('git-evil-handler' in "".join(content))
         self.assertFalse(self._req.cached)
 
     def _run_handle_service_request(self, content_length=None):
@@ -337,7 +338,8 @@ class SmartHandlersTestCase(WebTestCase):
 
     def test_get_info_refs_unknown(self):
         self._environ['QUERY_STRING'] = 'service=git-evil-handler'
-        list(get_info_refs(self._req, 'backend', None))
+        content = list(get_info_refs(self._req, 'backend', None))
+        self.assertFalse('git-evil-handler' in "".join(content))
         self.assertEqual(HTTP_FORBIDDEN, self._status)
         self.assertFalse(self._req.cached)
 
@@ -480,7 +482,7 @@ class GunzipTestCase(HTTPGitApplicationTestCase):
         self.assertEqual(self._environ['HTTP_CONTENT_ENCODING'], 'gzip')
         self._environ['CONTENT_LENGTH'] = zlength
         self._environ['wsgi.input'] = zstream
-        app_output = self._app(self._environ, None)
+        self._app(self._environ, None)
         buf = self._environ['wsgi.input']
         self.assertIsNot(buf, zstream)
         buf.seek(0)

dulwich/tests/utils.py

@@ -26,6 +26,9 @@ import shutil
 import tempfile
 import time
 import types
+from unittest import (
+    SkipTest,
+    )
 import warnings
 
 from dulwich.index import (
@@ -46,9 +49,6 @@ from dulwich.pack import (
     create_delta,
     )
 from dulwich.repo import Repo
-from dulwich.tests import (
-    SkipTest,
-    )
 
 # Plain files are very frequently used in tests, so let the mode be very short.
 F = 0o100644  # Shorthand mode for Files.
@@ -323,4 +323,8 @@ def setup_warning_catcher():
         caught_warnings.append(args[0])
 
     warnings.showwarning = custom_showwarning
-    return caught_warnings
+
+    def restore_showwarning():
+        warnings.showwarning = original_showwarning
+
+    return caught_warnings, restore_showwarning

dulwich/walk.py

@@ -23,7 +23,7 @@ from collections import defaultdict
 
 import collections
 import heapq
-import itertools
+from itertools import chain
 
 from dulwich.diff_tree import (
     RENAME_CHANGE_TYPES,
@@ -100,7 +100,7 @@ class _CommitTimeQueue(object):
         self._extra_commits_left = _MAX_EXTRA_COMMITS
         self._is_finished = False
 
-        for commit_id in itertools.chain(walker.include, walker.excluded):
+        for commit_id in chain(walker.include, walker.excluded):
             self._push(commit_id)
 
     def _push(self, commit_id):
@@ -181,6 +181,8 @@ class _CommitTimeQueue(object):
         self._is_finished = True
         return None
 
+    __next__ = next
+
 
 class Walker(object):
     """Object for performing a walk of commits in a store.

dulwich/web.py

@@ -166,7 +166,7 @@ def get_info_refs(req, backend, mat):
     if service and not req.dumb:
         handler_cls = req.handlers.get(service, None)
         if handler_cls is None:
-            yield req.forbidden('Unsupported service %s' % service)
+            yield req.forbidden('Unsupported service')
             return
         req.nocache()
         write = req.respond(HTTP_OK, 'application/x-%s-advertisement' % service)
@@ -222,7 +222,7 @@ def handle_service_request(req, backend, mat):
     logger.info('Handling service request for %s', service)
     handler_cls = req.handlers.get(service, None)
     if handler_cls is None:
-        yield req.forbidden('Unsupported service %s' % service)
+        yield req.forbidden('Unsupported service')
         return
     req.nocache()
     write = req.respond(HTTP_OK, 'application/x-%s-result' % service)

examples/latest_change.py

@@ -17,5 +17,5 @@ try:
 except StopIteration:
     print("No file %s anywhere in history." % sys.argv[1])
 else:
-    print("%s was last changed at %s by %s (commit %s)" % (
+    print("%s was last changed by %s at %s (commit %s)" % (
         sys.argv[1], c.author, time.ctime(c.author_time), c.id))

setup.py

@@ -10,7 +10,7 @@ except ImportError:
     has_setuptools = False
 from distutils.core import Distribution
 
-dulwich_version_string = '0.9.6'
+dulwich_version_string = '0.9.7'
 
 include_dirs = []
 # Windows MSVC support
@@ -44,6 +44,7 @@ if sys.platform == 'darwin' and os.path.exists('/usr/bin/xcodebuild'):
         stderr=subprocess.PIPE, env={})
     out, err = p.communicate()
     for l in out.splitlines():
+        l = l.decode("utf8")
         # Also parse only first digit, because 3.2.1 can't be parsed nicely
         if l.startswith('Xcode') and int(l.split()[1].split('.')[0]) >= 4:
             os.environ['ARCHFLAGS'] = ''
@@ -71,8 +72,8 @@ setup(name='dulwich',
       The project is named after the part of London that Mr. and Mrs. Git live in
       in the particular Monty Python sketch.
       """,
-      packages=['dulwich', 'dulwich.tests', 'dulwich.tests.compat'],
-      scripts=['bin/dulwich', 'bin/dul-daemon', 'bin/dul-web', 'bin/dul-receive-pack', 'bin/dul-upload-pack'],
+      packages=['dulwich', 'dulwich.tests', 'dulwich.tests.compat', 'dulwich.contrib'],
+      scripts=['bin/dulwich', 'bin/dul-web', 'bin/dul-receive-pack', 'bin/dul-upload-pack'],
       ext_modules=[
           Extension('dulwich._objects', ['dulwich/_objects.c'],
                     include_dirs=include_dirs),
@@ -82,5 +83,8 @@ setup(name='dulwich',
               include_dirs=include_dirs),
       ],
       distclass=DulwichDistribution,
+      include_package_data=True,
+      use_2to3=True,
+      convert_2to3_doctests=['../docs/*', '../docs/tutorial/*', ],
       **setup_kwargs
       )