Merge tag 'upstream/0.9.6' into debian

Upstream version 0.9.6

Conflicts:
	dulwich.egg-info/SOURCES.txt

AUTHORS

@@ -0,0 +1,10 @@
+Jelmer Vernooij <jelmer@samba.org>
+James Westby <jw+debian@jameswestby.net>
+John Carr <john.carr@unrouted.co.uk>
+Dave Borowitz <dborowitz@google.com>
+Chris Eberle <eberle1080@gmail.com>
+"milki" <milki@rescomp.berkeley.edu>
+
+Hervé Cauwelier <herve@itaapy.com> wrote the original tutorial.
+
+See the revision history for a full list of contributors.

HACKING

@@ -0,0 +1,27 @@
+All functionality should be available in pure Python. Optional C
+implementations may be written for performance reasons, but should never
+replace the Python implementation. The C implementations should follow the
+kernel/git coding style.
+
+Where possible include updates to NEWS along with your improvements.
+
+New functionality and bug fixes should be accompanied with matching unit tests.
+
+Coding style
+------------
+Where possible, please follow PEP8 with regard to coding style.
+
+Furthermore, triple-quotes should always be """, single quotes are ' unless
+using " would result in less escaping within the string.
+
+Public methods, functions and classes should all have doc strings. Please use
+epydoc style docstrings to document parameters and return values.
+You can generate the documentation by running "make doc".
+
+Running the tests
+-----------------
+To run the testsuite, you should be able to simply run "make check". This
+will run the tests using unittest on Python 2.7 and higher, and using
+unittest2 (which you will need to have installed) on older versions of Python.
+
+ $ make check

MANIFEST.in

@@ -1,7 +1,9 @@
 include NEWS
-include README
+include AUTHORS
+include README.md
 include Makefile
 include COPYING
+include HACKING
 include dulwich/stdint.h
 recursive-include docs conf.py *.txt Makefile make.bat
 recursive-include examples *.py

Makefile

@@ -1,4 +1,5 @@
 PYTHON = python
+PYLINT = pylint
 SETUP = $(PYTHON) setup.py
 PYDOCTOR ?= pydoctor
 ifeq ($(shell $(PYTHON) -c "import sys; print sys.version_info >= (2, 7)"),True)
@@ -47,3 +48,6 @@ check-all: check check-pypy check-noextensions
 clean::
 	$(SETUP) clean --all
 	rm -f dulwich/*.so
+
+lint::
+	$(PYLINT) --rcfile=.pylintrc --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" dulwich

NEWS

@@ -1,3 +1,57 @@
+0.9.6	2014-04-23
+
+ IMPROVEMENTS
+
+ * Add support for recursive add in 'git add'.
+   (Ryan Faulkner, Jelmer Vernooij)
+
+ * Add porcelain 'list_tags'. (Ryan Faulkner)
+
+ * Add porcelain 'push'. (Ryan Faulkner)
+
+ * Add porcelain 'pull'. (Ryan Faulkner)
+
+ * Support 'http.proxy' in HttpGitClient.
+   (Jelmer Vernooij, #1096030)
+
+ * Support 'http.useragent' in HttpGitClient.
+   (Jelmer Vernooij)
+
+ * In server, wait for clients to send empty list of
+   wants when talking to empty repository.
+   (Damien Tournoud)
+
+ * Various changes to improve compatibility with
+   Python 3. (Gary van der Merwe)
+
+ BUG FIXES
+
+ * Support unseekable 'wsgi.input' streams.
+   (Jonas Haag)
+
+ * Raise TypeError when passing unicode() object
+   to Repo.__getitem__.
+   (Jonas Haag)
+
+ * Fix handling of `reset` command in dulwich.fastexport.
+   (Jelmer Vernooij, #1249029)
+
+ * In client, don't wait for server to close connection
+   first. Fixes hang when used against GitHub
+   server implementation. (Siddharth Agarwal)
+
+ * DeltaChainIterator: fix a corner case where an object is inflated as an
+   object already in the repository.
+   (Damien Tournoud, #135)
+
+ * Stop leaking file handles during pack reload. (Damien Tournoud)
+
+ * Avoid reopening packs during pack cache reload. (Jelmer Vernooij)
+
+ API CHANGES
+
+  * Drop support for Python 2.6. (Jelmer Vernooij)
+
 0.9.5	2014-02-23
 
  IMPROVEMENTS
@@ -32,6 +86,10 @@
    unicode object of length 20 or 40 to Repo.__getitem__.
    (Jelmer Vernooij)
 
+ * Use 'rm' rather than 'unlink' in tests, since the latter
+   does not exist on OpenBSD and other platforms.
+   (Dmitrij D. Czarkoff)
+
 0.9.4	2013-11-30
 
  IMPROVEMENTS

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: dulwich
-Version: 0.9.5
+Version: 0.9.6
 Summary: Python Git Library
 Home-page: https://samba.org/~jelmer/dulwich
 Author: Jelmer Vernooij

bin/dulwich

@@ -218,28 +218,50 @@ def cmd_diff_tree(args):
 
 def cmd_rev_list(args):
     opts, args = getopt(args, "", [])
-    porcelain.rev_list(".", args)
+    if len(args) < 2:
+        print "Usage: dulwich tag NAME"
+        sys.exit(1)
+    porcelain.tag(".", args[0])
+
+
+def cmd_tag(args):
+    opts, args = getopt(args, "", [])
+    porcelain.tag(".", args[0])
+
+
+def cmd_reset(args):
+    opts, args = getopt(args, "", ["hard", "soft", "mixed"])
+    mode = ""
+    if "--hard" in opts:
+        mode = "hard"
+    elif "--soft" in opts:
+        mode = "soft"
+    elif "--mixed" in opts:
+        mode = "mixed"
+    porcelain.tag(".", mode=mode, *args)
 
 
 commands = {
+    "add": cmd_add,
+    "archive": cmd_archive,
+    "clone": cmd_clone,
     "commit": cmd_commit,
     "commit-tree": cmd_commit_tree,
+    "diff": cmd_diff,
     "diff-tree": cmd_diff_tree,
-    "fetch-pack": cmd_fetch_pack,
-    "fetch": cmd_fetch,
     "dump-pack": cmd_dump_pack,
     "dump-index": cmd_dump_index,
+    "fetch-pack": cmd_fetch_pack,
+    "fetch": cmd_fetch,
     "init": cmd_init,
     "log": cmd_log,
-    "clone": cmd_clone,
-    "archive": cmd_archive,
-    "update-server-info": cmd_update_server_info,
-    "symbolic-ref": cmd_symbolic_ref,
-    "diff": cmd_diff,
-    "add": cmd_add,
+    "reset": cmd_reset,
+    "rev-list": cmd_rev_list,
     "rm": cmd_rm,
     "show": cmd_show,
-    "rev-list": cmd_rev_list,
+    "symbolic-ref": cmd_symbolic_ref,
+    "tag": cmd_tag,
+    "update-server-info": cmd_update_server_info,
     }
 
 if len(sys.argv) < 2:

docs/conf.py

@@ -30,7 +30,7 @@ try:
     if rst2pdf.version >= '0.16':
         extensions.append('rst2pdf.pdfbuilder')
 except ImportError:
-    print "[NOTE] In order to build PDF you need rst2pdf with version >=0.16"
+    print("[NOTE] In order to build PDF you need rst2pdf with version >=0.16")
 
 
 autoclass_content = "both"

docs/tutorial/remote.txt

@@ -55,10 +55,10 @@ which claims that the client doesn't have any objects::
    ...     def next(self): pass
 
 With the ``determine_wants`` function in place, we can now fetch a pack,
-which we will write to a ``StringIO`` object::
+which we will write to a ``BytesIO`` object::
 
-   >>> from cStringIO import StringIO
-   >>> f = StringIO()
+   >>> from io import BytesIO
+   >>> f = BytesIO()
    >>> remote_refs = client.fetch_pack("/", determine_wants,
    ...    DummyGraphWalker(), pack_data=f.write)
 

dulwich.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: dulwich
-Version: 0.9.5
+Version: 0.9.6
 Summary: Python Git Library
 Home-page: https://samba.org/~jelmer/dulwich
 Author: Jelmer Vernooij

dulwich.egg-info/SOURCES.txt

@@ -1,8 +1,10 @@
+AUTHORS
 COPYING
+HACKING
 MANIFEST.in
 Makefile
 NEWS
-README
+README.md
 setup.cfg
 setup.py
 bin/dul-daemon
@@ -26,7 +28,6 @@ docs/tutorial/remote.txt
 docs/tutorial/repo.txt
 docs/tutorial/tag.txt
 dulwich/__init__.py
-dulwich/_compat.py
 dulwich/_diff_tree.c
 dulwich/_objects.c
 dulwich/_pack.c

dulwich/__init__.py

@@ -21,4 +21,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 9, 5)
+__version__ = (0, 9, 6)

dulwich/_compat.py

@@ -1,281 +0,0 @@
-# _compat.py -- For dealing with python2.4 oddness
-# Copyright (C) 2008 Canonical Ltd.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; version 2
-# of the License or (at your option) a later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-# MA  02110-1301, USA.
-
-"""Misc utilities to work with python <2.7.
-
-These utilities can all be deleted when dulwich decides it wants to stop
-support for python <2.7.
-"""
-
-# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
-# Passes Python2.7's test suite and incorporates all the latest updates.
-# Copyright (C) Raymond Hettinger, MIT license
-
-try:
-    from thread import get_ident as _get_ident
-except ImportError:
-    from dummy_thread import get_ident as _get_ident
-
-try:
-    from _abcoll import KeysView, ValuesView, ItemsView
-except ImportError:
-    pass
-
-class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as for regular dictionaries.
-
-    # The internal self.__map dictionary maps keys to links in a doubly linked list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
-
-    def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  Signature is the same as for
-        regular dictionaries, but keyword arguments are not recommended
-        because their insertion order is arbitrary.
-
-        '''
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__root
-        except AttributeError:
-            self.__root = root = []                     # sentinel node
-            root[:] = [root, root, None]
-            self.__map = {}
-        self.__update(*args, **kwds)
-
-    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
-        'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link which goes at the end of the linked
-        # list, and the inherited dictionary is updated with the new key/value pair.
-        if key not in self:
-            root = self.__root
-            last = root[0]
-            last[1] = root[0] = self.__map[key] = [last, root, key]
-        dict_setitem(self, key, value)
-
-    def __delitem__(self, key, dict_delitem=dict.__delitem__):
-        'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which is
-        # then removed by updating the links in the predecessor and successor nodes.
-        dict_delitem(self, key)
-        link_prev, link_next, key = self.__map.pop(key)
-        link_prev[1] = link_next
-        link_next[0] = link_prev
-
-    def __iter__(self):
-        'od.__iter__() <==> iter(od)'
-        root = self.__root
-        curr = root[1]
-        while curr is not root:
-            yield curr[2]
-            curr = curr[1]
-
-    def __reversed__(self):
-        'od.__reversed__() <==> reversed(od)'
-        root = self.__root
-        curr = root[0]
-        while curr is not root:
-            yield curr[2]
-            curr = curr[0]
-
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        try:
-            for node in self.__map.itervalues():
-                del node[:]
-            root = self.__root
-            root[:] = [root, root, None]
-            self.__map.clear()
-        except AttributeError:
-            pass
-        dict.clear(self)
-
-    def popitem(self, last=True):
-        """od.popitem() -> (k, v), return and remove a (key, value) pair.
-        Pairs are returned in LIFO order if last is true or FIFO order if false.
-
-        """
-        if not self:
-            raise KeyError('dictionary is empty')
-        root = self.__root
-        if last:
-            link = root[0]
-            link_prev = link[0]
-            link_prev[1] = root
-            root[0] = link_prev
-        else:
-            link = root[1]
-            link_next = link[1]
-            root[1] = link_next
-            link_next[0] = root
-        key = link[2]
-        del self.__map[key]
-        value = dict.pop(self, key)
-        return key, value
-
-    # -- the following methods do not depend on the internal structure --
-
-    def keys(self):
-        """'od.keys() -> list of keys in od"""
-        return list(self)
-
-    def values(self):
-        """od.values() -> list of values in od"""
-        return [self[key] for key in self]
-
-    def items(self):
-        """od.items() -> list of (key, value) pairs in od"""
-        return [(key, self[key]) for key in self]
-
-    def iterkeys(self):
-        """od.iterkeys() -> an iterator over the keys in od"""
-        return iter(self)
-
-    def itervalues(self):
-        """od.itervalues -> an iterator over the values in od"""
-        for k in self:
-            yield self[k]
-
-    def iteritems(self):
-        """od.iteritems -> an iterator over the (key, value) items in od"""
-        for k in self:
-            yield (k, self[k])
-
-    def update(*args, **kwds):
-        """od.update(E, F) -> None.  Update od from dict/iterable E and F.
-
-        If E is a dict instance, does:           for k in E: od[k] = E[k]
-        If E has a .keys() method, does:         for k in E.keys(): od[k] = E[k]
-        Or if E is an iterable of items, does:   for k, v in E: od[k] = v
-        In either case, this is followed by:     for k, v in F.items(): od[k] = v
-        """
-        if len(args) > 2:
-            raise TypeError('update() takes at most 2 positional '
-                            'arguments (%d given)' % (len(args),))
-        elif not args:
-            raise TypeError('update() takes at least 1 argument (0 given)')
-        self = args[0]
-        # Make progressively weaker assumptions about "other"
-        other = ()
-        if len(args) == 2:
-            other = args[1]
-        if isinstance(other, dict):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, 'keys'):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
-        for key, value in kwds.items():
-            self[key] = value
-
-    __update = update  # let subclasses override update without breaking __init__
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        """od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
-        If key is not found, d is returned if given, otherwise KeyError is raised.
-
-        """
-        if key in self:
-            result = self[key]
-            del self[key]
-            return result
-        if default is self.__marker:
-            raise KeyError(key)
-        return default
-
-    def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
-        if key in self:
-            return self[key]
-        self[key] = default
-        return default
-
-    def __repr__(self, _repr_running={}):
-        'od.__repr__() <==> repr(od)'
-        call_key = id(self), _get_ident()
-        if call_key in _repr_running:
-            return '...'
-        _repr_running[call_key] = 1
-        try:
-            if not self:
-                return '%s()' % (self.__class__.__name__,)
-            return '%s(%r)' % (self.__class__.__name__, self.items())
-        finally:
-            del _repr_running[call_key]
-
-    def __reduce__(self):
-        'Return state information for pickling'
-        items = [[k, self[k]] for k in self]
-        inst_dict = vars(self).copy()
-        for k in vars(OrderedDict()):
-            inst_dict.pop(k, None)
-        if inst_dict:
-            return (self.__class__, (items,), inst_dict)
-        return self.__class__, (items,)
-
-    def copy(self):
-        'od.copy() -> a shallow copy of od'
-        return self.__class__(self)
-
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
-        and values equal to v (which defaults to None).
-
-        '''
-        d = cls()
-        for key in iterable:
-            d[key] = value
-        return d
-
-    def __eq__(self, other):
-        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
-        while comparison to a regular mapping is order-insensitive.
-
-        '''
-        if isinstance(other, OrderedDict):
-            return len(self)==len(other) and self.items() == other.items()
-        return dict.__eq__(self, other)
-
-    def __ne__(self, other):
-        return not self == other
-
-    # -- the following methods are only used in Python 2.7 --
-
-    def viewkeys(self):
-        "od.viewkeys() -> a set-like object providing a view on od's keys"
-        return KeysView(self)
-
-    def viewvalues(self):
-        "od.viewvalues() -> an object providing a view on od's values"
-        return ValuesView(self)
-
-    def viewitems(self):
-        "od.viewitems() -> a set-like object providing a view on od's items"
-        return ItemsView(self)

dulwich/client.py

@@ -38,7 +38,8 @@ Known capabilities that are not supported:
 
 __docformat__ = 'restructuredText'
 
-from cStringIO import StringIO
+from io import BytesIO
+import dulwich
 import select
 import socket
 import subprocess
@@ -68,12 +69,6 @@ from dulwich.refs import (
     )
 
 
-# Python 2.6.6 included these in urlparse.uses_netloc upstream. Do
-# monkeypatching to enable similar behaviour in earlier Pythons:
-for scheme in ('git', 'git+ssh'):
-    if scheme not in urlparse.uses_netloc:
-        urlparse.uses_netloc.append(scheme)
-
 def _fileno_can_read(fileno):
     """Check if a file descriptor is readable."""
     return len(select.select([fileno], [], [], 0)[0]) > 0
@@ -339,10 +334,6 @@ class GitClient(object):
                     self._report_status_parser.handle_packet(pkt)
         if self._report_status_parser is not None:
             self._report_status_parser.check()
-        # wait for EOF before returning
-        data = proto.read()
-        if data:
-            raise SendPackError('Unexpected response %r' % data)
 
     def _handle_upload_pack_head(self, proto, capabilities, graph_walker,
                                  wants, can_read):
@@ -355,13 +346,13 @@ class GitClient(object):
         :param can_read: function that returns a boolean that indicates
             whether there is extra graph data to read on proto
         """
-        assert isinstance(wants, list) and type(wants[0]) == str
+        assert isinstance(wants, list) and isinstance(wants[0], str)
         proto.write_pkt_line('want %s %s\n' % (
             wants[0], ' '.join(capabilities)))
         for want in wants[1:]:
             proto.write_pkt_line('want %s\n' % want)
         proto.write_pkt_line(None)
-        have = graph_walker.next()
+        have = next(graph_walker)
         while have:
             proto.write_pkt_line('have %s\n' % have)
             if can_read():
@@ -377,7 +368,7 @@ class GitClient(object):
                         raise AssertionError(
                             "%s not in ('continue', 'ready', 'common)" %
                             parts[2])
-            have = graph_walker.next()
+            have = next(graph_walker)
         proto.write_pkt_line('done\n')
 
     def _handle_upload_pack_tail(self, proto, capabilities, graph_walker,
@@ -405,10 +396,6 @@ class GitClient(object):
                 # Just ignore progress data
                 progress = lambda x: None
             self._read_side_band64k_data(proto, {1: pack_data, 2: progress})
-            # wait for EOF before returning
-            data = proto.read()
-            if data:
-                raise Exception('Unexpected response %r' % data)
         else:
             while True:
                 data = proto.read(rbufsize)
@@ -860,7 +847,7 @@ else:
             channel = client.get_transport().open_session()
 
             # Run commands
-            apply(channel.exec_command, command)
+            channel.exec_command(*command)
 
             return ParamikoWrapper(client, channel,
                     progress_stderr=progress_stderr)
@@ -892,13 +879,36 @@ class SSHGitClient(TraditionalGitClient):
                 con.can_read)
 
 
+def default_user_agent_string():
+    return "dulwich/%s" % ".".join([str(x) for x in dulwich.__version__])
+
+
+def default_urllib2_opener(config):
+    if config is not None:
+        proxy_server = config.get("http", "proxy")
+    else:
+        proxy_server = None
+    handlers = []
+    if proxy_server is not None:
+        handlers.append(urllib2.ProxyHandler({"http" : proxy_server}))
+    opener = urllib2.build_opener(*handlers)
+    if config is not None:
+        user_agent = config.get("http", "useragent")
+    else:
+        user_agent = None
+    if user_agent is None:
+        user_agent = default_user_agent_string()
+    opener.addheaders = [('User-agent', user_agent)]
+    return opener
+
+
 class HttpGitClient(GitClient):
 
-    def __init__(self, base_url, dumb=None, opener=None, *args, **kwargs):
+    def __init__(self, base_url, dumb=None, opener=None, config=None, *args, **kwargs):
         self.base_url = base_url.rstrip("/") + "/"
         self.dumb = dumb
         if opener is None:
-            self.opener = urllib2.build_opener()
+            self.opener = default_urllib2_opener(config)
         else:
             self.opener = opener
         GitClient.__init__(self, *args, **kwargs)
@@ -973,7 +983,7 @@ class HttpGitClient(GitClient):
             return old_refs
         if self.dumb:
             raise NotImplementedError(self.fetch_pack)
-        req_data = StringIO()
+        req_data = BytesIO()
         req_proto = Protocol(None, req_data.write)
         (have, want) = self._handle_receive_pack_head(
             req_proto, negotiated_capabilities, old_refs, new_refs)
@@ -1010,7 +1020,7 @@ class HttpGitClient(GitClient):
             return refs
         if self.dumb:
             raise NotImplementedError(self.send_pack)
-        req_data = StringIO()
+        req_data = BytesIO()
         req_proto = Protocol(None, req_data.write)
         self._handle_upload_pack_head(req_proto,
             negotiated_capabilities, graph_walker, wants,
@@ -1023,10 +1033,11 @@ class HttpGitClient(GitClient):
         return refs
 
 
-def get_transport_and_path_from_url(url, **kwargs):
+def get_transport_and_path_from_url(url, config=None, **kwargs):
     """Obtain a git client from a URL.
 
     :param url: URL to open
+    :param config: Optional config object
     :param thin_packs: Whether or not thin packs should be retrieved
     :param report_activity: Optional callback for reporting transport
         activity.
@@ -1043,7 +1054,8 @@ def get_transport_and_path_from_url(url, **kwargs):
         return SSHGitClient(parsed.hostname, port=parsed.port,
                             username=parsed.username, **kwargs), path
     elif parsed.scheme in ('http', 'https'):
-        return HttpGitClient(urlparse.urlunparse(parsed), **kwargs), parsed.path
+        return HttpGitClient(urlparse.urlunparse(parsed), config=config,
+                **kwargs), parsed.path
     elif parsed.scheme == 'file':
         return default_local_git_client_cls(**kwargs), parsed.path
 
@@ -1054,6 +1066,7 @@ def get_transport_and_path(location, **kwargs):
     """Obtain a git client from a URL.
 
     :param location: URL or path
+    :param config: Optional config object
     :param thin_packs: Whether or not thin packs should be retrieved
     :param report_activity: Optional callback for reporting transport
         activity.

dulwich/config.py

@@ -28,12 +28,11 @@ import errno
 import os
 import re
 
-try:
-    from collections import OrderedDict
-except ImportError:
-    from dulwich._compat import OrderedDict
+from collections import (
+    OrderedDict,
+    MutableMapping,
+    )
 
-from UserDict import DictMixin
 
 from dulwich.file import GitFile
 
@@ -96,8 +95,7 @@ class Config(object):
         raise NotImplementedError(self.itersections)
 
 
-
-class ConfigDict(Config, DictMixin):
+class ConfigDict(Config, MutableMapping):
     """Git configuration stored in a dictionary."""
 
     def __init__(self, values=None):
@@ -115,13 +113,19 @@ class ConfigDict(Config, DictMixin):
             other._values == self._values)
 
     def __getitem__(self, key):
-        return self._values[key]
+        return self._values.__getitem__(key)
 
     def __setitem__(self, key, value):
-        self._values[key] = value
+        return self._values.__setitem__(key, value)
 
-    def keys(self):
-        return self._values.keys()
+    def __delitem__(self, key):
+        return self._values.__delitem__(key)
+
+    def __iter__(self):
+        return self._values.__iter__()
+
+    def __len__(self):
+        return self._values.__len__()
 
     @classmethod
     def _parse_setting(cls, name):

dulwich/diff_tree.py

@@ -23,7 +23,7 @@ from collections import (
     namedtuple,
     )
 
-from cStringIO import StringIO
+from io import BytesIO
 import itertools
 import stat
 
@@ -281,7 +281,7 @@ def _count_blocks(obj):
     :return: A dict of block hashcode -> total bytes occurring.
     """
     block_counts = defaultdict(int)
-    block = StringIO()
+    block = BytesIO()
     n = 0
 
     # Cache attrs as locals to avoid expensive lookups in the inner loop.

dulwich/fastexport.py

@@ -204,7 +204,7 @@ class GitImportProcessor(processor.ImportProcessor):
     def reset_handler(self, cmd):
         """Process a ResetCommand."""
         self._reset_base(cmd.from_)
-        self.rep.refs[cmd.from_] = cmd.id
+        self.repo.refs[cmd.ref] = cmd.from_
 
     def tag_handler(self, cmd):
         """Process a TagCommand."""

dulwich/index.py

@@ -177,8 +177,8 @@ def cleanup_mode(mode):
         return stat.S_IFDIR
     elif S_ISGITLINK(mode):
         return S_IFGITLINK
-    ret = stat.S_IFREG | 0644
-    ret |= (mode & 0111)
+    ret = stat.S_IFREG | 0o644
+    ret |= (mode & 0o111)
     return ret
 
 
@@ -325,7 +325,7 @@ def commit_tree(object_store, blobs):
     def build_tree(path):
         tree = Tree()
         for basename, entry in trees[path].iteritems():
-            if type(entry) == dict:
+            if isinstance(entry, dict):
                 mode = stat.S_IFDIR
                 sha = build_tree(pathjoin(path, basename))
             else:

dulwich/object_store.py

@@ -21,7 +21,7 @@
 """Git object store interfaces and implementation."""
 
 
-from cStringIO import StringIO
+from io import BytesIO
 import errno
 import itertools
 import os
@@ -185,12 +185,12 @@ class BaseObjectStore(object):
         :return: List of SHAs that are in common
         """
         haves = []
-        sha = graphwalker.next()
+        sha = next(graphwalker)
         while sha:
             if sha in self:
                 haves.append(sha)
                 graphwalker.ack(sha)
-            sha = graphwalker.next()
+            sha = next(graphwalker)
         return haves
 
     def generate_pack_contents(self, have, want, progress=None):
@@ -251,7 +251,7 @@ class BaseObjectStore(object):
 class PackBasedObjectStore(BaseObjectStore):
 
     def __init__(self):
-        self._pack_cache = None
+        self._pack_cache = {}
 
     @property
     def alternates(self):
@@ -279,33 +279,30 @@ class PackBasedObjectStore(BaseObjectStore):
                 return True
         return False
 
-    def _load_packs(self):
-        raise NotImplementedError(self._load_packs)
-
     def _pack_cache_stale(self):
         """Check whether the pack cache is stale."""
         raise NotImplementedError(self._pack_cache_stale)
 
-    def _add_known_pack(self, pack):
+    def _add_known_pack(self, base_name, pack):
         """Add a newly appeared pack to the cache by path.
 
         """
-        if self._pack_cache is not None:
-            self._pack_cache.append(pack)
+        self._pack_cache[base_name] = pack
 
     def close(self):
         pack_cache = self._pack_cache
-        self._pack_cache = None
+        self._pack_cache = {}
         while pack_cache:
-            pack = pack_cache.pop()
+            (name, pack) = pack_cache.popitem()
             pack.close()
 
     @property
     def packs(self):
         """List with pack objects."""
         if self._pack_cache is None or self._pack_cache_stale():
-            self._pack_cache = self._load_packs()
-        return self._pack_cache
+            self._update_pack_cache()
+
+        return self._pack_cache.values()
 
     def _iter_alternate_objects(self):
         """Iterate over the SHAs of all the objects in alternate stores."""
@@ -410,6 +407,7 @@ class DiskObjectStore(PackBasedObjectStore):
         self.path = path
         self.pack_dir = os.path.join(self.path, PACKDIR)
         self._pack_cache_time = 0
+        self._pack_cache = {}
         self._alternates = None
 
     def __repr__(self):
@@ -475,31 +473,29 @@ class DiskObjectStore(PackBasedObjectStore):
             path = os.path.join(self.path, path)
         self.alternates.append(DiskObjectStore(path))
 
-    def _load_packs(self):
-        pack_files = []
+    def _update_pack_cache(self):
         try:
-            self._pack_cache_time = os.stat(self.pack_dir).st_mtime
             pack_dir_contents = os.listdir(self.pack_dir)
-            for name in pack_dir_contents:
-                # TODO: verify that idx exists first
-                if name.startswith("pack-") and name.endswith(".pack"):
-                    filename = os.path.join(self.pack_dir, name)
-                    pack_files.append((os.stat(filename).st_mtime, filename))
         except OSError as e:
             if e.errno == errno.ENOENT:
-                return []
-            raise
-        pack_files.sort(reverse=True)
-        suffix_len = len(".pack")
-        result = []
-        try:
-            for _, f in pack_files:
-                result.append(Pack(f[:-suffix_len]))
-        except:
-            for p in result:
-                p.close()
+                self._pack_cache_time = 0
+                self.close()
+                return
             raise
-        return result
+        self._pack_cache_time = os.stat(self.pack_dir).st_mtime
+        pack_files = set()
+        for name in pack_dir_contents:
+            # TODO: verify that idx exists first
+            if name.startswith("pack-") and name.endswith(".pack"):
+                pack_files.add(name[:-len(".pack")])
+
+        # Open newly appeared pack files
+        for f in pack_files:
+            if f not in self._pack_cache:
+                self._pack_cache[f] = Pack(os.path.join(self.pack_dir, f))
+        # Remove disappeared pack files
+        for f in set(self._pack_cache) - pack_files:
+            self._pack_cache.pop(f).close()
 
     def _pack_cache_stale(self):
         try:
@@ -586,7 +582,7 @@ class DiskObjectStore(PackBasedObjectStore):
         # Add the pack to the store and return it.
         final_pack = Pack(pack_base_name)
         final_pack.check_length_and_checksum()
-        self._add_known_pack(final_pack)
+        self._add_known_pack(pack_base_name, final_pack)
         return final_pack
 
     def add_thin_pack(self, read_all, read_some):
@@ -637,7 +633,7 @@ class DiskObjectStore(PackBasedObjectStore):
             p.close()
         os.rename(path, basename + ".pack")
         final_pack = Pack(basename)
-        self._add_known_pack(final_pack)
+        self._add_known_pack(basename, final_pack)
         return final_pack
 
     def add_pack(self):
@@ -765,9 +761,9 @@ class MemoryObjectStore(BaseObjectStore):
         :return: Fileobject to write to and a commit function to
             call when the pack is finished.
         """
-        f = StringIO()
+        f = BytesIO()
         def commit():
-            p = PackData.from_file(StringIO(f.getvalue()), f.tell())
+            p = PackData.from_file(BytesIO(f.getvalue()), f.tell())
             f.close()
             for obj in PackInflater.for_pack_data(p):
                 self._data[obj.id] = obj

dulwich/objects.py

@@ -20,9 +20,7 @@
 """Access to base git objects."""
 
 import binascii
-from cStringIO import (
-    StringIO,
-    )
+from io import BytesIO
 from collections import namedtuple
 import os
 import posixpath
@@ -58,7 +56,7 @@ _TAG_HEADER = "tag"
 _TAGGER_HEADER = "tagger"
 
 
-S_IFGITLINK = 0160000
+S_IFGITLINK = 0o160000
 
 def S_ISGITLINK(m):
     """Check if a mode indicates a submodule.
@@ -291,7 +289,7 @@ class ShaFile(object):
 
     def set_raw_string(self, text, sha=None):
         """Set the contents of this object from a serialized string."""
-        if type(text) != str:
+        if not isinstance(text, str):
             raise TypeError(text)
         self.set_raw_chunks([text], sha)
 
@@ -590,7 +588,7 @@ def _parse_message(chunks):
         order read from the text, possibly including duplicates. Includes a
         field named None for the freeform tag/commit text.
     """
-    f = StringIO("".join(chunks))
+    f = BytesIO("".join(chunks))
     k = None
     v = ""
     for l in f:
@@ -740,7 +738,7 @@ class TreeEntry(namedtuple('TreeEntry', ['path', 'mode', 'sha'])):
 
     def in_path(self, path):
         """Return a copy of this entry with the given path prepended."""
-        if type(self.path) != str:
+        if not isinstance(self.path, str):
             raise TypeError
         return TreeEntry(posixpath.join(path, self.path), self.mode, self.sha)
 
@@ -792,8 +790,8 @@ def sorted_tree_items(entries, name_order):
     :param entries: Dictionary mapping names to (mode, sha) tuples
     :return: Iterator over (name, mode, hexsha)
     """
-    cmp_func = name_order and cmp_entry_name_order or cmp_entry
-    for name, entry in sorted(entries.iteritems(), cmp=cmp_func):
+    key_func = name_order and key_entry_name_order or key_entry
+    for name, entry in sorted(entries.iteritems(), key=key_func):
         mode, hexsha = entry
         # Stricter type checks than normal to mirror checks in the C version.
         if not isinstance(mode, int) and not isinstance(mode, long):
@@ -804,18 +802,20 @@ def sorted_tree_items(entries, name_order):
         yield TreeEntry(name, mode, hexsha)
 
 
-def cmp_entry((name1, value1), (name2, value2)):
-    """Compare two tree entries in tree order."""
-    if stat.S_ISDIR(value1[0]):
-        name1 += "/"
-    if stat.S_ISDIR(value2[0]):
-        name2 += "/"
-    return cmp(name1, name2)
+def key_entry(entry):
+    """Sort key for tree entry.
+
+    :param entry: (name, value) tuplee
+    """
+    (name, value) = entry
+    if stat.S_ISDIR(value[0]):
+        name += "/"
+    return name
 
 
-def cmp_entry_name_order(entry1, entry2):
-    """Compare two tree entries in name order."""
-    return cmp(entry1[0], entry2[0])
+def key_entry_name_order(entry):
+    """Sort key for tree entry in name order."""
+    return entry[0]
 
 
 class Tree(ShaFile):
@@ -879,7 +879,7 @@ class Tree(ShaFile):
         :param name: The name of the entry, as a string.
         :param hexsha: The hex SHA of the entry as a string.
         """
-        if type(name) is int and type(mode) is str:
+        if isinstance(name, int) and isinstance(mode, str):
             (name, mode) = (mode, name)
             warnings.warn("Please use Tree.add(name, mode, hexsha)",
                 category=DeprecationWarning, stacklevel=2)
@@ -920,10 +920,10 @@ class Tree(ShaFile):
         """
         super(Tree, self).check()
         last = None
-        allowed_modes = (stat.S_IFREG | 0755, stat.S_IFREG | 0644,
+        allowed_modes = (stat.S_IFREG | 0o755, stat.S_IFREG | 0o644,
                          stat.S_IFLNK, stat.S_IFDIR, S_IFGITLINK,
                          # TODO: optionally exclude as in git fsck --strict
-                         stat.S_IFREG | 0664)
+                         stat.S_IFREG | 0o664)
         for name, mode, sha in parse_tree(''.join(self._chunked_text),
                                           True):
             check_hexsha(sha, 'invalid sha %s' % sha)
@@ -935,7 +935,7 @@ class Tree(ShaFile):
 
             entry = (name, (mode, sha))
             if last:
-                if cmp_entry(last, entry) > 0:
+                if key_entry(last) > key_entry(entry):
                     raise ObjectFormatException('entries not sorted')
                 if name == last[0]:
                     raise ObjectFormatException('duplicate entry %s' % name)

dulwich/pack.py

@@ -33,9 +33,7 @@ a pointer in to the corresponding packfile.
 from collections import defaultdict
 
 import binascii
-from cStringIO import (
-    StringIO,
-    )
+from io import BytesIO
 from collections import (
     deque,
     )
@@ -624,7 +622,7 @@ class PackIndex2(FilePackIndex):
         offset = self._pack_offset_table_offset + i * 4
         offset = unpack_from('>L', self._contents, offset)[0]
         if offset & (2**31):
-            offset = self._pack_offset_largetable_offset + (offset&(2**31-1)) * 8L
+            offset = self._pack_offset_largetable_offset + (offset&(2**31-1)) * 8
             offset = unpack_from('>Q', self._contents, offset)[0]
         return offset
 
@@ -719,8 +717,9 @@ def unpack_object(read_all, read_some=None, compute_crc32=False,
     return unpacked, unused
 
 
-def _compute_object_size((num, obj)):
+def _compute_object_size(value):
     """Compute the size of a unresolved object for use with LRUSizeCache."""
+    (num, obj) = value
     if num in DELTA_TYPES:
         return chunks_length(obj[1])
     return chunks_length(obj)
@@ -741,7 +740,7 @@ class PackStreamReader(object):
             self.read_some = read_some
         self.sha = sha1()
         self._offset = 0
-        self._rbuf = StringIO()
+        self._rbuf = BytesIO()
         # trailer is a deque to avoid memory allocation on small reads
         self._trailer = deque()
         self._zlib_bufsize = zlib_bufsize
@@ -794,7 +793,7 @@ class PackStreamReader(object):
         if buf_len >= size:
             return self._rbuf.read(size)
         buf_data = self._rbuf.read()
-        self._rbuf = StringIO()
+        self._rbuf = BytesIO()
         return buf_data + self._read(self.read_all, size - buf_len)
 
     def recv(self, size):
@@ -803,7 +802,7 @@ class PackStreamReader(object):
         if buf_len:
             data = self._rbuf.read(size)
             if size >= buf_len:
-                self._rbuf = StringIO()
+                self._rbuf = BytesIO()
             return data
         return self._read(self.read_some, size)
 
@@ -840,7 +839,7 @@ class PackStreamReader(object):
             unpacked.offset = offset
 
             # prepend any unused data to current read buffer
-            buf = StringIO()
+            buf = BytesIO()
             buf.write(unused)
             buf.write(self._rbuf.read())
             buf.seek(0)
@@ -1265,6 +1264,8 @@ class DeltaChainIterator(object):
             return
 
         for base_sha, pending in sorted(self._pending_ref.iteritems()):
+            if base_sha not in self._pending_ref:
+                continue
             try:
                 type_num, chunks = self._resolve_ext_ref(base_sha)
             except KeyError:
@@ -1650,9 +1651,9 @@ def apply_delta(src_buf, delta):
     :param src_buf: Source buffer
     :param delta: Delta instructions
     """
-    if type(src_buf) != str:
+    if not isinstance(src_buf, str):
         src_buf = ''.join(src_buf)
-    if type(delta) != str:
+    if not isinstance(delta, str):
         delta = ''.join(delta)
     out = []
     index = 0
@@ -1806,7 +1807,7 @@ class Pack(object):
             self._idx.close()
 
     def __eq__(self, other):
-        return type(self) == type(other) and self.index == other.index
+        return isinstance(self, type(other)) and self.index == other.index
 
     def __len__(self):
         """Number of entries in this pack."""

dulwich/patch.py

@@ -22,8 +22,9 @@ These patches are basically unified diffs with some extra metadata tacked
 on.
 """
 
+from io import BytesIO
 from difflib import SequenceMatcher
-import rfc822
+import email.parser
 import time
 
 from dulwich.objects import (
@@ -114,20 +115,20 @@ def is_binary(content):
     return '\0' in content[:FIRST_FEW_BYTES]
 
 
-def write_object_diff(f, store, (old_path, old_mode, old_id),
-                                (new_path, new_mode, new_id),
-                                diff_binary=False):
+def write_object_diff(f, store, old_file, new_file, diff_binary=False):
     """Write the diff for an object.
 
     :param f: File-like object to write to
     :param store: Store to retrieve objects from, if necessary
-    :param (old_path, old_mode, old_hexsha): Old file
-    :param (new_path, new_mode, new_hexsha): New file
+    :param old_file: (path, mode, hexsha) tuple
+    :param new_file: (path, mode, hexsha) tuple
     :param diff_binary: Whether to diff files even if they
         are considered binary files by is_binary().
 
     :note: the tuple elements should be None for nonexistant files
     """
+    (old_path, old_mode, old_id) = old_file
+    (new_path, new_mode, new_id) = new_file
     def shortid(hexsha):
         if hexsha is None:
             return "0" * 7
@@ -177,16 +178,17 @@ def write_object_diff(f, store, (old_path, old_mode, old_id),
             old_path, new_path))
 
 
-def write_blob_diff(f, (old_path, old_mode, old_blob),
-                       (new_path, new_mode, new_blob)):
+def write_blob_diff(f, old_file, new_file):
     """Write diff file header.
 
     :param f: File-like object to write to
-    :param (old_path, old_mode, old_blob): Previous file (None if nonexisting)
-    :param (new_path, new_mode, new_blob): New file (None if nonexisting)
+    :param old_file: (path, mode, hexsha) tuple (None if nonexisting)
+    :param new_file: (path, mode, hexsha) tuple (None if nonexisting)
 
     :note: The use of write_object_diff is recommended over this function.
     """
+    (old_path, old_mode, old_blob) = old_file
+    (new_path, new_mode, new_blob) = new_file
     def blob_id(blob):
         if blob is None:
             return "0" * 7
@@ -245,7 +247,8 @@ def git_am_patch_split(f):
     :param f: File-like object to parse
     :return: Tuple with commit object, diff contents and git version
     """
-    msg = rfc822.Message(f)
+    parser = email.parser.Parser()
+    msg = parser.parse(f)
     c = Commit()
     c.author = msg["from"]
     c.committer = msg["from"]
@@ -258,7 +261,10 @@ def git_am_patch_split(f):
         subject = msg["subject"][close+2:]
     c.message = subject.replace("\n", "") + "\n"
     first = True
-    for l in f:
+
+    body = BytesIO(msg.get_payload())
+
+    for l in body:
         if l == "---\n":
             break
         if first:
@@ -270,12 +276,12 @@ def git_am_patch_split(f):
         else:
             c.message += l
     diff = ""
-    for l in f:
+    for l in body:
         if l == "-- \n":
             break
         diff += l
     try:
-        version = f.next().rstrip("\n")
+        version = next(body).rstrip("\n")
     except StopIteration:
         version = None
     return c, diff, version

dulwich/porcelain.py

@@ -16,22 +16,6 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-import os
-import sys
-import time
-
-from dulwich import index
-from dulwich.client import get_transport_and_path
-from dulwich.objects import (
-    Commit,
-    Tag,
-    parse_timezone,
-    )
-from dulwich.objectspec import parse_object
-from dulwich.patch import write_tree_diff
-from dulwich.repo import (BaseRepo, Repo)
-from dulwich.server import update_server_info as server_update_server_info
-
 """Simple wrapper that provides porcelain-like functions on top of Dulwich.
 
 Currently implemented:
@@ -42,7 +26,10 @@ Currently implemented:
  * commit-tree
  * diff-tree
  * init
- * remove
+ * list-tags
+ * pull
+ * push
+ * rm
  * reset
  * rev-list
  * tag
@@ -55,6 +42,25 @@ Differences in behaviour are considered bugs.
 
 __docformat__ = 'restructuredText'
 
+import os
+import sys
+import time
+
+from dulwich import index
+from dulwich.client import get_transport_and_path
+from dulwich.errors import (
+    SendPackError,
+    UpdateRefsError,
+    )
+from dulwich.objects import (
+    Tag,
+    parse_timezone,
+    )
+from dulwich.objectspec import parse_object
+from dulwich.patch import write_tree_diff
+from dulwich.repo import (BaseRepo, Repo)
+from dulwich.server import update_server_info as server_update_server_info
+
 
 def open_repo(path_or_repo):
     """Open an argument that can be a repository or a path for a repository."""
@@ -187,10 +193,19 @@ def add(repo=".", paths=None):
     """Add files to the staging area.
 
     :param repo: Repository for the files
-    :param paths: Paths to add
+    :param paths: Paths to add.  No value passed stages all modified files.
     """
-    # FIXME: Support patterns, directories, no argument.
+    # FIXME: Support patterns, directories.
     r = open_repo(repo)
+    if not paths:
+        # If nothing is specified, add all non-ignored files.
+        paths = []
+        for dirpath, dirnames, filenames in os.walk(r.path):
+            # Skip .git and below.
+            if '.git' in dirnames:
+                dirnames.remove('.git')
+            for filename in filenames:
+                paths.append(os.path.join(dirpath[len(r.path)+1:], filename))
     r.stage(paths)
 
 
@@ -343,29 +358,60 @@ def rev_list(repo, commits, outstream=sys.stdout):
         outstream.write("%s\n" % entry.commit.id)
 
 
-def tag(repo, tag, author, message):
+def tag(repo, tag, author=None, message=None, annotated=False,
+        objectish="HEAD", tag_time=None, tag_timezone=None):
     """Creates a tag in git via dulwich calls:
 
     :param repo: Path to repository
     :param tag: tag string
-    :param author: tag author
-    :param repo: tag message
+    :param author: tag author (optional, if annotated is set)
+    :param message: tag message (optional)
+    :param annotated: whether to create an annotated tag
+    :param objectish: object the tag should point at, defaults to HEAD
+    :param tag_time: Optional time for annotated tag
+    :param tag_timezone: Optional timezone for annotated tag
     """
 
     r = open_repo(repo)
+    object = parse_object(r, objectish)
+
+    if annotated:
+        # Create the tag object
+        tag_obj = Tag()
+        if author is None:
+            # TODO(jelmer): Don't use repo private method.
+            author = r._get_user_identity()
+        tag_obj.tagger = author
+        tag_obj.message = message
+        tag_obj.name = tag
+        tag_obj.object = (type(object), object.id)
+        tag_obj.tag_time = tag_time
+        if tag_time is None:
+            tag_time = int(time.time())
+        if tag_timezone is None:
+            # TODO(jelmer) Use current user timezone rather than UTC
+            tag_timezone = 0
+        elif isinstance(tag_timezone, str):
+            tag_timezone = parse_timezone(tag_timezone)
+        tag_obj.tag_timezone = tag_timezone
+        r.object_store.add_object(tag_obj)
+        tag_id = tag_obj.id
+    else:
+        tag_id = object.id
+
+    r.refs['refs/tags/' + tag] = tag_id
 
-    # Create the tag object
-    tag_obj = Tag()
-    tag_obj.tagger = author
-    tag_obj.message = message
-    tag_obj.name = tag
-    tag_obj.object = (Commit, r.refs['HEAD'])
-    tag_obj.tag_time = int(time.time())
-    tag_obj.tag_timezone = parse_timezone('-0200')[0]
 
-    # Add tag to the object store
-    r.object_store.add_object(tag_obj)
-    r.refs['refs/tags/' + tag] = tag_obj.id
+def list_tags(repo, outstream=sys.stdout):
+    """List all tags.
+
+    :param repo: Path to repository
+    :param outstream: Stream to write tags to
+    """
+    r = open_repo(repo)
+    tags = list(r.refs.as_dict("refs/tags"))
+    tags.sort()
+    return tags
 
 
 def reset(repo, mode, committish="HEAD"):
@@ -383,3 +429,59 @@ def reset(repo, mode, committish="HEAD"):
     indexfile = r.index_path()
     tree = r[committish].tree
     index.build_index_from_tree(r.path, indexfile, r.object_store, tree)
+
+
+def push(repo, remote_location, refs_path,
+         outstream=sys.stdout, errstream=sys.stderr):
+    """Remote push with dulwich via dulwich.client
+
+    :param repo: Path to repository
+    :param remote_location: Location of the remote
+    :param refs_path: relative path to the refs to push to remote
+    :param outstream: A stream file to write output
+    :param errstream: A stream file to write errors
+    """
+
+    # Open the repo
+    r = open_repo(repo)
+
+    # Get the client and path
+    client, path = get_transport_and_path(remote_location)
+
+    def update_refs(refs):
+        new_refs = r.get_refs()
+        refs[refs_path] = new_refs['HEAD']
+        del new_refs['HEAD']
+        return refs
+
+    try:
+        client.send_pack(path, update_refs,
+            r.object_store.generate_pack_contents, progress=errstream.write)
+        outstream.write("Push to %s successful.\n" % remote_location)
+    except (UpdateRefsError, SendPackError) as e:
+        outstream.write("Push to %s failed.\n" % remote_location)
+        errstream.write("Push to %s failed -> '%s'\n" % e.message)
+
+
+def pull(repo, remote_location, refs_path,
+         outstream=sys.stdout, errstream=sys.stderr):
+    """Pull from remote via dulwich.client
+
+    :param repo: Path to repository
+    :param remote_location: Location of the remote
+    :param refs_path: relative path to the fetched refs
+    :param outstream: A stream file to write to output
+    :param errstream: A stream file to write to errors
+    """
+
+    # Open the repo
+    r = open_repo(repo)
+
+    client, path = get_transport_and_path(remote_location)
+    remote_refs = client.fetch(path, r, progress=errstream.write)
+    r['HEAD'] = remote_refs[refs_path]
+
+    # Perform 'git checkout .' - syncs staged changes
+    indexfile = r.index_path()
+    tree = r["HEAD"].tree
+    index.build_index_from_tree(r.path, indexfile, r.object_store, tree)

dulwich/protocol.py

@@ -19,7 +19,7 @@
 
 """Generic functions for talking the git smart server protocol."""
 
-from cStringIO import StringIO
+from io import BytesIO
 from os import (
     SEEK_END,
     )
@@ -137,7 +137,7 @@ class Protocol(object):
         """
         if self._readahead is not None:
             raise ValueError('Attempted to unread multiple pkt-lines.')
-        self._readahead = StringIO(pkt_line(data))
+        self._readahead = BytesIO(pkt_line(data))
 
     def read_pkt_seq(self):
         """Read a sequence of pkt-lines from the remote git process.
@@ -240,7 +240,7 @@ class ReceivableProtocol(Protocol):
         super(ReceivableProtocol, self).__init__(self.read, write,
                                                  report_activity)
         self._recv = recv
-        self._rbuf = StringIO()
+        self._rbuf = BytesIO()
         self._rbufsize = rbufsize
 
     def read(self, size):
@@ -252,10 +252,10 @@ class ReceivableProtocol(Protocol):
         #  - use SEEK_END instead of the magic number.
         # Copyright (c) 2001-2010 Python Software Foundation; All Rights Reserved
         # Licensed under the Python Software Foundation License.
-        # TODO: see if buffer is more efficient than cStringIO.
+        # TODO: see if buffer is more efficient than cBytesIO.
         assert size > 0
 
-        # Our use of StringIO rather than lists of string objects returned by
+        # Our use of BytesIO rather than lists of string objects returned by
         # recv() minimizes memory usage and fragmentation that occurs when
         # rbufsize is large compared to the typical return value of recv().
         buf = self._rbuf
@@ -267,18 +267,18 @@ class ReceivableProtocol(Protocol):
             # Already have size bytes in our buffer?  Extract and return.
             buf.seek(start)
             rv = buf.read(size)
-            self._rbuf = StringIO()
+            self._rbuf = BytesIO()
             self._rbuf.write(buf.read())
             self._rbuf.seek(0)
             return rv
 
-        self._rbuf = StringIO()  # reset _rbuf.  we consume it via buf.
+        self._rbuf = BytesIO()  # reset _rbuf.  we consume it via buf.
         while True:
             left = size - buf_len
             # recv() will malloc the amount of memory given as its
             # parameter even though it often returns much less data
             # than that.  The returned data string is short lived
-            # as we copy it into a StringIO and free it.  This avoids
+            # as we copy it into a BytesIO and free it.  This avoids
             # fragmentation issues on many platforms.
             data = self._recv(left)
             if not data:
@@ -319,7 +319,7 @@ class ReceivableProtocol(Protocol):
             if len(data) == size:
                 # shortcut: skip the buffer if we read exactly size bytes
                 return data
-            buf = StringIO()
+            buf = BytesIO()
             buf.write(data)
             buf.seek(0)
             del data  # explicit free
@@ -381,7 +381,7 @@ class BufferedPktLineWriter(object):
         """
         self._write = write
         self._bufsize = bufsize
-        self._wbuf = StringIO()
+        self._wbuf = BytesIO()
         self._buflen = 0
 
     def write(self, data):
@@ -405,7 +405,7 @@ class BufferedPktLineWriter(object):
         if data:
             self._write(data)
         self._len = 0
-        self._wbuf = StringIO()
+        self._wbuf = BytesIO()
 
 
 class PktLineParser(object):
@@ -414,7 +414,7 @@ class PktLineParser(object):
 
     def __init__(self, handle_pkt):
         self.handle_pkt = handle_pkt
-        self._readahead = StringIO()
+        self._readahead = BytesIO()
 
     def parse(self, data):
         """Parse a fragment of data and call back for any completed packets.
@@ -433,7 +433,7 @@ class PktLineParser(object):
                 buf = buf[size:]
             else:
                 break
-        self._readahead = StringIO()
+        self._readahead = BytesIO()
         self._readahead.write(buf)
 
     def get_tail(self):

dulwich/refs.py

@@ -59,7 +59,7 @@ def check_ref_format(refname):
     if '..' in refname:
         return False
     for c in refname:
-        if ord(c) < 040 or c in '\177 ~^:?*[':
+        if ord(c) < 0o40 or c in '\177 ~^:?*[':
             return False
     if refname[-1] in '/.':
         return False
@@ -447,7 +447,7 @@ class DiskRefsContainer(RefsContainer):
                     return {}
                 raise
             try:
-                first_line = iter(f).next().rstrip()
+                first_line = next(iter(f)).rstrip()
                 if (first_line.startswith("# pack-refs") and " peeled" in
                         first_line):
                     for sha, name, peeled in read_packed_refs_with_peeled(f):
@@ -498,7 +498,7 @@ class DiskRefsContainer(RefsContainer):
                 header = f.read(len(SYMREF))
                 if header == SYMREF:
                     # Read only the first line
-                    return header + iter(f).next().rstrip("\r\n")
+                    return header + next(iter(f)).rstrip("\r\n")
                 else:
                     # Read only the first 40 bytes
                     return header + f.read(40 - len(SYMREF))
@@ -649,7 +649,7 @@ class DiskRefsContainer(RefsContainer):
             # may only be packed
             try:
                 os.remove(filename)
-            except OSError, e:
+            except OSError as e:
                 if e.errno != errno.ENOENT:
                     raise
             self._remove_packed_ref(name)
@@ -667,7 +667,7 @@ def _split_ref_line(line):
     sha, name = fields
     try:
         hex_to_sha(sha)
-    except (AssertionError, TypeError), e:
+    except (AssertionError, TypeError) as e:
         raise PackedRefsException(e)
     if not check_ref_format(name):
         raise PackedRefsException("invalid ref name '%s'" % name)
@@ -708,7 +708,7 @@ def read_packed_refs_with_peeled(f):
                 raise PackedRefsException("unexpected peeled ref line")
             try:
                 hex_to_sha(l[1:])
-            except (AssertionError, TypeError), e:
+            except (AssertionError, TypeError) as e:
                 raise PackedRefsException(e)
             sha, name = _split_ref_line(last)
             last = None

dulwich/repo.py

@@ -27,7 +27,7 @@ local disk (Repo).
 
 """
 
-from cStringIO import StringIO
+from io import BytesIO
 import errno
 import os
 
@@ -176,7 +176,7 @@ class BaseRepo(object):
         """Initialize a default set of named files."""
         from dulwich.config import ConfigFile
         self._put_named_file('description', "Unnamed repository")
-        f = StringIO()
+        f = BytesIO()
         cf = ConfigFile()
         cf.set("core", "repositoryformatversion", "0")
         cf.set("core", "filemode", "true")
@@ -246,7 +246,7 @@ class BaseRepo(object):
         :return: iterator over objects, with __len__ implemented
         """
         wants = determine_wants(self.get_refs())
-        if type(wants) is not list:
+        if not isinstance(wants, list):
             raise TypeError("determine_wants() did not return a list")
 
         shallows = getattr(graph_walker, 'shallow', frozenset())
@@ -439,7 +439,10 @@ class BaseRepo(object):
         :return: A `ShaFile` object, such as a Commit or Blob
         :raise KeyError: when the specified ref or object does not exist
         """
-        if len(name) in (20, 40) and isinstance(name, str):
+        if not isinstance(name, str):
+            raise TypeError("'name' must be bytestring, not %.80s" %
+                    type(name).__name__)
+        if len(name) in (20, 40):
             try:
                 return self.object_store[name]
             except (KeyError, ValueError):
@@ -548,7 +551,7 @@ class BaseRepo(object):
 
         try:
             self.hooks['pre-commit'].execute()
-        except HookError, e:
+        except HookError as e:
             raise CommitError(e)
         except KeyError:  # no hook defined, silent fallthrough
             pass
@@ -591,28 +594,33 @@ class BaseRepo(object):
             c.message = self.hooks['commit-msg'].execute(message)
             if c.message is None:
                 c.message = message
-        except HookError, e:
+        except HookError as e:
             raise CommitError(e)
         except KeyError:  # no hook defined, message not modified
             c.message = message
 
-        try:
-            old_head = self.refs[ref]
-            c.parents = [old_head] + merge_heads
-            self.object_store.add_object(c)
-            ok = self.refs.set_if_equals(ref, old_head, c.id)
-        except KeyError:
+        if ref is None:
+            # Create a dangling commit
             c.parents = merge_heads
             self.object_store.add_object(c)
-            ok = self.refs.add_if_new(ref, c.id)
-        if not ok:
-            # Fail if the atomic compare-and-swap failed, leaving the commit and
-            # all its objects as garbage.
-            raise CommitError("%s changed during commit" % (ref,))
+        else:
+            try:
+                old_head = self.refs[ref]
+                c.parents = [old_head] + merge_heads
+                self.object_store.add_object(c)
+                ok = self.refs.set_if_equals(ref, old_head, c.id)
+            except KeyError:
+                c.parents = merge_heads
+                self.object_store.add_object(c)
+                ok = self.refs.add_if_new(ref, c.id)
+            if not ok:
+                # Fail if the atomic compare-and-swap failed, leaving the commit and
+                # all its objects as garbage.
+                raise CommitError("%s changed during commit" % (ref,))
 
         try:
             self.hooks['post-commit'].execute()
-        except HookError, e:  # silent failure
+        except HookError as e:  # silent failure
             warnings.warn("post-commit hook failed: %s" % e, UserWarning)
         except KeyError:  # no hook defined, silent fallthrough
             pass
@@ -700,7 +708,7 @@ class Repo(BaseRepo):
         path = path.lstrip(os.path.sep)
         try:
             return open(os.path.join(self.controldir(), path), 'rb')
-        except (IOError, OSError), e:
+        except (IOError, OSError) as e:
             if e.errno == errno.ENOENT:
                 return None
             raise
@@ -812,7 +820,7 @@ class Repo(BaseRepo):
         path = os.path.join(self._controldir, 'config')
         try:
             return ConfigFile.from_path(path)
-        except (IOError, OSError), e:
+        except (IOError, OSError) as e:
             if e.errno != errno.ENOENT:
                 raise
             ret = ConfigFile()
@@ -831,7 +839,7 @@ class Repo(BaseRepo):
                 return f.read()
             finally:
                 f.close()
-        except (IOError, OSError), e:
+        except (IOError, OSError) as e:
             if e.errno != errno.ENOENT:
                 raise
             return None
@@ -899,9 +907,11 @@ class MemoryRepo(BaseRepo):
     """
 
     def __init__(self):
+        from dulwich.config import ConfigFile
         BaseRepo.__init__(self, MemoryObjectStore(), DictRefsContainer({}))
         self._named_files = {}
         self.bare = True
+        self._config = ConfigFile()
 
     def _put_named_file(self, path, contents):
         """Write a file to the control dir with the given name and contents.
@@ -924,7 +934,7 @@ class MemoryRepo(BaseRepo):
         contents = self._named_files.get(path, None)
         if contents is None:
             return None
-        return StringIO(contents)
+        return BytesIO(contents)
 
     def open_index(self):
         """Fail to open index for this repo, since it is bare.
@@ -938,8 +948,7 @@ class MemoryRepo(BaseRepo):
 
         :return: `ConfigFile` object.
         """
-        from dulwich.config import ConfigFile
-        return ConfigFile()
+        return self._config
 
     def get_description(self):
         """Retrieve the repository description.

dulwich/server.py

@@ -322,7 +322,7 @@ def _split_proto_line(line, allowed):
                 return tuple(fields)
             elif command == 'deepen':
                 return command, int(fields[1])
-    except (TypeError, AssertionError), e:
+    except (TypeError, AssertionError) as e:
         raise GitProtocolError(e)
     raise GitProtocolError('Received invalid line from client: %s' % line)
 
@@ -411,10 +411,6 @@ class ProtocolGraphWalker(object):
         :param heads: a dict of refname->SHA1 to advertise
         :return: a list of SHA1s requested by the client
         """
-        if not heads:
-            # The repo is empty, so short-circuit the whole process.
-            self.proto.write_pkt_line(None)
-            return []
         values = set(heads.itervalues())
         if self.advertise_refs or not self.http_req:
             for i, (ref, sha) in enumerate(sorted(heads.iteritems())):
@@ -478,7 +474,7 @@ class ProtocolGraphWalker(object):
         if not self._cached:
             if not self._impl and self.http_req:
                 return None
-            return self._impl.next()
+            return next(self._impl)
         self._cache_index += 1
         if self._cache_index > len(self._cache):
             return None
@@ -714,7 +710,7 @@ class ReceivePackHandler(Handler):
                 recv = getattr(self.proto, "recv", None)
                 p = self.repo.object_store.add_thin_pack(self.proto.read, recv)
                 status.append(('unpack', 'ok'))
-            except all_exceptions, e:
+            except all_exceptions as e:
                 status.append(('unpack', str(e).replace('\n', '')))
                 # The pack may still have been moved in, but it may contain broken
                 # objects. We trust a later GC to clean it up.
@@ -740,7 +736,7 @@ class ReceivePackHandler(Handler):
                         self.repo.refs[ref] = sha
                     except all_exceptions:
                         ref_status = 'failed to write'
-            except KeyError, e:
+            except KeyError as e:
                 ref_status = 'bad ref'
             status.append((ref, ref_status))
 

dulwich/tests/compat/server_utils.py

@@ -130,7 +130,7 @@ class ServerTests(object):
         run_git_or_fail(['fetch', self.url(port)] + self.branch_args(),
                         cwd=self._old_repo.path)
         # flush the pack cache so any new packs are picked up
-        self._old_repo.object_store._pack_cache = None
+        self._old_repo.object_store._pack_cache_time = 0
         self.assertReposEqual(self._old_repo, self._new_repo)
 
     def test_fetch_from_dulwich_no_op(self):
@@ -144,7 +144,7 @@ class ServerTests(object):
         run_git_or_fail(['fetch', self.url(port)] + self.branch_args(),
                         cwd=self._old_repo.path)
         # flush the pack cache so any new packs are picked up
-        self._old_repo.object_store._pack_cache = None
+        self._old_repo.object_store._pack_cache_time = 0
         self.assertReposEqual(self._old_repo, self._new_repo)
 
     def test_clone_from_dulwich_empty(self):
@@ -234,85 +234,6 @@ class ServerTests(object):
         self.assertReposEqual(clone, self._source_repo)
 
 
-class ShutdownServerMixIn:
-    """Mixin that allows serve_forever to be shut down.
-
-    The methods in this mixin are backported from SocketServer.py in the Python
-    2.6.4 standard library. The mixin is unnecessary in 2.6 and later, when
-    BaseServer supports the shutdown method directly.
-    """
-
-    def __init__(self):
-        self.__is_shut_down = threading.Event()
-        self.__serving = False
-
-    def serve_forever(self, poll_interval=0.5):
-        """Handle one request at a time until shutdown.
-
-        Polls for shutdown every poll_interval seconds. Ignores
-        self.timeout. If you need to do periodic tasks, do them in
-        another thread.
-        """
-        self.__serving = True
-        self.__is_shut_down.clear()
-        while self.__serving:
-            # XXX: Consider using another file descriptor or
-            # connecting to the socket to wake this up instead of
-            # polling. Polling reduces our responsiveness to a
-            # shutdown request and wastes cpu at all other times.
-            r, w, e = select.select([self], [], [], poll_interval)
-            if r:
-                self._handle_request_noblock()
-        self.__is_shut_down.set()
-
-    serve = serve_forever  # override alias from TCPGitServer
-
-    def shutdown(self):
-        """Stops the serve_forever loop.
-
-        Blocks until the loop has finished. This must be called while
-        serve_forever() is running in another thread, or it will deadlock.
-        """
-        self.__serving = False
-        self.__is_shut_down.wait()
-
-    def handle_request(self):
-        """Handle one request, possibly blocking.
-
-        Respects self.timeout.
-        """
-        # Support people who used socket.settimeout() to escape
-        # handle_request before self.timeout was available.
-        timeout = self.socket.gettimeout()
-        if timeout is None:
-            timeout = self.timeout
-        elif self.timeout is not None:
-            timeout = min(timeout, self.timeout)
-        fd_sets = select.select([self], [], [], timeout)
-        if not fd_sets[0]:
-            self.handle_timeout()
-            return
-        self._handle_request_noblock()
-
-    def _handle_request_noblock(self):
-        """Handle one request, without blocking.
-
-        I assume that select.select has returned that the socket is
-        readable before this function was called, so there should be
-        no risk of blocking in get_request().
-        """
-        try:
-            request, client_address = self.get_request()
-        except socket.error:
-            return
-        if self.verify_request(request, client_address):
-            try:
-                self.process_request(request, client_address)
-            except:
-                self.handle_error(request, client_address)
-                self.close_request(request)
-
-
 # TODO(dborowitz): Come up with a better way of testing various permutations of
 # capabilities. The only reason it is the way it is now is that side-band-64k
 # was only recently introduced into git-receive-pack.
@@ -325,8 +246,9 @@ class NoSideBand64kReceivePackHandler(ReceivePackHandler):
                      if c != 'side-band-64k')
 
 
-def ignore_error((e_type, e_value, e_tb)):
+def ignore_error(error):
     """Check whether this error is safe to ignore."""
+    (e_type, e_value, e_tb) = error
     return (issubclass(e_type, socket.error) and
             e_value[0] in (errno.ECONNRESET, errno.EPIPE))
 

dulwich/tests/compat/test_client.py

@@ -19,7 +19,7 @@
 
 """Compatibilty tests between the Dulwich client and the cgit server."""
 
-from cStringIO import StringIO
+from io import BytesIO
 import BaseHTTPServer
 import SimpleHTTPServer
 import copy
@@ -53,9 +53,6 @@ from dulwich.tests.compat.utils import (
     import_repo_to_dir,
     run_git_or_fail,
     )
-from dulwich.tests.compat.server_utils import (
-    ShutdownServerMixIn,
-    )
 
 
 class DulwichClientTestBase(object):
@@ -114,7 +111,7 @@ class DulwichClientTestBase(object):
     def make_dummy_commit(self, dest):
         b = objects.Blob.from_string('hi')
         dest.object_store.add_object(b)
-        t = index.commit_tree(dest.object_store, [('hi', b.id, 0100644)])
+        t = index.commit_tree(dest.object_store, [('hi', b.id, 0o100644)])
         c = objects.Commit()
         c.author = c.committer = 'Foo Bar <foo@example.com>'
         c.author_time = c.commit_time = 0
@@ -146,7 +143,7 @@ class DulwichClientTestBase(object):
         c = self._client()
         try:
             c.send_pack(self._build_path('/dest'), lambda _: sendrefs, gen_pack)
-        except errors.UpdateRefsError, e:
+        except errors.UpdateRefsError as e:
             self.assertEqual('refs/heads/master failed to update', str(e))
             self.assertEqual({'refs/heads/branch': 'ok',
                               'refs/heads/master': 'non-fast-forward'},
@@ -160,7 +157,7 @@ class DulwichClientTestBase(object):
         c = self._client()
         try:
             c.send_pack(self._build_path('/dest'), lambda _: sendrefs, gen_pack)
-        except errors.UpdateRefsError, e:
+        except errors.UpdateRefsError as e:
             self.assertEqual('refs/heads/branch, refs/heads/master failed to '
                              'update', str(e))
             self.assertEqual({'refs/heads/branch': 'non-fast-forward',
@@ -169,7 +166,7 @@ class DulwichClientTestBase(object):
 
     def test_archive(self):
         c = self._client()
-        f = StringIO()
+        f = BytesIO()
         c.archive(self._build_path('/server_new.export'), 'HEAD', f.write)
         f.seek(0)
         tf = tarfile.open(fileobj=f)
@@ -439,18 +436,6 @@ class HTTPGitServer(BaseHTTPServer.HTTPServer):
         return 'http://%s:%s/' % (self.server_name, self.server_port)
 
 
-if not getattr(HTTPGitServer, 'shutdown', None):
-    _HTTPGitServer = HTTPGitServer
-
-    class TCPGitServer(ShutdownServerMixIn, HTTPGitServer):
-        """Subclass of HTTPGitServer that can be shut down."""
-
-        def __init__(self, *args, **kwargs):
-            # BaseServer is old-style so we have to call both __init__s
-            ShutdownServerMixIn.__init__(self)
-            _HTTPGitServer.__init__(self, *args, **kwargs)
-
-
 class DulwichHttpClientTest(CompatTestCase, DulwichClientTestBase):
 
     min_git_version = (1, 7, 0, 2)

dulwich/tests/compat/test_repository.py

@@ -20,7 +20,7 @@
 """Compatibility tests for dulwich repositories."""
 
 
-from cStringIO import StringIO
+from io import BytesIO
 import itertools
 import os
 
@@ -54,7 +54,7 @@ class ObjectStoreTestCase(CompatTestCase):
 
     def _parse_refs(self, output):
         refs = {}
-        for line in StringIO(output):
+        for line in BytesIO(output):
             fields = line.rstrip('\n').split(' ')
             self.assertEqual(3, len(fields))
             refname, type_name, sha = fields
@@ -64,7 +64,7 @@ class ObjectStoreTestCase(CompatTestCase):
         return refs
 
     def _parse_objects(self, output):
-        return set(s.rstrip('\n').split(' ')[0] for s in StringIO(output))
+        return set(s.rstrip('\n').split(' ')[0] for s in BytesIO(output))
 
     def test_bare(self):
         self.assertTrue(self._repo.bare)

dulwich/tests/compat/test_server.py

@@ -32,7 +32,6 @@ from dulwich.server import (
     )
 from dulwich.tests.compat.server_utils import (
     ServerTests,
-    ShutdownServerMixIn,
     NoSideBand64kReceivePackHandler,
     )
 from dulwich.tests.compat.utils import (
@@ -40,20 +39,6 @@ from dulwich.tests.compat.utils import (
     )
 
 
-if not getattr(TCPGitServer, 'shutdown', None):
-    _TCPGitServer = TCPGitServer
-
-    class TCPGitServer(ShutdownServerMixIn, TCPGitServer):
-        """Subclass of TCPGitServer that can be shut down."""
-
-        def __init__(self, *args, **kwargs):
-            # BaseServer is old-style so we have to call both __init__s
-            ShutdownServerMixIn.__init__(self)
-            _TCPGitServer.__init__(self, *args, **kwargs)
-
-        serve = ShutdownServerMixIn.serve_forever
-
-
 class GitServerTestCase(ServerTests, CompatTestCase):
     """Tests for client/server compatibility.
 

dulwich/tests/compat/test_utils.py

@@ -87,6 +87,6 @@ class GitVersionTests(TestCase):
             self.assertRequireSucceeds((1, 7, 0, 2))
             self.assertRequireFails((1, 7, 0, 3))
             self.assertRequireFails((1, 7, 1))
-        except SkipTest, e:
+        except SkipTest as e:
             # This test is designed to catch all SkipTest exceptions.
             self.fail('Test unexpectedly skipped: %s' % e)

dulwich/tests/compat/test_web.py

@@ -42,7 +42,6 @@ from dulwich.web import (
 
 from dulwich.tests.compat.server_utils import (
     ServerTests,
-    ShutdownServerMixIn,
     NoSideBand64kReceivePackHandler,
     )
 from dulwich.tests.compat.utils import (
@@ -50,20 +49,6 @@ from dulwich.tests.compat.utils import (
     )
 
 
-if getattr(simple_server.WSGIServer, 'shutdown', None):
-    WSGIServer = WSGIServerLogger
-else:
-    class WSGIServer(ShutdownServerMixIn, WSGIServerLogger):
-        """Subclass of WSGIServer that can be shut down."""
-
-        def __init__(self, *args, **kwargs):
-            # BaseServer is old-style so we have to call both __init__s
-            ShutdownServerMixIn.__init__(self)
-            simple_server.WSGIServer.__init__(self, *args, **kwargs)
-
-        serve = ShutdownServerMixIn.serve_forever
-
-
 class WebTests(ServerTests):
     """Base tests for web server tests.
 
@@ -77,7 +62,7 @@ class WebTests(ServerTests):
         backend = DictBackend({'/': repo})
         app = self._make_app(backend)
         dul_server = simple_server.make_server(
-          'localhost', 0, app, server_class=WSGIServer,
+          'localhost', 0, app, server_class=WSGIServerLogger,
           handler_class=WSGIRequestHandlerLogger)
         self.addCleanup(dul_server.shutdown)
         threading.Thread(target=dul_server.serve_forever).start()

dulwich/tests/compat/utils.py

@@ -187,7 +187,7 @@ def check_for_daemon(limit=10, delay=0.1, timeout=0.1, port=TCP_GIT_PORT):
     :returns: A boolean, true if a daemon is running on the specified port,
         false if not.
     """
-    for _ in xrange(limit):
+    for _ in range(limit):
         time.sleep(delay)
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         s.settimeout(delay)
@@ -195,7 +195,7 @@ def check_for_daemon(limit=10, delay=0.1, timeout=0.1, port=TCP_GIT_PORT):
             s.connect(('localhost', port))
             s.close()
             return True
-        except socket.error, e:
+        except socket.error as e:
             if getattr(e, 'errno', False) and e.errno != errno.ECONNREFUSED:
                 raise
             elif e.args[0] != errno.ECONNREFUSED:

dulwich/tests/test_client.py

@@ -16,7 +16,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-from cStringIO import StringIO
+from io import BytesIO
 
 from dulwich import (
     client,
@@ -71,8 +71,8 @@ class GitClientTests(TestCase):
 
     def setUp(self):
         super(GitClientTests, self).setUp()
-        self.rout = StringIO()
-        self.rin = StringIO()
+        self.rout = BytesIO()
+        self.rin = BytesIO()
         self.client = DummyClient(lambda x: True, self.rin.read,
                                   self.rout.write)
 
@@ -202,7 +202,7 @@ class GitClientTests(TestCase):
         def generate_pack_contents(have, want):
             return {}
 
-        f = StringIO()
+        f = BytesIO()
         empty_pack = write_pack_objects(f, {})
         self.client.send_pack('/', determine_wants, generate_pack_contents)
         self.assertEqual(
@@ -240,7 +240,7 @@ class GitClientTests(TestCase):
         def generate_pack_contents(have, want):
             return [(commit, None), (tree, ''), ]
 
-        f = StringIO()
+        f = BytesIO()
         pack = write_pack_objects(f, generate_pack_contents(None, None))
         self.client.send_pack('/', determine_wants, generate_pack_contents)
         self.assertEqual(
@@ -563,7 +563,7 @@ class LocalGitClientTests(TestCase):
     def test_fetch_empty(self):
         c = LocalGitClient()
         s = open_repo('a.git')
-        out = StringIO()
+        out = BytesIO()
         walker = {}
         c.fetch_pack(s.path, lambda heads: [], graph_walker=walker,
             pack_data=out.write)
@@ -573,7 +573,7 @@ class LocalGitClientTests(TestCase):
     def test_fetch_pack_none(self):
         c = LocalGitClient()
         s = open_repo('a.git')
-        out = StringIO()
+        out = BytesIO()
         walker = MemoryRepo().get_graph_walker()
         c.fetch_pack(s.path,
             lambda heads: ["a90fa2d900a17e99b433217e988c4eb4a2e9a097"],

dulwich/tests/test_config.py

@@ -18,7 +18,7 @@
 
 """Tests for reading and writing configuration files."""
 
-from cStringIO import StringIO
+from io import BytesIO
 from dulwich.config import (
     ConfigDict,
     ConfigFile,
@@ -37,7 +37,7 @@ from dulwich.tests import TestCase
 class ConfigFileTests(TestCase):
 
     def from_file(self, text):
-        return ConfigFile.from_file(StringIO(text))
+        return ConfigFile.from_file(BytesIO(text))
 
     def test_empty(self):
         ConfigFile()
@@ -129,21 +129,21 @@ class ConfigFileTests(TestCase):
 
     def test_write_to_file_empty(self):
         c = ConfigFile()
-        f = StringIO()
+        f = BytesIO()
         c.write_to_file(f)
         self.assertEqual("", f.getvalue())
 
     def test_write_to_file_section(self):
         c = ConfigFile()
         c.set(("core", ), "foo", "bar")
-        f = StringIO()
+        f = BytesIO()
         c.write_to_file(f)
         self.assertEqual("[core]\n\tfoo = bar\n", f.getvalue())
 
     def test_write_to_file_subsection(self):
         c = ConfigFile()
         c.set(("branch", "blie"), "foo", "bar")
-        f = StringIO()
+        f = BytesIO()
         c.write_to_file(f)
         self.assertEqual("[branch \"blie\"]\n\tfoo = bar\n", f.getvalue())
 

dulwich/tests/test_diff_tree.py

@@ -103,42 +103,42 @@ class TreeChangesTest(DiffTestCase):
         blob_a2 = make_object(Blob, data='a2')
         blob_b1 = make_object(Blob, data='b1')
         blob_c2 = make_object(Blob, data='c2')
-        tree1 = self.commit_tree([('a', blob_a1, 0100644),
-                                  ('b', blob_b1, 0100755)])
-        tree2 = self.commit_tree([('a', blob_a2, 0100644),
-                                  ('c', blob_c2, 0100755)])
+        tree1 = self.commit_tree([('a', blob_a1, 0o100644),
+                                  ('b', blob_b1, 0o100755)])
+        tree2 = self.commit_tree([('a', blob_a2, 0o100644),
+                                  ('c', blob_c2, 0o100755)])
 
         self.assertEqual([], merge_entries('', self.empty_tree,
                                            self.empty_tree))
         self.assertEqual([
-          ((None, None, None), ('a', 0100644, blob_a1.id)),
-          ((None, None, None), ('b', 0100755, blob_b1.id)),
+          ((None, None, None), ('a', 0o100644, blob_a1.id)),
+          ((None, None, None), ('b', 0o100755, blob_b1.id)),
           ], merge_entries('', self.empty_tree, tree1))
         self.assertEqual([
-          ((None, None, None), ('x/a', 0100644, blob_a1.id)),
-          ((None, None, None), ('x/b', 0100755, blob_b1.id)),
+          ((None, None, None), ('x/a', 0o100644, blob_a1.id)),
+          ((None, None, None), ('x/b', 0o100755, blob_b1.id)),
           ], merge_entries('x', self.empty_tree, tree1))
 
         self.assertEqual([
-          (('a', 0100644, blob_a2.id), (None, None, None)),
-          (('c', 0100755, blob_c2.id), (None, None, None)),
+          (('a', 0o100644, blob_a2.id), (None, None, None)),
+          (('c', 0o100755, blob_c2.id), (None, None, None)),
           ], merge_entries('', tree2, self.empty_tree))
 
         self.assertEqual([
-          (('a', 0100644, blob_a1.id), ('a', 0100644, blob_a2.id)),
-          (('b', 0100755, blob_b1.id), (None, None, None)),
-          ((None, None, None), ('c', 0100755, blob_c2.id)),
+          (('a', 0o100644, blob_a1.id), ('a', 0o100644, blob_a2.id)),
+          (('b', 0o100755, blob_b1.id), (None, None, None)),
+          ((None, None, None), ('c', 0o100755, blob_c2.id)),
           ], merge_entries('', tree1, tree2))
 
         self.assertEqual([
-          (('a', 0100644, blob_a2.id), ('a', 0100644, blob_a1.id)),
-          ((None, None, None), ('b', 0100755, blob_b1.id)),
-          (('c', 0100755, blob_c2.id), (None, None, None)),
+          (('a', 0o100644, blob_a2.id), ('a', 0o100644, blob_a1.id)),
+          ((None, None, None), ('b', 0o100755, blob_b1.id)),
+          (('c', 0o100755, blob_c2.id), (None, None, None)),
           ], merge_entries('', tree2, tree1))
 
-        self.assertMergeFails(merge_entries, 0xdeadbeef, 0100644, '1' * 40)
+        self.assertMergeFails(merge_entries, 0xdeadbeef, 0o100644, '1' * 40)
         self.assertMergeFails(merge_entries, 'a', 'deadbeef', '1' * 40)
-        self.assertMergeFails(merge_entries, 'a', 0100644, 0xdeadbeef)
+        self.assertMergeFails(merge_entries, 'a', 0o100644, 0xdeadbeef)
 
     test_merge_entries = functest_builder(_do_test_merge_entries,
                                           _merge_entries_py)
@@ -147,10 +147,10 @@ class TreeChangesTest(DiffTestCase):
 
     def _do_test_is_tree(self, is_tree):
         self.assertFalse(is_tree(TreeEntry(None, None, None)))
-        self.assertFalse(is_tree(TreeEntry('a', 0100644, 'a' * 40)))
-        self.assertFalse(is_tree(TreeEntry('a', 0100755, 'a' * 40)))
-        self.assertFalse(is_tree(TreeEntry('a', 0120000, 'a' * 40)))
-        self.assertTrue(is_tree(TreeEntry('a', 0040000, 'a' * 40)))
+        self.assertFalse(is_tree(TreeEntry('a', 0o100644, 'a' * 40)))
+        self.assertFalse(is_tree(TreeEntry('a', 0o100755, 'a' * 40)))
+        self.assertFalse(is_tree(TreeEntry('a', 0o120000, 'a' * 40)))
+        self.assertTrue(is_tree(TreeEntry('a', 0o040000, 'a' * 40)))
         self.assertRaises(TypeError, is_tree, TreeEntry('a', 'x', 'a' * 40))
         self.assertRaises(AttributeError, is_tree, 1234)
 
@@ -180,15 +180,15 @@ class TreeChangesTest(DiffTestCase):
     def test_tree_changes_add_delete(self):
         blob_a = make_object(Blob, data='a')
         blob_b = make_object(Blob, data='b')
-        tree = self.commit_tree([('a', blob_a, 0100644),
-                                 ('x/b', blob_b, 0100755)])
+        tree = self.commit_tree([('a', blob_a, 0o100644),
+                                 ('x/b', blob_b, 0o100755)])
         self.assertChangesEqual(
-          [TreeChange.add(('a', 0100644, blob_a.id)),
-           TreeChange.add(('x/b', 0100755, blob_b.id))],
+          [TreeChange.add(('a', 0o100644, blob_a.id)),
+           TreeChange.add(('x/b', 0o100755, blob_b.id))],
           self.empty_tree, tree)
         self.assertChangesEqual(
-          [TreeChange.delete(('a', 0100644, blob_a.id)),
-           TreeChange.delete(('x/b', 0100755, blob_b.id))],
+          [TreeChange.delete(('a', 0o100644, blob_a.id)),
+           TreeChange.delete(('x/b', 0o100755, blob_b.id))],
           tree, self.empty_tree)
 
     def test_tree_changes_modify_contents(self):
@@ -202,20 +202,20 @@ class TreeChangesTest(DiffTestCase):
 
     def test_tree_changes_modify_mode(self):
         blob_a = make_object(Blob, data='a')
-        tree1 = self.commit_tree([('a', blob_a, 0100644)])
-        tree2 = self.commit_tree([('a', blob_a, 0100755)])
+        tree1 = self.commit_tree([('a', blob_a, 0o100644)])
+        tree2 = self.commit_tree([('a', blob_a, 0o100755)])
         self.assertChangesEqual(
-          [TreeChange(CHANGE_MODIFY, ('a', 0100644, blob_a.id),
-                      ('a', 0100755, blob_a.id))], tree1, tree2)
+          [TreeChange(CHANGE_MODIFY, ('a', 0o100644, blob_a.id),
+                      ('a', 0o100755, blob_a.id))], tree1, tree2)
 
     def test_tree_changes_change_type(self):
         blob_a1 = make_object(Blob, data='a')
         blob_a2 = make_object(Blob, data='/foo/bar')
-        tree1 = self.commit_tree([('a', blob_a1, 0100644)])
-        tree2 = self.commit_tree([('a', blob_a2, 0120000)])
+        tree1 = self.commit_tree([('a', blob_a1, 0o100644)])
+        tree2 = self.commit_tree([('a', blob_a2, 0o120000)])
         self.assertChangesEqual(
-          [TreeChange.delete(('a', 0100644, blob_a1.id)),
-           TreeChange.add(('a', 0120000, blob_a2.id))],
+          [TreeChange.delete(('a', 0o100644, blob_a1.id)),
+           TreeChange.add(('a', 0o120000, blob_a2.id))],
           tree1, tree2)
 
     def test_tree_changes_to_tree(self):
@@ -392,7 +392,7 @@ class TreeChangesTest(DiffTestCase):
         self.assertChangesForMergeEqual([], [has, doesnt_have], doesnt_have)
 
     def test_tree_changes_for_merge_octopus_no_conflict(self):
-        r = range(5)
+        r = list(range(5))
         blobs = [make_object(Blob, data=str(i)) for i in r]
         parents = [self.commit_tree([('a', blobs[i])]) for i in r]
         for i in r:
@@ -403,7 +403,7 @@ class TreeChangesTest(DiffTestCase):
         # Because the octopus merge strategy is limited, I doubt it's possible
         # to create this with the git command line. But the output is well-
         # defined, so test it anyway.
-        r = range(5)
+        r = list(range(5))
         parent_blobs = [make_object(Blob, data=str(i)) for i in r]
         merge_blob = make_object(Blob, data='merge')
         parents = [self.commit_tree([('a', parent_blobs[i])]) for i in r]
@@ -591,20 +591,20 @@ class RenameDetectionTest(DiffTestCase):
 
     def test_exact_rename_split_different_type(self):
         blob = make_object(Blob, data='/foo')
-        tree1 = self.commit_tree([('a', blob, 0100644)])
-        tree2 = self.commit_tree([('a', blob, 0120000)])
+        tree1 = self.commit_tree([('a', blob, 0o100644)])
+        tree2 = self.commit_tree([('a', blob, 0o120000)])
         self.assertEqual(
-          [TreeChange.add(('a', 0120000, blob.id)),
-           TreeChange.delete(('a', 0100644, blob.id))],
+          [TreeChange.add(('a', 0o120000, blob.id)),
+           TreeChange.delete(('a', 0o100644, blob.id))],
           self.detect_renames(tree1, tree2))
 
     def test_exact_rename_and_different_type(self):
         blob1 = make_object(Blob, data='1')
         blob2 = make_object(Blob, data='2')
         tree1 = self.commit_tree([('a', blob1)])
-        tree2 = self.commit_tree([('a', blob2, 0120000), ('b', blob1)])
+        tree2 = self.commit_tree([('a', blob2, 0o120000), ('b', blob1)])
         self.assertEqual(
-          [TreeChange.add(('a', 0120000, blob2.id)),
+          [TreeChange.add(('a', 0o120000, blob2.id)),
            TreeChange(CHANGE_RENAME, ('a', F, blob1.id), ('b', F, blob1.id))],
           self.detect_renames(tree1, tree2))
 
@@ -649,10 +649,10 @@ class RenameDetectionTest(DiffTestCase):
     def test_exact_copy_change_mode(self):
         blob = make_object(Blob, data='a\nb\nc\nd\n')
         tree1 = self.commit_tree([('a', blob)])
-        tree2 = self.commit_tree([('a', blob, 0100755), ('b', blob)])
+        tree2 = self.commit_tree([('a', blob, 0o100755), ('b', blob)])
         self.assertEqual(
           [TreeChange(CHANGE_MODIFY, ('a', F, blob.id),
-                      ('a', 0100755, blob.id)),
+                      ('a', 0o100755, blob.id)),
            TreeChange(CHANGE_COPY, ('a', F, blob.id), ('b', F, blob.id))],
           self.detect_renames(tree1, tree2))
 
@@ -760,13 +760,13 @@ class RenameDetectionTest(DiffTestCase):
         blob2 = make_object(Blob, data='blob2')
         link1 = '1' * 40
         link2 = '2' * 40
-        tree1 = self.commit_tree([('a', blob1), ('b', link1, 0160000)])
-        tree2 = self.commit_tree([('c', blob2), ('d', link2, 0160000)])
+        tree1 = self.commit_tree([('a', blob1), ('b', link1, 0o160000)])
+        tree2 = self.commit_tree([('c', blob2), ('d', link2, 0o160000)])
         self.assertEqual(
-          [TreeChange.delete(('a', 0100644, blob1.id)),
-           TreeChange.delete(('b', 0160000, link1)),
-           TreeChange.add(('c', 0100644, blob2.id)),
-           TreeChange.add(('d', 0160000, link2))],
+          [TreeChange.delete(('a', 0o100644, blob1.id)),
+           TreeChange.delete(('b', 0o160000, link1)),
+           TreeChange.add(('c', 0o100644, blob2.id)),
+           TreeChange.add(('d', 0o160000, link2))],
           self.detect_renames(tree1, tree2))
 
     def test_exact_rename_swap(self):

dulwich/tests/test_fastexport.py

@@ -17,9 +17,10 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-from cStringIO import StringIO
+from io import BytesIO
 import stat
 
+
 from dulwich.object_store import (
     MemoryObjectStore,
     )
@@ -35,6 +36,9 @@ from dulwich.tests import (
     SkipTest,
     TestCase,
     )
+from dulwich.tests.utils import (
+    build_commit_graph,
+    )
 
 
 class GitFastExporterTests(TestCase):
@@ -43,7 +47,7 @@ class GitFastExporterTests(TestCase):
     def setUp(self):
         super(GitFastExporterTests, self).setUp()
         self.store = MemoryObjectStore()
-        self.stream = StringIO()
+        self.stream = BytesIO()
         try:
             from dulwich.fastexport import GitFastExporter
         except ImportError:
@@ -61,7 +65,7 @@ class GitFastExporterTests(TestCase):
         b = Blob()
         b.data = "FOO"
         t = Tree()
-        t.add("foo", stat.S_IFREG | 0644, b.id)
+        t.add("foo", stat.S_IFREG | 0o644, b.id)
         c = Commit()
         c.committer = c.author = "Jelmer <jelmer@host>"
         c.author_time = c.commit_time = 1271345553
@@ -96,6 +100,13 @@ class GitImportProcessorTests(TestCase):
             raise SkipTest("python-fastimport not available")
         self.processor = GitImportProcessor(self.repo)
 
+    def test_reset_handler(self):
+        from fastimport import commands
+        [c1] = build_commit_graph(self.repo.object_store, [[1]])
+        cmd = commands.ResetCommand("refs/heads/foo", c1.id)
+        self.processor.reset_handler(cmd)
+        self.assertEquals(c1.id, self.repo.get_refs()["refs/heads/foo"])
+
     def test_commit_handler(self):
         from fastimport import commands
         cmd = commands.CommitCommand("refs/heads/foo", "mrkr",
@@ -115,7 +126,7 @@ class GitImportProcessorTests(TestCase):
         self.assertEqual(commit, self.repo["refs/heads/foo"])
 
     def test_import_stream(self):
-        markers = self.processor.import_stream(StringIO("""blob
+        markers = self.processor.import_stream(BytesIO("""blob
 mark :1
 data 11
 text for a
@@ -139,11 +150,11 @@ M 100644 :1 a
         cmd = commands.CommitCommand("refs/heads/foo", "mrkr",
             ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
             ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
-            "FOO", None, [], [commands.FileModifyCommand("path", 0100644, ":23", None)])
+            "FOO", None, [], [commands.FileModifyCommand("path", 0o100644, ":23", None)])
         self.processor.commit_handler(cmd)
         commit = self.repo[self.processor.last_commit]
         self.assertEqual([
-            ('path', 0100644, '6320cd248dd8aeaab759d5871f8781b5c0505172')],
+            ('path', 0o100644, '6320cd248dd8aeaab759d5871f8781b5c0505172')],
             self.repo[commit.tree].items())
 
     def simple_commit(self):
@@ -153,7 +164,7 @@ M 100644 :1 a
         cmd = commands.CommitCommand("refs/heads/foo", "mrkr",
             ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
             ("Jelmer", "jelmer@samba.org", 432432432.0, 3600),
-            "FOO", None, [], [commands.FileModifyCommand("path", 0100644, ":23", None)])
+            "FOO", None, [], [commands.FileModifyCommand("path", 0o100644, ":23", None)])
         self.processor.commit_handler(cmd)
         commit = self.repo[self.processor.last_commit]
         return commit
@@ -177,8 +188,8 @@ M 100644 :1 a
         self.simple_commit()
         commit = self.make_file_commit([commands.FileCopyCommand("path", "new_path")])
         self.assertEqual([
-            ('new_path', 0100644, '6320cd248dd8aeaab759d5871f8781b5c0505172'),
-            ('path', 0100644, '6320cd248dd8aeaab759d5871f8781b5c0505172'),
+            ('new_path', 0o100644, '6320cd248dd8aeaab759d5871f8781b5c0505172'),
+            ('path', 0o100644, '6320cd248dd8aeaab759d5871f8781b5c0505172'),
             ], self.repo[commit.tree].items())
 
     def test_file_move(self):
@@ -186,7 +197,7 @@ M 100644 :1 a
         self.simple_commit()
         commit = self.make_file_commit([commands.FileRenameCommand("path", "new_path")])
         self.assertEqual([
-            ('new_path', 0100644, '6320cd248dd8aeaab759d5871f8781b5c0505172'),
+            ('new_path', 0o100644, '6320cd248dd8aeaab759d5871f8781b5c0505172'),
             ], self.repo[commit.tree].items())
 
     def test_file_delete(self):

dulwich/tests/test_file.py

@@ -155,7 +155,7 @@ class GitFileTests(TestCase):
         try:
             f2 = GitFile(foo, 'wb')
             self.fail()
-        except OSError, e:
+        except OSError as e:
             self.assertEqual(errno.EEXIST, e.errno)
         f1.write(' contents')
         f1.close()

dulwich/tests/test_hooks.py

@@ -113,7 +113,7 @@ exit 0
 
         (fd, path) = tempfile.mkstemp()
         post_commit_msg = """#!/bin/sh
-unlink %(file)s
+rm %(file)s
 """ % {'file': path}
 
         post_commit_msg_fail = """#!/bin/sh

dulwich/tests/test_index.py

@@ -19,9 +19,7 @@
 """Tests for the index."""
 
 
-from cStringIO import (
-    StringIO,
-    )
+from io import BytesIO
 import os
 import shutil
 import stat
@@ -145,39 +143,39 @@ class CommitTreeTests(TestCase):
 class CleanupModeTests(TestCase):
 
     def test_file(self):
-        self.assertEqual(0100644, cleanup_mode(0100000))
+        self.assertEqual(0o100644, cleanup_mode(0o100000))
 
     def test_executable(self):
-        self.assertEqual(0100755, cleanup_mode(0100711))
+        self.assertEqual(0o100755, cleanup_mode(0o100711))
 
     def test_symlink(self):
-        self.assertEqual(0120000, cleanup_mode(0120711))
+        self.assertEqual(0o120000, cleanup_mode(0o120711))
 
     def test_dir(self):
-        self.assertEqual(0040000, cleanup_mode(040531))
+        self.assertEqual(0o040000, cleanup_mode(0o40531))
 
     def test_submodule(self):
-        self.assertEqual(0160000, cleanup_mode(0160744))
+        self.assertEqual(0o160000, cleanup_mode(0o160744))
 
 
 class WriteCacheTimeTests(TestCase):
 
     def test_write_string(self):
-        f = StringIO()
+        f = BytesIO()
         self.assertRaises(TypeError, write_cache_time, f, "foo")
 
     def test_write_int(self):
-        f = StringIO()
+        f = BytesIO()
         write_cache_time(f, 434343)
         self.assertEqual(struct.pack(">LL", 434343, 0), f.getvalue())
 
     def test_write_tuple(self):
-        f = StringIO()
+        f = BytesIO()
         write_cache_time(f, (434343, 21))
         self.assertEqual(struct.pack(">LL", 434343, 21), f.getvalue())
 
     def test_write_float(self):
-        f = StringIO()
+        f = BytesIO()
         write_cache_time(f, 434343.000000021)
         self.assertEqual(struct.pack(">LL", 434343, 21), f.getvalue())
 
@@ -185,14 +183,14 @@ class WriteCacheTimeTests(TestCase):
 class IndexEntryFromStatTests(TestCase):
 
     def test_simple(self):
-        st = os.stat_result((16877, 131078, 64769L,
+        st = os.stat_result((16877, 131078, 64769,
                 154, 1000, 1000, 12288,
                 1323629595, 1324180496, 1324180496))
         entry = index_entry_from_stat(st, "22" * 20, 0)
         self.assertEqual(entry, (
             1324180496,
             1324180496,
-            64769L,
+            64769,
             131078,
             16384,
             1000,
@@ -202,15 +200,15 @@ class IndexEntryFromStatTests(TestCase):
             0))
 
     def test_override_mode(self):
-        st = os.stat_result((stat.S_IFREG + 0644, 131078, 64769L,
+        st = os.stat_result((stat.S_IFREG + 0o644, 131078, 64769,
                 154, 1000, 1000, 12288,
                 1323629595, 1324180496, 1324180496))
         entry = index_entry_from_stat(st, "22" * 20, 0,
-                mode=stat.S_IFREG + 0755)
+                mode=stat.S_IFREG + 0o755)
         self.assertEqual(entry, (
             1324180496,
             1324180496,
-            64769L,
+            64769,
             131078,
             33261,
             1000,
@@ -270,9 +268,9 @@ class BuildIndexTests(TestCase):
         filee = Blob.from_string('d')
 
         tree = Tree()
-        tree['a'] = (stat.S_IFREG | 0644, filea.id)
-        tree['b'] = (stat.S_IFREG | 0644, fileb.id)
-        tree['c/d'] = (stat.S_IFREG | 0644, filed.id)
+        tree['a'] = (stat.S_IFREG | 0o644, filea.id)
+        tree['b'] = (stat.S_IFREG | 0o644, fileb.id)
+        tree['c/d'] = (stat.S_IFREG | 0o644, filed.id)
         tree['c/e'] = (stat.S_IFLNK, filee.id)  # symlink
 
         repo.object_store.add_objects([(o, None)
@@ -289,21 +287,21 @@ class BuildIndexTests(TestCase):
         apath = os.path.join(repo.path, 'a')
         self.assertTrue(os.path.exists(apath))
         self.assertReasonableIndexEntry(index['a'],
-            stat.S_IFREG | 0644, 6, filea.id)
+            stat.S_IFREG | 0o644, 6, filea.id)
         self.assertFileContents(apath, 'file a')
 
         # fileb
         bpath = os.path.join(repo.path, 'b')
         self.assertTrue(os.path.exists(bpath))
         self.assertReasonableIndexEntry(index['b'],
-            stat.S_IFREG | 0644, 6, fileb.id)
+            stat.S_IFREG | 0o644, 6, fileb.id)
         self.assertFileContents(bpath, 'file b')
 
         # filed
         dpath = os.path.join(repo.path, 'c', 'd')
         self.assertTrue(os.path.exists(dpath))
         self.assertReasonableIndexEntry(index['c/d'], 
-            stat.S_IFREG | 0644, 6, filed.id)
+            stat.S_IFREG | 0o644, 6, filed.id)
         self.assertFileContents(dpath, 'file d')
 
         # symlink to d

dulwich/tests/test_missing_obj_finder.py

@@ -23,7 +23,7 @@ from dulwich.objects import (
     Blob,
     )
 from dulwich.tests import TestCase
-from utils import (
+from dulwich.tests.utils import (
     make_object,
     build_commit_graph,
     )

dulwich/tests/test_object_store.py

@@ -19,7 +19,7 @@
 """Tests for the object store interface."""
 
 
-from cStringIO import StringIO
+from io import BytesIO
 import os
 import shutil
 import tempfile
@@ -109,15 +109,15 @@ class ObjectStoreTests(object):
         for blob in [blob_a1, blob_a2, blob_b]:
             self.store.add_object(blob)
 
-        blobs_1 = [('a', blob_a1.id, 0100644), ('b', blob_b.id, 0100644)]
+        blobs_1 = [('a', blob_a1.id, 0o100644), ('b', blob_b.id, 0o100644)]
         tree1_id = commit_tree(self.store, blobs_1)
-        blobs_2 = [('a', blob_a2.id, 0100644), ('b', blob_b.id, 0100644)]
+        blobs_2 = [('a', blob_a2.id, 0o100644), ('b', blob_b.id, 0o100644)]
         tree2_id = commit_tree(self.store, blobs_2)
-        change_a = (('a', 'a'), (0100644, 0100644), (blob_a1.id, blob_a2.id))
+        change_a = (('a', 'a'), (0o100644, 0o100644), (blob_a1.id, blob_a2.id))
         self.assertEqual([change_a],
                           list(self.store.tree_changes(tree1_id, tree2_id)))
         self.assertEqual(
-          [change_a, (('b', 'b'), (0100644, 0100644), (blob_b.id, blob_b.id))],
+          [change_a, (('b', 'b'), (0o100644, 0o100644), (blob_b.id, blob_b.id))],
           list(self.store.tree_changes(tree1_id, tree2_id,
                                        want_unchanged=True)))
 
@@ -129,11 +129,11 @@ class ObjectStoreTests(object):
             self.store.add_object(blob)
 
         blobs = [
-          ('a', blob_a.id, 0100644),
-          ('ad/b', blob_b.id, 0100644),
-          ('ad/bd/c', blob_c.id, 0100755),
-          ('ad/c', blob_c.id, 0100644),
-          ('c', blob_c.id, 0100644),
+          ('a', blob_a.id, 0o100644),
+          ('ad/b', blob_b.id, 0o100644),
+          ('ad/bd/c', blob_c.id, 0o100755),
+          ('ad/c', blob_c.id, 0o100644),
+          ('c', blob_c.id, 0o100644),
           ]
         tree_id = commit_tree(self.store, blobs)
         self.assertEqual([TreeEntry(p, m, h) for (p, h, m) in blobs],
@@ -147,9 +147,9 @@ class ObjectStoreTests(object):
             self.store.add_object(blob)
 
         blobs = [
-          ('a', blob_a.id, 0100644),
-          ('ad/b', blob_b.id, 0100644),
-          ('ad/bd/c', blob_c.id, 0100755),
+          ('a', blob_a.id, 0o100644),
+          ('ad/b', blob_b.id, 0o100644),
+          ('ad/bd/c', blob_c.id, 0o100755),
           ]
         tree_id = commit_tree(self.store, blobs)
         tree = self.store[tree_id]
@@ -157,12 +157,12 @@ class ObjectStoreTests(object):
         tree_bd = self.store[tree_ad['bd'][1]]
 
         expected = [
-          TreeEntry('', 0040000, tree_id),
-          TreeEntry('a', 0100644, blob_a.id),
-          TreeEntry('ad', 0040000, tree_ad.id),
-          TreeEntry('ad/b', 0100644, blob_b.id),
-          TreeEntry('ad/bd', 0040000, tree_bd.id),
-          TreeEntry('ad/bd/c', 0100755, blob_c.id),
+          TreeEntry('', 0o040000, tree_id),
+          TreeEntry('a', 0o100644, blob_a.id),
+          TreeEntry('ad', 0o040000, tree_ad.id),
+          TreeEntry('ad/b', 0o100644, blob_b.id),
+          TreeEntry('ad/bd', 0o040000, tree_bd.id),
+          TreeEntry('ad/bd/c', 0o100755, blob_c.id),
           ]
         actual = self.store.iter_tree_contents(tree_id, include_trees=True)
         self.assertEqual(expected, list(actual))
@@ -217,7 +217,7 @@ class MemoryObjectStoreTests(ObjectStoreTests, TestCase):
         blob = make_object(Blob, data='yummy data')
         o.add_object(blob)
 
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (REF_DELTA, (blob.id, 'more yummy data')),
           ], store=o)
@@ -315,7 +315,7 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
         blob = make_object(Blob, data='yummy data')
         o.add_object(blob)
 
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (REF_DELTA, (blob.id, 'more yummy data')),
           ], store=o)
@@ -329,12 +329,10 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
             self.assertEqual((Blob.type_num, 'more yummy data'),
                              o.get_raw(packed_blob_sha))
         finally:
-            # FIXME: DiskObjectStore should have close() which do the following:
-            for p in o._pack_cache or []:
-                p.close()
-
+            o.close()
             pack.close()
 
+
 class TreeLookupPathTests(TestCase):
 
     def setUp(self):
@@ -347,11 +345,11 @@ class TreeLookupPathTests(TestCase):
             self.store.add_object(blob)
 
         blobs = [
-          ('a', blob_a.id, 0100644),
-          ('ad/b', blob_b.id, 0100644),
-          ('ad/bd/c', blob_c.id, 0100755),
-          ('ad/c', blob_c.id, 0100644),
-          ('c', blob_c.id, 0100644),
+          ('a', blob_a.id, 0o100644),
+          ('ad/b', blob_b.id, 0o100644),
+          ('ad/bd/c', blob_c.id, 0o100755),
+          ('ad/c', blob_c.id, 0o100644),
+          ('c', blob_c.id, 0o100644),
           ]
         self.tree_id = commit_tree(self.store, blobs)
 
@@ -386,32 +384,32 @@ class ObjectStoreGraphWalkerTests(TestCase):
 
     def test_empty(self):
         gw = self.get_walker([], {})
-        self.assertIs(None, gw.next())
+        self.assertIs(None, next(gw))
         gw.ack("aa" * 20)
-        self.assertIs(None, gw.next())
+        self.assertIs(None, next(gw))
 
     def test_descends(self):
         gw = self.get_walker(["a"], {"a": ["b"], "b": []})
-        self.assertEqual("a", gw.next())
-        self.assertEqual("b", gw.next())
+        self.assertEqual("a", next(gw))
+        self.assertEqual("b", next(gw))
 
     def test_present(self):
         gw = self.get_walker(["a"], {"a": ["b"], "b": []})
         gw.ack("a")
-        self.assertIs(None, gw.next())
+        self.assertIs(None, next(gw))
 
     def test_parent_present(self):
         gw = self.get_walker(["a"], {"a": ["b"], "b": []})
-        self.assertEqual("a", gw.next())
+        self.assertEqual("a", next(gw))
         gw.ack("a")
-        self.assertIs(None, gw.next())
+        self.assertIs(None, next(gw))
 
     def test_child_ack_later(self):
         gw = self.get_walker(["a"], {"a": ["b"], "b": ["c"], "c": []})
-        self.assertEqual("a", gw.next())
-        self.assertEqual("b", gw.next())
+        self.assertEqual("a", next(gw))
+        self.assertEqual("b", next(gw))
         gw.ack("a")
-        self.assertIs(None, gw.next())
+        self.assertIs(None, next(gw))
 
     def test_only_once(self):
         # a  b
@@ -426,9 +424,9 @@ class ObjectStoreGraphWalkerTests(TestCase):
                 "d": ["e"],
                 "e": [],
                 })
-        self.assertEqual("a", gw.next())
-        self.assertEqual("c", gw.next())
+        self.assertEqual("a", next(gw))
+        self.assertEqual("c", next(gw))
         gw.ack("a")
-        self.assertEqual("b", gw.next())
-        self.assertEqual("d", gw.next())
-        self.assertIs(None, gw.next())
+        self.assertEqual("b", next(gw))
+        self.assertEqual("d", next(gw))
+        self.assertIs(None, next(gw))

dulwich/tests/test_objects.py

@@ -22,7 +22,7 @@
 # TODO: Round-trip parse-serialize-parse and serialize-parse-serialize tests.
 
 
-from cStringIO import StringIO
+from io import BytesIO
 import datetime
 from itertools import (
     permutations,
@@ -56,7 +56,7 @@ from dulwich.objects import (
 from dulwich.tests import (
     TestCase,
     )
-from utils import (
+from dulwich.tests.utils import (
     make_commit,
     make_object,
     functest_builder,
@@ -124,7 +124,7 @@ class BlobReadTests(TestCase):
     def test_legacy_from_file(self):
         b1 = Blob.from_string("foo")
         b_raw = b1.as_legacy_object()
-        b2 = b1.from_file(StringIO(b_raw))
+        b2 = b1.from_file(BytesIO(b_raw))
         self.assertEqual(b1, b2)
 
     def test_chunks(self):
@@ -251,7 +251,7 @@ class ShaFileTests(TestCase):
         # zlib on some systems uses smaller buffers,
         # resulting in a different header.
         # See https://github.com/libgit2/libgit2/pull/464
-        sf = ShaFile.from_file(StringIO(small_buffer_zlib_object))
+        sf = ShaFile.from_file(BytesIO(small_buffer_zlib_object))
         self.assertEqual(sf.type_name, "tag")
         self.assertEqual(sf.tagger, " <@localhost>")
 
@@ -508,7 +508,7 @@ class CommitParseTests(ShaFileCheckTests):
 
     def test_check_duplicates(self):
         # duplicate each of the header fields
-        for i in xrange(5):
+        for i in range(5):
             lines = self.make_commit_lines(parents=[a_sha], encoding='UTF-8')
             lines.insert(i, lines[i])
             text = '\n'.join(lines)
@@ -533,13 +533,13 @@ class CommitParseTests(ShaFileCheckTests):
 
 
 _TREE_ITEMS = {
-  'a.c': (0100755, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
+  'a.c': (0o100755, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
   'a': (stat.S_IFDIR, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
   'a/c': (stat.S_IFDIR, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
   }
 
 _SORTED_TREE_ITEMS = [
-  TreeEntry('a.c', 0100755, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
+  TreeEntry('a.c', 0o100755, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
   TreeEntry('a', stat.S_IFDIR, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
   TreeEntry('a/c', stat.S_IFDIR, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
   ]
@@ -550,8 +550,8 @@ class TreeTests(ShaFileCheckTests):
     def test_add(self):
         myhexsha = "d80c186a03f423a81b39df39dc87fd269736ca86"
         x = Tree()
-        x.add("myname", 0100755, myhexsha)
-        self.assertEqual(x["myname"], (0100755, myhexsha))
+        x.add("myname", 0o100755, myhexsha)
+        self.assertEqual(x["myname"], (0o100755, myhexsha))
         self.assertEqual('100755 myname\0' + hex_to_sha(myhexsha),
                 x.as_raw_string())
 
@@ -560,23 +560,23 @@ class TreeTests(ShaFileCheckTests):
         x = Tree()
         warnings.simplefilter("ignore", DeprecationWarning)
         try:
-            x.add(0100755, "myname", myhexsha)
+            x.add(0o100755, "myname", myhexsha)
         finally:
             warnings.resetwarnings()
-        self.assertEqual(x["myname"], (0100755, myhexsha))
+        self.assertEqual(x["myname"], (0o100755, myhexsha))
         self.assertEqual('100755 myname\0' + hex_to_sha(myhexsha),
                 x.as_raw_string())
 
     def test_simple(self):
         myhexsha = "d80c186a03f423a81b39df39dc87fd269736ca86"
         x = Tree()
-        x["myname"] = (0100755, myhexsha)
+        x["myname"] = (0o100755, myhexsha)
         self.assertEqual('100755 myname\0' + hex_to_sha(myhexsha),
                 x.as_raw_string())
 
     def test_tree_update_id(self):
         x = Tree()
-        x["a.c"] = (0100755, "d80c186a03f423a81b39df39dc87fd269736ca86")
+        x["a.c"] = (0o100755, "d80c186a03f423a81b39df39dc87fd269736ca86")
         self.assertEqual("0c5c6bc2c081accfbc250331b19e43b904ab9cdd", x.id)
         x["a.b"] = (stat.S_IFDIR, "d80c186a03f423a81b39df39dc87fd269736ca86")
         self.assertEqual("07bfcb5f3ada15bbebdfa3bbb8fd858a363925c8", x.id)
@@ -596,7 +596,7 @@ class TreeTests(ShaFileCheckTests):
     def _do_test_parse_tree(self, parse_tree):
         dir = os.path.join(os.path.dirname(__file__), 'data', 'trees')
         o = Tree.from_path(hex_to_filename(dir, tree_sha))
-        self.assertEqual([('a', 0100644, a_sha), ('b', 0100644, b_sha)],
+        self.assertEqual([('a', 0o100644, a_sha), ('b', 0o100644, b_sha)],
                           list(parse_tree(o.as_raw_string())))
         # test a broken tree that has a leading 0 on the file mode
         broken_tree = '0100644 foo\0' + hex_to_sha(a_sha)
@@ -604,7 +604,7 @@ class TreeTests(ShaFileCheckTests):
         def eval_parse_tree(*args, **kwargs):
             return list(parse_tree(*args, **kwargs))
 
-        self.assertEqual([('foo', 0100644, a_sha)],
+        self.assertEqual([('foo', 0o100644, a_sha)],
                           eval_parse_tree(broken_tree))
         self.assertRaises(ObjectFormatException,
                           eval_parse_tree, broken_tree, strict=True)
@@ -631,7 +631,7 @@ class TreeTests(ShaFileCheckTests):
 
         myhexsha = 'd80c186a03f423a81b39df39dc87fd269736ca86'
         self.assertRaises(errors, do_sort, {'foo': ('xxx', myhexsha)})
-        self.assertRaises(errors, do_sort, {'foo': (0100755, 12345)})
+        self.assertRaises(errors, do_sort, {'foo': (0o100755, 12345)})
 
     test_sorted_tree_items = functest_builder(_do_test_sorted_tree_items,
                                               _sorted_tree_items_py)
@@ -642,7 +642,7 @@ class TreeTests(ShaFileCheckTests):
         self.assertEqual([
           TreeEntry('a', stat.S_IFDIR,
                     'd80c186a03f423a81b39df39dc87fd269736ca86'),
-          TreeEntry('a.c', 0100755, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
+          TreeEntry('a.c', 0o100755, 'd80c186a03f423a81b39df39dc87fd269736ca86'),
           TreeEntry('a/c', stat.S_IFDIR,
                     'd80c186a03f423a81b39df39dc87fd269736ca86'),
           ], list(sorted_tree_items(_TREE_ITEMS, True)))
@@ -687,7 +687,7 @@ class TreeTests(ShaFileCheckTests):
 
     def test_iter(self):
         t = Tree()
-        t["foo"] = (0100644, a_sha)
+        t["foo"] = (0o100644, a_sha)
         self.assertEqual(set(["foo"]), set(t))
 
 
@@ -785,7 +785,7 @@ class TagParseTests(ShaFileCheckTests):
 
     def test_check_duplicates(self):
         # duplicate each of the header fields
-        for i in xrange(4):
+        for i in range(4):
             lines = self.make_tag_lines()
             lines.insert(i, lines[i])
             self.assertCheckFails(Tag, '\n'.join(lines))

dulwich/tests/test_pack.py

@@ -20,7 +20,7 @@
 """Tests for Dulwich packs."""
 
 
-from cStringIO import StringIO
+from io import BytesIO
 from hashlib import sha1
 import os
 import shutil
@@ -71,7 +71,7 @@ from dulwich.pack import (
 from dulwich.tests import (
     TestCase,
     )
-from utils import (
+from dulwich.tests.utils import (
     make_object,
     build_pack,
     )
@@ -108,7 +108,7 @@ class PackTests(TestCase):
     def assertSucceeds(self, func, *args, **kwargs):
         try:
             func(*args, **kwargs)
-        except ChecksumMismatch, e:
+        except ChecksumMismatch as e:
             self.fail(e)
 
 
@@ -216,18 +216,18 @@ class TestPackData(PackTests):
         for offset, type_num, chunks, crc32 in p.iterobjects():
             actual.append((offset, type_num, ''.join(chunks), crc32))
         self.assertEqual([
-          (12, 1, commit_data, 3775879613L),
-          (138, 2, tree_data, 912998690L),
-          (178, 3, 'test 1\n', 1373561701L)
+          (12, 1, commit_data, 3775879613),
+          (138, 2, tree_data, 912998690),
+          (178, 3, 'test 1\n', 1373561701)
           ], actual)
 
     def test_iterentries(self):
         p = self.get_pack_data(pack1_sha)
         entries = set((sha_to_hex(s), o, c) for s, o, c in p.iterentries())
         self.assertEqual(set([
-          ('6f670c0fb53f9463760b7295fbb814e965fb20c8', 178, 1373561701L),
-          ('b2a2766a2879c209ab1176e7e778b81ae422eeaa', 138, 912998690L),
-          ('f18faa16531ac570a3fdc8c7ca16682548dafd12', 12, 3775879613L),
+          ('6f670c0fb53f9463760b7295fbb814e965fb20c8', 178, 1373561701),
+          ('b2a2766a2879c209ab1176e7e778b81ae422eeaa', 138, 912998690),
+          ('f18faa16531ac570a3fdc8c7ca16682548dafd12', 12, 3775879613),
           ]), entries)
 
     def test_create_index_v1(self):
@@ -247,7 +247,7 @@ class TestPackData(PackTests):
         self.assertEqual(idx1, idx2)
 
     def test_compute_file_sha(self):
-        f = StringIO('abcd1234wxyz')
+        f = BytesIO('abcd1234wxyz')
         self.assertEqual(sha1('abcd1234wxyz').hexdigest(),
                          compute_file_sha(f).hexdigest())
         self.assertEqual(sha1('abcd1234wxyz').hexdigest(),
@@ -385,10 +385,10 @@ class TestPack(PackTests):
         Pack.from_objects(data, index).check_length_and_checksum()
 
         data._file.seek(12)
-        bad_file = StringIO()
+        bad_file = BytesIO()
         write_pack_header(bad_file, 9999)
         bad_file.write(data._file.read())
-        bad_file = StringIO(bad_file.getvalue())
+        bad_file = BytesIO(bad_file.getvalue())
         bad_data = PackData('', file=bad_file)
         bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
         self.assertRaises(AssertionError, lambda: bad_pack.data)
@@ -401,7 +401,7 @@ class TestPack(PackTests):
         Pack.from_objects(data, index).check_length_and_checksum()
 
         data._file.seek(0)
-        bad_file = StringIO(data._file.read()[:-20] + ('\xff' * 20))
+        bad_file = BytesIO(data._file.read()[:-20] + ('\xff' * 20))
         bad_data = PackData('', file=bad_file)
         bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
         self.assertRaises(ChecksumMismatch, lambda: bad_pack.data)
@@ -476,13 +476,13 @@ class TestThinPack(PackTests):
 class WritePackTests(TestCase):
 
     def test_write_pack_header(self):
-        f = StringIO()
+        f = BytesIO()
         write_pack_header(f, 42)
         self.assertEqual('PACK\x00\x00\x00\x02\x00\x00\x00*',
                 f.getvalue())
 
     def test_write_pack_object(self):
-        f = StringIO()
+        f = BytesIO()
         f.write('header')
         offset = f.tell()
         crc32 = write_pack_object(f, Blob.type_num, 'blob')
@@ -499,7 +499,7 @@ class WritePackTests(TestCase):
         self.assertEqual('x', unused)
 
     def test_write_pack_object_sha(self):
-        f = StringIO()
+        f = BytesIO()
         f.write('header')
         offset = f.tell()
         sha_a = sha1('foo')
@@ -518,7 +518,7 @@ class BaseTestPackIndexWriting(object):
     def assertSucceeds(self, func, *args, **kwargs):
         try:
             func(*args, **kwargs)
-        except ChecksumMismatch, e:
+        except ChecksumMismatch as e:
             self.fail(e)
 
     def index(self, filename, entries, pack_checksum):
@@ -589,7 +589,7 @@ class BaseTestFilePackIndexWriting(BaseTestPackIndexWriting):
         return idx
 
     def writeIndex(self, filename, entries, pack_checksum):
-        # FIXME: Write to StringIO instead rather than hitting disk ?
+        # FIXME: Write to BytesIO instead rather than hitting disk ?
         f = GitFile(filename, "wb")
         try:
             self._write_fn(f, entries, pack_checksum)
@@ -655,7 +655,7 @@ class ReadZlibTests(TestCase):
 
     def setUp(self):
         super(ReadZlibTests, self).setUp()
-        self.read = StringIO(self.comp + self.extra).read
+        self.read = BytesIO(self.comp + self.extra).read
         self.unpacked = UnpackedObject(Tree.type_num, None, len(self.decomp), 0)
 
     def test_decompress_size(self):
@@ -671,16 +671,16 @@ class ReadZlibTests(TestCase):
                           self.unpacked)
 
     def test_decompress_truncated(self):
-        read = StringIO(self.comp[:10]).read
+        read = BytesIO(self.comp[:10]).read
         self.assertRaises(zlib.error, read_zlib_chunks, read, self.unpacked)
 
-        read = StringIO(self.comp).read
+        read = BytesIO(self.comp).read
         self.assertRaises(zlib.error, read_zlib_chunks, read, self.unpacked)
 
     def test_decompress_empty(self):
         unpacked = UnpackedObject(Tree.type_num, None, 0, None)
         comp = zlib.compress('')
-        read = StringIO(comp + self.extra).read
+        read = BytesIO(comp + self.extra).read
         unused = read_zlib_chunks(read, unpacked)
         self.assertEqual('', ''.join(unpacked.decomp_chunks))
         self.assertNotEquals('', unused)
@@ -747,13 +747,13 @@ class DeltifyTests(TestCase):
 class TestPackStreamReader(TestCase):
 
     def test_read_objects_emtpy(self):
-        f = StringIO()
+        f = BytesIO()
         build_pack(f, [])
         reader = PackStreamReader(f.read)
         self.assertEqual(0, len(list(reader.read_objects())))
 
     def test_read_objects(self):
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (Blob.type_num, 'blob'),
           (OFS_DELTA, (0, 'blob1')),
@@ -781,7 +781,7 @@ class TestPackStreamReader(TestCase):
         self.assertEqual(entries[1][4], unpacked_delta.crc32)
 
     def test_read_objects_buffered(self):
-        f = StringIO()
+        f = BytesIO()
         build_pack(f, [
           (Blob.type_num, 'blob'),
           (OFS_DELTA, (0, 'blob1')),
@@ -790,7 +790,7 @@ class TestPackStreamReader(TestCase):
         self.assertEqual(2, len(list(reader.read_objects())))
 
     def test_read_objects_empty(self):
-        reader = PackStreamReader(StringIO().read)
+        reader = PackStreamReader(BytesIO().read)
         self.assertEqual([], list(reader.read_objects()))
 
 
@@ -851,7 +851,7 @@ class DeltaChainIteratorTests(TestCase):
         self.assertEqual(expected, list(pack_iter._walk_all_chains()))
 
     def test_no_deltas(self):
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (Commit.type_num, 'commit'),
           (Blob.type_num, 'blob'),
@@ -860,7 +860,7 @@ class DeltaChainIteratorTests(TestCase):
         self.assertEntriesMatch([0, 1, 2], entries, self.make_pack_iter(f))
 
     def test_ofs_deltas(self):
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (Blob.type_num, 'blob'),
           (OFS_DELTA, (0, 'blob1')),
@@ -869,7 +869,7 @@ class DeltaChainIteratorTests(TestCase):
         self.assertEntriesMatch([0, 1, 2], entries, self.make_pack_iter(f))
 
     def test_ofs_deltas_chain(self):
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (Blob.type_num, 'blob'),
           (OFS_DELTA, (0, 'blob1')),
@@ -878,7 +878,7 @@ class DeltaChainIteratorTests(TestCase):
         self.assertEntriesMatch([0, 1, 2], entries, self.make_pack_iter(f))
 
     def test_ref_deltas(self):
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (REF_DELTA, (1, 'blob1')),
           (Blob.type_num, ('blob')),
@@ -887,7 +887,7 @@ class DeltaChainIteratorTests(TestCase):
         self.assertEntriesMatch([1, 0, 2], entries, self.make_pack_iter(f))
 
     def test_ref_deltas_chain(self):
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (REF_DELTA, (2, 'blob1')),
           (Blob.type_num, ('blob')),
@@ -898,7 +898,7 @@ class DeltaChainIteratorTests(TestCase):
     def test_ofs_and_ref_deltas(self):
         # Deltas pending on this offset are popped before deltas depending on
         # this ref.
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (REF_DELTA, (1, 'blob1')),
           (Blob.type_num, ('blob')),
@@ -907,7 +907,7 @@ class DeltaChainIteratorTests(TestCase):
         self.assertEntriesMatch([1, 2, 0], entries, self.make_pack_iter(f))
 
     def test_mixed_chain(self):
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (Blob.type_num, 'blob'),
           (REF_DELTA, (2, 'blob2')),
@@ -921,24 +921,24 @@ class DeltaChainIteratorTests(TestCase):
     def test_long_chain(self):
         n = 100
         objects_spec = [(Blob.type_num, 'blob')]
-        for i in xrange(n):
+        for i in range(n):
             objects_spec.append((OFS_DELTA, (i, 'blob%i' % i)))
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, objects_spec)
-        self.assertEntriesMatch(xrange(n + 1), entries, self.make_pack_iter(f))
+        self.assertEntriesMatch(range(n + 1), entries, self.make_pack_iter(f))
 
     def test_branchy_chain(self):
         n = 100
         objects_spec = [(Blob.type_num, 'blob')]
-        for i in xrange(n):
+        for i in range(n):
             objects_spec.append((OFS_DELTA, (0, 'blob%i' % i)))
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, objects_spec)
-        self.assertEntriesMatch(xrange(n + 1), entries, self.make_pack_iter(f))
+        self.assertEntriesMatch(range(n + 1), entries, self.make_pack_iter(f))
 
     def test_ext_ref(self):
         blob, = self.store_blobs(['blob'])
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [(REF_DELTA, (blob.id, 'blob1'))],
                              store=self.store)
         pack_iter = self.make_pack_iter(f)
@@ -947,7 +947,7 @@ class DeltaChainIteratorTests(TestCase):
 
     def test_ext_ref_chain(self):
         blob, = self.store_blobs(['blob'])
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (REF_DELTA, (1, 'blob2')),
           (REF_DELTA, (blob.id, 'blob1')),
@@ -956,9 +956,25 @@ class DeltaChainIteratorTests(TestCase):
         self.assertEntriesMatch([1, 0], entries, pack_iter)
         self.assertEqual([hex_to_sha(blob.id)], pack_iter.ext_refs())
 
+    def test_ext_ref_chain_degenerate(self):
+        # Test a degenerate case where the sender is sending a REF_DELTA
+        # object that expands to an object already in the repository.
+        blob, = self.store_blobs(['blob'])
+        blob2, = self.store_blobs(['blob2'])
+        assert blob.id < blob2.id
+
+        f = BytesIO()
+        entries = build_pack(f, [
+          (REF_DELTA, (blob.id, 'blob2')),
+          (REF_DELTA, (0, 'blob3')),
+          ], store=self.store)
+        pack_iter = self.make_pack_iter(f)
+        self.assertEntriesMatch([0, 1], entries, pack_iter)
+        self.assertEqual([hex_to_sha(blob.id)], pack_iter.ext_refs())
+
     def test_ext_ref_multiple_times(self):
         blob, = self.store_blobs(['blob'])
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (REF_DELTA, (blob.id, 'blob1')),
           (REF_DELTA, (blob.id, 'blob2')),
@@ -969,7 +985,7 @@ class DeltaChainIteratorTests(TestCase):
 
     def test_multiple_ext_refs(self):
         b1, b2 = self.store_blobs(['foo', 'bar'])
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (REF_DELTA, (b1.id, 'foo1')),
           (REF_DELTA, (b2.id, 'bar2')),
@@ -981,19 +997,19 @@ class DeltaChainIteratorTests(TestCase):
 
     def test_bad_ext_ref_non_thin_pack(self):
         blob, = self.store_blobs(['blob'])
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [(REF_DELTA, (blob.id, 'blob1'))],
                              store=self.store)
         pack_iter = self.make_pack_iter(f, thin=False)
         try:
             list(pack_iter._walk_all_chains())
             self.fail()
-        except KeyError, e:
+        except KeyError as e:
             self.assertEqual(([blob.id],), e.args)
 
     def test_bad_ext_ref_thin_pack(self):
         b1, b2, b3 = self.store_blobs(['foo', 'bar', 'baz'])
-        f = StringIO()
+        f = BytesIO()
         entries = build_pack(f, [
           (REF_DELTA, (1, 'foo99')),
           (REF_DELTA, (b1.id, 'foo1')),
@@ -1006,5 +1022,5 @@ class DeltaChainIteratorTests(TestCase):
         try:
             list(pack_iter._walk_all_chains())
             self.fail()
-        except KeyError, e:
+        except KeyError as e:
             self.assertEqual((sorted([b2.id, b3.id]),), e.args)

dulwich/tests/test_patch.py

@@ -18,7 +18,7 @@
 
 """Tests for patch.py."""
 
-from cStringIO import StringIO
+from io import BytesIO
 
 from dulwich.objects import (
     Blob,
@@ -45,7 +45,7 @@ from dulwich.tests import (
 class WriteCommitPatchTests(TestCase):
 
     def test_simple(self):
-        f = StringIO()
+        f = BytesIO()
         c = Commit()
         c.committer = c.author = "Jelmer <jelmer@samba.org>"
         c.commit_time = c.author_time = 1271350201
@@ -88,7 +88,7 @@ Subject: [PATCH 1/2] Remove executable bit from prey.ico (triggers a lintian war
 -- 
 1.7.0.4
 """
-        c, diff, version = git_am_patch_split(StringIO(text))
+        c, diff, version = git_am_patch_split(BytesIO(text))
         self.assertEqual("Jelmer Vernooij <jelmer@samba.org>", c.committer)
         self.assertEqual("Jelmer Vernooij <jelmer@samba.org>", c.author)
         self.assertEqual("Remove executable bit from prey.ico "
@@ -118,7 +118,7 @@ Subject:  [Dulwich-users] [PATCH] Added unit tests for
 -- 
 1.7.0.4
 """
-        c, diff, version = git_am_patch_split(StringIO(text))
+        c, diff, version = git_am_patch_split(BytesIO(text))
         self.assertEqual('Added unit tests for dulwich.object_store.tree_lookup_path.\n\n* dulwich/tests/test_object_store.py\n  (TreeLookupPathTests): This test case contains a few tests that ensure the\n   tree_lookup_path function works as expected.\n', c.message)
 
     def test_extract_pseudo_from_header(self):
@@ -141,7 +141,7 @@ From: Jelmer Vernooy <jelmer@debian.org>
 -- 
 1.7.0.4
 """
-        c, diff, version = git_am_patch_split(StringIO(text))
+        c, diff, version = git_am_patch_split(BytesIO(text))
         self.assertEqual("Jelmer Vernooy <jelmer@debian.org>", c.author)
         self.assertEqual('Added unit tests for dulwich.object_store.tree_lookup_path.\n\n* dulwich/tests/test_object_store.py\n  (TreeLookupPathTests): This test case contains a few tests that ensure the\n   tree_lookup_path function works as expected.\n', c.message)
 
@@ -160,7 +160,7 @@ From: Jelmer Vernooy <jelmer@debian.org>
  mode change 100755 => 100644 pixmaps/prey.ico
 
 """
-        c, diff, version = git_am_patch_split(StringIO(text))
+        c, diff, version = git_am_patch_split(BytesIO(text))
         self.assertEqual(None, version)
 
     def test_extract_mercurial(self):
@@ -171,7 +171,7 @@ From: Jelmer Vernooy <jelmer@debian.org>
 @@ -158,7 +158,7 @@
  
  '''
-         c, diff, version = git_am_patch_split(StringIO(text))
+         c, diff, version = git_am_patch_split(BytesIO(text))
 -        self.assertIs(None, version)
 +        self.assertEqual(None, version)
  
@@ -196,7 +196,7 @@ Unsubscribe : https://launchpad.net/~dulwich-users
 More help   : https://help.launchpad.net/ListHelp
 
 """ % expected_diff
-        c, diff, version = git_am_patch_split(StringIO(text))
+        c, diff, version = git_am_patch_split(BytesIO(text))
         self.assertEqual(expected_diff, diff)
         self.assertEqual(None, version)
 
@@ -205,9 +205,9 @@ class DiffTests(TestCase):
     """Tests for write_blob_diff and write_tree_diff."""
 
     def test_blob_diff(self):
-        f = StringIO()
-        write_blob_diff(f, ("foo.txt", 0644, Blob.from_string("old\nsame\n")),
-                           ("bar.txt", 0644, Blob.from_string("new\nsame\n")))
+        f = BytesIO()
+        write_blob_diff(f, ("foo.txt", 0o644, Blob.from_string("old\nsame\n")),
+                           ("bar.txt", 0o644, Blob.from_string("new\nsame\n")))
         self.assertEqual([
             "diff --git a/foo.txt b/bar.txt",
             "index 3b0f961..a116b51 644",
@@ -220,9 +220,9 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_blob_add(self):
-        f = StringIO()
+        f = BytesIO()
         write_blob_diff(f, (None, None, None),
-                           ("bar.txt", 0644, Blob.from_string("new\nsame\n")))
+                           ("bar.txt", 0o644, Blob.from_string("new\nsame\n")))
         self.assertEqual([
             'diff --git /dev/null b/bar.txt',
              'new mode 644',
@@ -235,8 +235,8 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_blob_remove(self):
-        f = StringIO()
-        write_blob_diff(f, ("bar.txt", 0644, Blob.from_string("new\nsame\n")),
+        f = BytesIO()
+        write_blob_diff(f, ("bar.txt", 0o644, Blob.from_string("new\nsame\n")),
                            (None, None, None))
         self.assertEqual([
             'diff --git a/bar.txt /dev/null',
@@ -250,7 +250,7 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_tree_diff(self):
-        f = StringIO()
+        f = BytesIO()
         store = MemoryObjectStore()
         added = Blob.from_string("add\n")
         removed = Blob.from_string("removed\n")
@@ -258,13 +258,13 @@ class DiffTests(TestCase):
         changed2 = Blob.from_string("unchanged\nadded\n")
         unchanged = Blob.from_string("unchanged\n")
         tree1 = Tree()
-        tree1.add("removed.txt", 0644, removed.id)
-        tree1.add("changed.txt", 0644, changed1.id)
-        tree1.add("unchanged.txt", 0644, changed1.id)
+        tree1.add("removed.txt", 0o644, removed.id)
+        tree1.add("changed.txt", 0o644, changed1.id)
+        tree1.add("unchanged.txt", 0o644, changed1.id)
         tree2 = Tree()
-        tree2.add("added.txt", 0644, added.id)
-        tree2.add("changed.txt", 0644, changed2.id)
-        tree2.add("unchanged.txt", 0644, changed1.id)
+        tree2.add("added.txt", 0o644, added.id)
+        tree2.add("changed.txt", 0o644, changed2.id)
+        tree2.add("unchanged.txt", 0o644, changed1.id)
         store.add_objects([(o, None) for o in [
             tree1, tree2, added, removed, changed1, changed2, unchanged]])
         write_tree_diff(f, store, tree1.id, tree2.id)
@@ -294,7 +294,7 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_tree_diff_submodule(self):
-        f = StringIO()
+        f = BytesIO()
         store = MemoryObjectStore()
         tree1 = Tree()
         tree1.add("asubmodule", S_IFGITLINK,
@@ -315,13 +315,13 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_object_diff_blob(self):
-        f = StringIO()
+        f = BytesIO()
         b1 = Blob.from_string("old\nsame\n")
         b2 = Blob.from_string("new\nsame\n")
         store = MemoryObjectStore()
         store.add_objects([(b1, None), (b2, None)])
-        write_object_diff(f, store, ("foo.txt", 0644, b1.id),
-                                    ("bar.txt", 0644, b2.id))
+        write_object_diff(f, store, ("foo.txt", 0o644, b1.id),
+                                    ("bar.txt", 0o644, b2.id))
         self.assertEqual([
             "diff --git a/foo.txt b/bar.txt",
             "index 3b0f961..a116b51 644",
@@ -334,12 +334,12 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_object_diff_add_blob(self):
-        f = StringIO()
+        f = BytesIO()
         store = MemoryObjectStore()
         b2 = Blob.from_string("new\nsame\n")
         store.add_object(b2)
         write_object_diff(f, store, (None, None, None),
-                                    ("bar.txt", 0644, b2.id))
+                                    ("bar.txt", 0o644, b2.id))
         self.assertEqual([
             'diff --git /dev/null b/bar.txt',
              'new mode 644',
@@ -352,11 +352,11 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_object_diff_remove_blob(self):
-        f = StringIO()
+        f = BytesIO()
         b1 = Blob.from_string("new\nsame\n")
         store = MemoryObjectStore()
         store.add_object(b1)
-        write_object_diff(f, store, ("bar.txt", 0644, b1.id),
+        write_object_diff(f, store, ("bar.txt", 0o644, b1.id),
                                     (None, None, None))
         self.assertEqual([
             'diff --git a/bar.txt /dev/null',
@@ -370,7 +370,7 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_object_diff_bin_blob_force(self):
-        f = StringIO()
+        f = BytesIO()
         # Prepare two slightly different PNG headers
         b1 = Blob.from_string(
             "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52"
@@ -380,8 +380,8 @@ class DiffTests(TestCase):
             "\x00\x00\x01\xd5\x00\x00\x00\x9f\x08\x03\x00\x00\x00\x98\xd3\xb3")
         store = MemoryObjectStore()
         store.add_objects([(b1, None), (b2, None)])
-        write_object_diff(f, store, ('foo.png', 0644, b1.id),
-                                    ('bar.png', 0644, b2.id), diff_binary=True)
+        write_object_diff(f, store, ('foo.png', 0o644, b1.id),
+                                    ('bar.png', 0o644, b2.id), diff_binary=True)
         self.assertEqual([
             'diff --git a/foo.png b/bar.png',
             'index f73e47d..06364b7 644',
@@ -398,7 +398,7 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_object_diff_bin_blob(self):
-        f = StringIO()
+        f = BytesIO()
         # Prepare two slightly different PNG headers
         b1 = Blob.from_string(
             "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52"
@@ -408,8 +408,8 @@ class DiffTests(TestCase):
             "\x00\x00\x01\xd5\x00\x00\x00\x9f\x08\x03\x00\x00\x00\x98\xd3\xb3")
         store = MemoryObjectStore()
         store.add_objects([(b1, None), (b2, None)])
-        write_object_diff(f, store, ('foo.png', 0644, b1.id),
-                                    ('bar.png', 0644, b2.id))
+        write_object_diff(f, store, ('foo.png', 0o644, b1.id),
+                                    ('bar.png', 0o644, b2.id))
         self.assertEqual([
             'diff --git a/foo.png b/bar.png',
             'index f73e47d..06364b7 644',
@@ -417,14 +417,14 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_object_diff_add_bin_blob(self):
-        f = StringIO()
+        f = BytesIO()
         b2 = Blob.from_string(
             '\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52'
             '\x00\x00\x01\xd5\x00\x00\x00\x9f\x08\x03\x00\x00\x00\x98\xd3\xb3')
         store = MemoryObjectStore()
         store.add_object(b2)
         write_object_diff(f, store, (None, None, None),
-                                    ('bar.png', 0644, b2.id))
+                                    ('bar.png', 0o644, b2.id))
         self.assertEqual([
             'diff --git /dev/null b/bar.png',
             'new mode 644',
@@ -433,13 +433,13 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_object_diff_remove_bin_blob(self):
-        f = StringIO()
+        f = BytesIO()
         b1 = Blob.from_string(
             '\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52'
             '\x00\x00\x01\xd5\x00\x00\x00\x9f\x08\x04\x00\x00\x00\x05\x04\x8b')
         store = MemoryObjectStore()
         store.add_object(b1)
-        write_object_diff(f, store, ('foo.png', 0644, b1.id),
+        write_object_diff(f, store, ('foo.png', 0o644, b1.id),
                                     (None, None, None))
         self.assertEqual([
             'diff --git a/foo.png /dev/null',
@@ -449,12 +449,12 @@ class DiffTests(TestCase):
             ], f.getvalue().splitlines())
 
     def test_object_diff_kind_change(self):
-        f = StringIO()
+        f = BytesIO()
         b1 = Blob.from_string("new\nsame\n")
         store = MemoryObjectStore()
         store.add_object(b1)
-        write_object_diff(f, store, ("bar.txt", 0644, b1.id),
-            ("bar.txt", 0160000, "06d0bdd9e2e20377b3180e4986b14c8549b393e4"))
+        write_object_diff(f, store, ("bar.txt", 0o644, b1.id),
+            ("bar.txt", 0o160000, "06d0bdd9e2e20377b3180e4986b14c8549b393e4"))
         self.assertEqual([
             'diff --git a/bar.txt b/bar.txt',
             'old mode 644',

dulwich/tests/test_porcelain.py

@@ -18,7 +18,7 @@
 
 """Tests for dulwich.porcelain."""
 
-from cStringIO import StringIO
+from io import BytesIO
 import os
 import shutil
 import tarfile
@@ -28,6 +28,7 @@ from dulwich import porcelain
 from dulwich.diff_tree import tree_changes
 from dulwich.objects import (
     Blob,
+    Tag,
     Tree,
     )
 from dulwich.repo import Repo
@@ -55,8 +56,8 @@ class ArchiveTests(PorcelainTestCase):
     def test_simple(self):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1], [3, 1, 2]])
         self.repo.refs["refs/heads/master"] = c3.id
-        out = StringIO()
-        err = StringIO()
+        out = BytesIO()
+        err = BytesIO()
         porcelain.archive(self.repo.path, "refs/heads/master", outstream=out,
             errstream=err)
         self.assertEquals("", err.getvalue())
@@ -84,7 +85,7 @@ class CommitTests(PorcelainTestCase):
         self.repo.refs["refs/heads/foo"] = c3.id
         sha = porcelain.commit(self.repo.path, message="Some message",
                 author="Joe <joe@example.com>", committer="Bob <bob@example.com>")
-        self.assertTrue(type(sha) is str)
+        self.assertTrue(isinstance(sha, str))
         self.assertEquals(len(sha), 40)
 
 
@@ -101,7 +102,7 @@ class CloneTests(PorcelainTestCase):
                                         commit_spec, trees)
         self.repo.refs["refs/heads/master"] = c3.id
         target_path = tempfile.mkdtemp()
-        outstream = StringIO()
+        outstream = BytesIO()
         self.addCleanup(shutil.rmtree, target_path)
         r = porcelain.clone(self.repo.path, target_path,
                             checkout=False, outstream=outstream)
@@ -121,7 +122,7 @@ class CloneTests(PorcelainTestCase):
                                         commit_spec, trees)
         self.repo.refs["refs/heads/master"] = c3.id
         target_path = tempfile.mkdtemp()
-        outstream = StringIO()
+        outstream = BytesIO()
         self.addCleanup(shutil.rmtree, target_path)
         r = porcelain.clone(self.repo.path, target_path,
                             checkout=True, outstream=outstream)
@@ -141,7 +142,7 @@ class CloneTests(PorcelainTestCase):
                                         commit_spec, trees)
         self.repo.refs["refs/heads/master"] = c3.id
         target_path = tempfile.mkdtemp()
-        outstream = StringIO()
+        outstream = BytesIO()
         self.addCleanup(shutil.rmtree, target_path)
         r = porcelain.clone(self.repo.path, target_path,
                             bare=True, outstream=outstream)
@@ -158,7 +159,7 @@ class CloneTests(PorcelainTestCase):
         (c1, ) = build_commit_graph(self.repo.object_store, commit_spec, trees)
         self.repo.refs["refs/heads/master"] = c1.id
         target_path = tempfile.mkdtemp()
-        outstream = StringIO()
+        outstream = BytesIO()
         self.addCleanup(shutil.rmtree, target_path)
         self.assertRaises(ValueError, porcelain.clone, self.repo.path,
             target_path, checkout=True, bare=True, outstream=outstream)
@@ -179,12 +180,30 @@ class InitTests(TestCase):
 
 class AddTests(PorcelainTestCase):
 
+    def test_add_default_paths(self):
+
+        # create a file for initial commit
+        with open(os.path.join(self.repo.path, 'blah'), 'w') as f:
+            f.write("\n")
+        porcelain.add(repo=self.repo.path, paths=['blah'])
+        porcelain.commit(repo=self.repo.path, message='test',
+            author='test', committer='test')
+
+        # Add a second test file and a file in a directory
+        with open(os.path.join(self.repo.path, 'foo'), 'w') as f:
+            f.write("\n")
+        os.mkdir(os.path.join(self.repo.path, 'adir'))
+        with open(os.path.join(self.repo.path, 'adir', 'afile'), 'w') as f:
+            f.write("\n")
+        porcelain.add(self.repo.path)
+
+        # Check that foo was added and nothing in .git was modified
+        index = self.repo.open_index()
+        self.assertEquals(list(index), ['blah', 'foo', 'adir/afile'])
+
     def test_add_file(self):
-        f = open(os.path.join(self.repo.path, 'foo'), 'w')
-        try:
+        with open(os.path.join(self.repo.path, 'foo'), 'w') as f:
             f.write("BAR")
-        finally:
-            f.close()
         porcelain.add(self.repo.path, paths=["foo"])
 
 
@@ -206,7 +225,7 @@ class LogTests(PorcelainTestCase):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
             [3, 1, 2]])
         self.repo.refs["HEAD"] = c3.id
-        outstream = StringIO()
+        outstream = BytesIO()
         porcelain.log(self.repo.path, outstream=outstream)
         self.assertEquals(3, outstream.getvalue().count("-" * 50))
 
@@ -214,7 +233,7 @@ class LogTests(PorcelainTestCase):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
             [3, 1, 2]])
         self.repo.refs["HEAD"] = c3.id
-        outstream = StringIO()
+        outstream = BytesIO()
         porcelain.log(self.repo.path, outstream=outstream, max_entries=1)
         self.assertEquals(1, outstream.getvalue().count("-" * 50))
 
@@ -225,7 +244,7 @@ class ShowTests(PorcelainTestCase):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
             [3, 1, 2]])
         self.repo.refs["HEAD"] = c3.id
-        outstream = StringIO()
+        outstream = BytesIO()
         porcelain.show(self.repo.path, objects=c3.id, outstream=outstream)
         self.assertTrue(outstream.getvalue().startswith("-" * 50))
 
@@ -233,14 +252,14 @@ class ShowTests(PorcelainTestCase):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
             [3, 1, 2]])
         self.repo.refs["HEAD"] = c3.id
-        outstream = StringIO()
+        outstream = BytesIO()
         porcelain.show(self.repo.path, objects=[c3.id], outstream=outstream)
         self.assertTrue(outstream.getvalue().startswith("-" * 50))
 
     def test_blob(self):
         b = Blob.from_string("The Foo\n")
         self.repo.object_store.add_object(b)
-        outstream = StringIO()
+        outstream = BytesIO()
         porcelain.show(self.repo.path, objects=[b.id], outstream=outstream)
         self.assertEquals(outstream.getvalue(), "The Foo\n")
 
@@ -252,7 +271,7 @@ class SymbolicRefTests(PorcelainTestCase):
             [3, 1, 2]])
         self.repo.refs["HEAD"] = c3.id
 
-        outstream = StringIO()
+        outstream = BytesIO()
         self.assertRaises(ValueError, porcelain.symbolic_ref, self.repo.path, 'foobar')
 
     def test_set_force_wrong_symbolic_ref(self):
@@ -292,7 +311,7 @@ class DiffTreeTests(PorcelainTestCase):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
             [3, 1, 2]])
         self.repo.refs["HEAD"] = c3.id
-        outstream = StringIO()
+        outstream = BytesIO()
         porcelain.diff_tree(self.repo.path, c2.tree, c3.tree, outstream=outstream)
         self.assertEquals(outstream.getvalue(), "")
 
@@ -305,14 +324,14 @@ class CommitTreeTests(PorcelainTestCase):
         b = Blob()
         b.data = "foo the bar"
         t = Tree()
-        t.add("somename", 0100644, b.id)
+        t.add("somename", 0o100644, b.id)
         self.repo.object_store.add_object(t)
         self.repo.object_store.add_object(b)
         sha = porcelain.commit_tree(
             self.repo.path, t.id, message="Withcommit.",
             author="Joe <joe@example.com>",
             committer="Jane <jane@example.com>")
-        self.assertTrue(type(sha) is str)
+        self.assertTrue(isinstance(sha, str))
         self.assertEquals(len(sha), 40)
 
 
@@ -321,7 +340,7 @@ class RevListTests(PorcelainTestCase):
     def test_simple(self):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
             [3, 1, 2]])
-        outstream = StringIO()
+        outstream = BytesIO()
         porcelain.rev_list(
             self.repo.path, [c3.id], outstream=outstream)
         self.assertEquals(
@@ -331,19 +350,46 @@ class RevListTests(PorcelainTestCase):
 
 class TagTests(PorcelainTestCase):
 
-    def test_simple(self):
-        tag = 'tryme'
-        author = 'foo'
-        message = 'bar'
+    def test_annotated(self):
+        c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
+            [3, 1, 2]])
+        self.repo.refs["HEAD"] = c3.id
+
+        porcelain.tag(self.repo.path, "tryme", 'foo <foo@bar.com>', 'bar',
+                annotated=True)
+
+        tags = self.repo.refs.as_dict("refs/tags")
+        self.assertEquals(tags.keys(), ["tryme"])
+        tag = self.repo['refs/tags/tryme']
+        self.assertTrue(isinstance(tag, Tag))
+        self.assertEquals("foo <foo@bar.com>", tag.tagger)
+        self.assertEquals("bar", tag.message)
 
+    def test_unannotated(self):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
             [3, 1, 2]])
         self.repo.refs["HEAD"] = c3.id
 
-        porcelain.tag(self.repo.path, tag, author, message)
+        porcelain.tag(self.repo.path, "tryme", annotated=False)
 
         tags = self.repo.refs.as_dict("refs/tags")
-        self.assertEquals(tags.keys()[0], tag)
+        self.assertEquals(tags.keys(), ["tryme"])
+        tag = self.repo['refs/tags/tryme']
+        self.assertEquals(tags.values(), [self.repo.head()])
+
+
+class ListTagsTests(PorcelainTestCase):
+
+    def test_empty(self):
+        tags = porcelain.list_tags(self.repo.path)
+        self.assertEquals([], tags)
+
+    def test_simple(self):
+        self.repo.refs["refs/tags/foo"] = "aa" * 20
+        self.repo.refs["refs/tags/bar/bla"] = "bb" * 20
+        tags = porcelain.list_tags(self.repo.path)
+
+        self.assertEquals(["bar/bla", "foo"], tags)
 
 
 class ResetTests(PorcelainTestCase):
@@ -373,3 +419,80 @@ class ResetTests(PorcelainTestCase):
                        self.repo['HEAD'].tree))
 
         self.assertEquals([], changes)
+
+
+class PushTests(PorcelainTestCase):
+
+    def test_simple(self):
+        """
+        Basic test of porcelain push where self.repo is the remote.  First
+        clone the remote, commit a file to the clone, then push the changes
+        back to the remote.
+        """
+        outstream = BytesIO()
+        errstream = BytesIO()
+
+        porcelain.commit(repo=self.repo.path, message='init',
+            author='', committer='')
+
+        # Setup target repo cloned from temp test repo
+        clone_path = tempfile.mkdtemp()
+        porcelain.clone(self.repo.path, target=clone_path, outstream=outstream)
+
+        # create a second file to be pushed back to origin
+        handle, fullpath = tempfile.mkstemp(dir=clone_path)
+        porcelain.add(repo=clone_path, paths=[os.path.basename(fullpath)])
+        porcelain.commit(repo=clone_path, message='push',
+            author='', committer='')
+
+        # Setup a non-checked out branch in the remote
+        refs_path = os.path.join('refs', 'heads', 'foo')
+        self.repo[refs_path] = self.repo['HEAD']
+
+        # Push to the remote
+        porcelain.push(clone_path, self.repo.path, refs_path, outstream=outstream,
+                errstream=errstream)
+
+        # Check that the target and source
+        r_clone = Repo(clone_path)
+
+        # Get the change in the target repo corresponding to the add
+        # this will be in the foo branch.
+        change = list(tree_changes(self.repo, self.repo['HEAD'].tree,
+                                   self.repo['refs/heads/foo'].tree))[0]
+
+        self.assertEquals(r_clone['HEAD'].id, self.repo[refs_path].id)
+        self.assertEquals(os.path.basename(fullpath), change.new.path)
+
+
+class PullTests(PorcelainTestCase):
+
+    def test_simple(self):
+        outstream = BytesIO()
+        errstream = BytesIO()
+
+        # create a file for initial commit
+        handle, fullpath = tempfile.mkstemp(dir=self.repo.path)
+        filename = os.path.basename(fullpath)
+        porcelain.add(repo=self.repo.path, paths=filename)
+        porcelain.commit(repo=self.repo.path, message='test',
+                         author='test', committer='test')
+
+        # Setup target repo
+        target_path = tempfile.mkdtemp()
+        porcelain.clone(self.repo.path, target=target_path, outstream=outstream)
+
+        # create a second file to be pushed
+        handle, fullpath = tempfile.mkstemp(dir=self.repo.path)
+        filename = os.path.basename(fullpath)
+        porcelain.add(repo=self.repo.path, paths=filename)
+        porcelain.commit(repo=self.repo.path, message='test2',
+            author='test2', committer='test2')
+
+        # Pull changes into the cloned repo
+        porcelain.pull(target_path, self.repo.path, 'refs/heads/master',
+            outstream=outstream, errstream=errstream)
+
+        # Check the target repo for pushed changes
+        r = Repo(target_path)
+        self.assertEquals(r['HEAD'].id, self.repo['HEAD'].id)

dulwich/tests/test_protocol.py

@@ -19,7 +19,7 @@
 """Tests for the smart protocol utility functions."""
 
 
-from StringIO import StringIO
+from io import BytesIO
 
 from dulwich.errors import (
     HangupException,
@@ -105,16 +105,16 @@ class ProtocolTests(BaseProtocolTests, TestCase):
 
     def setUp(self):
         TestCase.setUp(self)
-        self.rout = StringIO()
-        self.rin = StringIO()
+        self.rout = BytesIO()
+        self.rin = BytesIO()
         self.proto = Protocol(self.rin.read, self.rout.write)
 
 
-class ReceivableStringIO(StringIO):
-    """StringIO with socket-like recv semantics for testing."""
+class ReceivableBytesIO(BytesIO):
+    """BytesIO with socket-like recv semantics for testing."""
 
     def __init__(self):
-        StringIO.__init__(self)
+        BytesIO.__init__(self)
         self.allow_read_past_eof = False
 
     def recv(self, size):
@@ -132,8 +132,8 @@ class ReceivableProtocolTests(BaseProtocolTests, TestCase):
 
     def setUp(self):
         TestCase.setUp(self)
-        self.rout = StringIO()
-        self.rin = ReceivableStringIO()
+        self.rout = BytesIO()
+        self.rin = ReceivableBytesIO()
         self.proto = ReceivableProtocol(self.rin.recv, self.rout.write)
         self.proto._rbufsize = 8
 
@@ -151,7 +151,7 @@ class ReceivableProtocolTests(BaseProtocolTests, TestCase):
         data = ''
         # We ask for 8 bytes each time and actually read 7, so it should take
         # exactly 10 iterations.
-        for _ in xrange(10):
+        for _ in range(10):
             data += self.proto.recv(10)
         # any more reads would block
         self.assertRaises(AssertionError, self.proto.recv, 10)
@@ -176,17 +176,17 @@ class ReceivableProtocolTests(BaseProtocolTests, TestCase):
 
     def test_mixed(self):
         # arbitrary non-repeating string
-        all_data = ','.join(str(i) for i in xrange(100))
+        all_data = ','.join(str(i) for i in range(100))
         self.rin.write(all_data)
         self.rin.seek(0)
         data = ''
 
-        for i in xrange(1, 100):
+        for i in range(1, 100):
             data += self.proto.recv(i)
             # if we get to the end, do a non-blocking read instead of blocking
             if len(data) + i > len(all_data):
                 data += self.proto.recv(i)
-                # ReceivableStringIO leaves off the last byte unless we ask
+                # ReceivableBytesIO leaves off the last byte unless we ask
                 # nicely
                 data += self.proto.recv(1)
                 break
@@ -232,7 +232,7 @@ class BufferedPktLineWriterTests(TestCase):
 
     def setUp(self):
         TestCase.setUp(self)
-        self._output = StringIO()
+        self._output = BytesIO()
         self._writer = BufferedPktLineWriter(self._output.write, bufsize=16)
 
     def assertOutputEquals(self, expected):

dulwich/tests/test_refs.py

@@ -19,7 +19,7 @@
 
 """Tests for dulwich.refs."""
 
-from cStringIO import StringIO
+from io import BytesIO
 import os
 import tempfile
 
@@ -90,16 +90,16 @@ class PackedRefsFileTests(TestCase):
                           '%s bad/../refname' % ONES)
 
     def test_read_without_peeled(self):
-        f = StringIO('# comment\n%s ref/1\n%s ref/2' % (ONES, TWOS))
+        f = BytesIO('# comment\n%s ref/1\n%s ref/2' % (ONES, TWOS))
         self.assertEqual([(ONES, 'ref/1'), (TWOS, 'ref/2')],
                          list(read_packed_refs(f)))
 
     def test_read_without_peeled_errors(self):
-        f = StringIO('%s ref/1\n^%s' % (ONES, TWOS))
+        f = BytesIO('%s ref/1\n^%s' % (ONES, TWOS))
         self.assertRaises(errors.PackedRefsException, list, read_packed_refs(f))
 
     def test_read_with_peeled(self):
-        f = StringIO('%s ref/1\n%s ref/2\n^%s\n%s ref/4' % (
+        f = BytesIO('%s ref/1\n%s ref/2\n^%s\n%s ref/4' % (
           ONES, TWOS, THREES, FOURS))
         self.assertEqual([
           (ONES, 'ref/1', None),
@@ -108,14 +108,14 @@ class PackedRefsFileTests(TestCase):
           ], list(read_packed_refs_with_peeled(f)))
 
     def test_read_with_peeled_errors(self):
-        f = StringIO('^%s\n%s ref/1' % (TWOS, ONES))
+        f = BytesIO('^%s\n%s ref/1' % (TWOS, ONES))
         self.assertRaises(errors.PackedRefsException, list, read_packed_refs(f))
 
-        f = StringIO('%s ref/1\n^%s\n^%s' % (ONES, TWOS, THREES))
+        f = BytesIO('%s ref/1\n^%s\n^%s' % (ONES, TWOS, THREES))
         self.assertRaises(errors.PackedRefsException, list, read_packed_refs(f))
 
     def test_write_with_peeled(self):
-        f = StringIO()
+        f = BytesIO()
         write_packed_refs(f, {'ref/1': ONES, 'ref/2': TWOS},
                           {'ref/1': THREES})
         self.assertEqual(
@@ -123,7 +123,7 @@ class PackedRefsFileTests(TestCase):
           ONES, THREES, TWOS), f.getvalue())
 
     def test_write_without_peeled(self):
-        f = StringIO()
+        f = BytesIO()
         write_packed_refs(f, {'ref/1': ONES, 'ref/2': TWOS})
         self.assertEqual("%s ref/1\n%s ref/2\n" % (ONES, TWOS), f.getvalue())
 
@@ -299,7 +299,7 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
 
         # ensure HEAD was not modified
         f = open(os.path.join(self._refs.path, 'HEAD'), 'rb')
-        self.assertEqual('ref: refs/heads/master', iter(f).next().rstrip('\n'))
+        self.assertEqual('ref: refs/heads/master', next(iter(f)).rstrip('\n'))
         f.close()
 
         # ensure the symbolic link was written through
@@ -429,14 +429,14 @@ class InfoRefsContainerTests(TestCase):
 
     def test_invalid_refname(self):
         text = _TEST_REFS_SERIALIZED + '00' * 20 + '\trefs/stash\n'
-        refs = InfoRefsContainer(StringIO(text))
+        refs = InfoRefsContainer(BytesIO(text))
         expected_refs = dict(_TEST_REFS)
         del expected_refs['HEAD']
         expected_refs["refs/stash"] = "00" * 20
         self.assertEqual(expected_refs, refs.as_dict())
 
     def test_keys(self):
-        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        refs = InfoRefsContainer(BytesIO(_TEST_REFS_SERIALIZED))
         actual_keys = set(refs.keys())
         self.assertEqual(set(refs.allkeys()), actual_keys)
         # ignore the symref loop if it exists
@@ -454,19 +454,19 @@ class InfoRefsContainerTests(TestCase):
                          sorted(refs.keys('refs/tags')))
 
     def test_as_dict(self):
-        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        refs = InfoRefsContainer(BytesIO(_TEST_REFS_SERIALIZED))
         # refs/heads/loop does not show up even if it exists
         expected_refs = dict(_TEST_REFS)
         del expected_refs['HEAD']
         self.assertEqual(expected_refs, refs.as_dict())
 
     def test_contains(self):
-        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        refs = InfoRefsContainer(BytesIO(_TEST_REFS_SERIALIZED))
         self.assertTrue('refs/heads/master' in refs)
         self.assertFalse('refs/heads/bar' in refs)
 
     def test_get_peeled(self):
-        refs = InfoRefsContainer(StringIO(_TEST_REFS_SERIALIZED))
+        refs = InfoRefsContainer(BytesIO(_TEST_REFS_SERIALIZED))
         # refs/heads/loop does not show up even if it exists
         self.assertEqual(
             _TEST_REFS['refs/heads/master'],

dulwich/tests/test_repository.py

@@ -108,11 +108,23 @@ class RepositoryTests(TestCase):
         self.assertEqual('a90fa2d900a17e99b433217e988c4eb4a2e9a097',
                           r["refs/tags/foo"].id)
 
-    def test_getitem_notfound_unicode(self):
+    def test_getitem_unicode(self):
         r = self._repo = open_repo('a.git')
-        # In the future, this might raise a TypeError since we don't
-        # handle unicode strings properly (what encoding?) for refs.
-        self.assertRaises(KeyError, r.__getitem__, u"11" * 19 + "--")
+
+        test_keys = [
+            ('refs/heads/master', True),
+            ('a90fa2d900a17e99b433217e988c4eb4a2e9a097', True),
+            ('11' * 19 + '--', False),
+        ]
+
+        for k, contained in test_keys:
+            self.assertEqual(k in r, contained)
+
+        for k, _ in test_keys:
+            self.assertRaisesRegexp(
+                TypeError, "'name' must be bytestring, not unicode",
+                r.__getitem__, unicode(k)
+            )
 
     def test_delitem(self):
         r = self._repo = open_repo('a.git')
@@ -448,7 +460,7 @@ exit 0
 
         (fd, path) = tempfile.mkstemp(dir=repo_dir)
         post_commit_msg = """#!/bin/sh
-unlink %(file)s
+rm %(file)s
 """ % {'file': path}
 
         root_sha = r.do_commit(
@@ -603,6 +615,21 @@ class BuildRepoTests(TestCase):
             "Jelmer <jelmer@apache.org>",
             r[commit_sha].committer)
 
+    def test_commit_config_identity_in_memoryrepo(self):
+        # commit falls back to the users' identity if it wasn't specified
+        r = MemoryRepo.init_bare([], {})
+        c = r.get_config()
+        c.set(("user", ), "name", "Jelmer")
+        c.set(("user", ), "email", "jelmer@apache.org")
+
+        commit_sha = r.do_commit('message', tree=objects.Tree().id)
+        self.assertEqual(
+            "Jelmer <jelmer@apache.org>",
+            r[commit_sha].author)
+        self.assertEqual(
+            "Jelmer <jelmer@apache.org>",
+            r[commit_sha].committer)
+
     def test_commit_fail_ref(self):
         r = self._repo
 
@@ -671,6 +698,46 @@ class BuildRepoTests(TestCase):
             [self._root_commit, merge_1],
             r[commit_sha].parents)
 
+    def test_commit_dangling_commit(self):
+        r = self._repo
+
+        old_shas = set(r.object_store)
+        old_refs = r.get_refs()
+        commit_sha = r.do_commit('commit with no ref',
+             committer='Test Committer <test@nodomain.com>',
+             author='Test Author <test@nodomain.com>',
+             commit_timestamp=12395, commit_timezone=0,
+             author_timestamp=12395, author_timezone=0,
+             ref=None)
+        new_shas = set(r.object_store) - old_shas
+
+        # New sha is added, but no new refs
+        self.assertEqual(1, len(new_shas))
+        new_commit = r[new_shas.pop()]
+        self.assertEqual(r[self._root_commit].tree, new_commit.tree)
+        self.assertEqual([], r[commit_sha].parents)
+        self.assertEqual(old_refs, r.get_refs())
+
+    def test_commit_dangling_commit_with_parents(self):
+        r = self._repo
+
+        old_shas = set(r.object_store)
+        old_refs = r.get_refs()
+        commit_sha = r.do_commit('commit with no ref',
+             committer='Test Committer <test@nodomain.com>',
+             author='Test Author <test@nodomain.com>',
+             commit_timestamp=12395, commit_timezone=0,
+             author_timestamp=12395, author_timezone=0,
+             ref=None, merge_heads=[self._root_commit])
+        new_shas = set(r.object_store) - old_shas
+
+        # New sha is added, but no new refs
+        self.assertEqual(1, len(new_shas))
+        new_commit = r[new_shas.pop()]
+        self.assertEqual(r[self._root_commit].tree, new_commit.tree)
+        self.assertEqual([self._root_commit], r[commit_sha].parents)
+        self.assertEqual(old_refs, r.get_refs())
+
     def test_stage_deleted(self):
         r = self._repo
         os.remove(os.path.join(r.path, 'a'))

dulwich/tests/test_server.py

@@ -18,7 +18,7 @@
 
 """Tests for the smart protocol server."""
 
-from cStringIO import StringIO
+from io import BytesIO
 import os
 import tempfile
 
@@ -26,6 +26,7 @@ from dulwich.errors import (
     GitProtocolError,
     NotGitRepository,
     UnexpectedCommandError,
+    HangupException,
     )
 from dulwich.objects import (
     Commit,
@@ -78,13 +79,18 @@ class TestProto(object):
         self._received = {0: [], 1: [], 2: [], 3: []}
 
     def set_output(self, output_lines):
-        self._output = ['%s\n' % line.rstrip() for line in output_lines]
+        self._output = output_lines
 
     def read_pkt_line(self):
         if self._output:
-            return self._output.pop(0)
+            data = self._output.pop(0)
+            if data is not None:
+                return '%s\n' % data.rstrip()
+            else:
+                # flush-pkt ('0000').
+                return None
         else:
-            return None
+            raise HangupException()
 
     def write_sideband(self, band, data):
         self._received[band].append(data)
@@ -120,7 +126,7 @@ class HandlerTestCase(TestCase):
     def assertSucceeds(self, func, *args, **kwargs):
         try:
             func(*args, **kwargs)
-        except GitProtocolError, e:
+        except GitProtocolError as e:
             self.fail(e)
 
     def test_capability_line(self):
@@ -219,7 +225,7 @@ class FindShallowTests(TestCase):
     def make_linear_commits(self, n, message=''):
         commits = []
         parents = []
-        for _ in xrange(n):
+        for _ in range(n):
             commits.append(self.make_commit(parents=parents, message=message))
             parents = [commits[-1].id]
         return commits
@@ -308,6 +314,27 @@ class ReceivePackHandlerTestCase(TestCase):
         self.assertEqual(status[1][1], 'ok')
 
 
+class ProtocolGraphWalkerEmptyTestCase(TestCase):
+    def setUp(self):
+        super(ProtocolGraphWalkerEmptyTestCase, self).setUp()
+        self._repo = MemoryRepo.init_bare([], {})
+        backend = DictBackend({'/': self._repo})
+        self._walker = ProtocolGraphWalker(
+            TestUploadPackHandler(backend, ['/', 'host=lolcats'], TestProto()),
+            self._repo.object_store, self._repo.get_peeled)
+
+    def test_empty_repository(self):
+        # The server should wait for a flush packet.
+        self._walker.proto.set_output([])
+        self.assertRaises(HangupException, self._walker.determine_wants, {})
+        self.assertEqual(None, self._walker.proto.get_received_line())
+
+        self._walker.proto.set_output([None])
+        self.assertEqual([], self._walker.determine_wants({}))
+        self.assertEqual(None, self._walker.proto.get_received_line())
+
+
+
 class ProtocolGraphWalkerTestCase(TestCase):
 
     def setUp(self):
@@ -371,12 +398,14 @@ class ProtocolGraphWalkerTestCase(TestCase):
         self.assertEqual((None, None), _split_proto_line('', allowed))
 
     def test_determine_wants(self):
+        self._walker.proto.set_output([None])
         self.assertEqual([], self._walker.determine_wants({}))
         self.assertEqual(None, self._walker.proto.get_received_line())
 
         self._walker.proto.set_output([
           'want %s multi_ack' % ONE,
           'want %s' % TWO,
+          None,
           ])
         heads = {
           'refs/heads/ref1': ONE,
@@ -390,20 +419,20 @@ class ProtocolGraphWalkerTestCase(TestCase):
         self.assertEqual([], self._walker.determine_wants(heads))
         self._walker.advertise_refs = False
 
-        self._walker.proto.set_output(['want %s multi_ack' % FOUR])
+        self._walker.proto.set_output(['want %s multi_ack' % FOUR, None])
         self.assertRaises(GitProtocolError, self._walker.determine_wants, heads)
 
-        self._walker.proto.set_output([])
+        self._walker.proto.set_output([None])
         self.assertEqual([], self._walker.determine_wants(heads))
 
-        self._walker.proto.set_output(['want %s multi_ack' % ONE, 'foo'])
+        self._walker.proto.set_output(['want %s multi_ack' % ONE, 'foo', None])
         self.assertRaises(GitProtocolError, self._walker.determine_wants, heads)
 
-        self._walker.proto.set_output(['want %s multi_ack' % FOUR])
+        self._walker.proto.set_output(['want %s multi_ack' % FOUR, None])
         self.assertRaises(GitProtocolError, self._walker.determine_wants, heads)
 
     def test_determine_wants_advertisement(self):
-        self._walker.proto.set_output([])
+        self._walker.proto.set_output([None])
         # advertise branch tips plus tag
         heads = {
           'refs/heads/ref4': FOUR,
@@ -439,7 +468,7 @@ class ProtocolGraphWalkerTestCase(TestCase):
     # TODO: test commit time cutoff
 
     def _handle_shallow_request(self, lines, heads):
-        self._walker.proto.set_output(lines)
+        self._walker.proto.set_output(lines + [None])
         self._walker._handle_shallow_request(heads)
 
     def assertReceived(self, expected):
@@ -537,7 +566,7 @@ class AckGraphWalkerImplTestCase(TestCase):
         self.assertAck(None, 'nak')
 
     def assertNextEquals(self, sha):
-        self.assertEqual(sha, self._impl.next())
+        self.assertEqual(sha, next(self._impl))
 
 
 class SingleAckGraphWalkerImplTestCase(AckGraphWalkerImplTestCase):
@@ -839,8 +868,8 @@ class ServeCommandTests(TestCase):
         commit = make_commit(id=ONE, parents=[], commit_time=111)
         self.backend.repos["/"] = MemoryRepo.init_bare(
             [commit], {"refs/heads/master": commit.id})
-        outf = StringIO()
-        exitcode = self.serve_command(ReceivePackHandler, ["/"], StringIO("0000"), outf)
+        outf = BytesIO()
+        exitcode = self.serve_command(ReceivePackHandler, ["/"], BytesIO("0000"), outf)
         outlines = outf.getvalue().splitlines()
         self.assertEqual(2, len(outlines))
         self.assertEqual("1111111111111111111111111111111111111111 refs/heads/master",

dulwich/tests/test_utils.py

@@ -28,7 +28,7 @@ from dulwich.objects import (
 from dulwich.tests import (
     TestCase,
     )
-from utils import (
+from dulwich.tests.utils import (
     make_object,
     build_commit_graph,
     )
@@ -66,9 +66,9 @@ class BuildCommitGraphTest(TestCase):
         a2 = make_object(Blob, data='aaa2')
         c1, c2 = build_commit_graph(self.store, [[1], [2, 1]],
                                     trees={1: [('a', a1)],
-                                           2: [('a', a2, 0100644)]})
-        self.assertEqual((0100644, a1.id), self.store[c1.tree]['a'])
-        self.assertEqual((0100644, a2.id), self.store[c2.tree]['a'])
+                                           2: [('a', a2, 0o100644)]})
+        self.assertEqual((0o100644, a1.id), self.store[c1.tree]['a'])
+        self.assertEqual((0o100644, a2.id), self.store[c2.tree]['a'])
 
     def test_attrs(self):
         c1, c2 = build_commit_graph(self.store, [[1], [2, 1]],

dulwich/tests/test_walk.py

@@ -47,7 +47,7 @@ from dulwich.walk import (
     _topo_reorder
     )
 from dulwich.tests import TestCase
-from utils import (
+from dulwich.tests.utils import (
     F,
     make_object,
     build_commit_graph,
@@ -88,7 +88,7 @@ class WalkerTest(TestCase):
 
     def make_linear_commits(self, num_commits, **kwargs):
         commit_spec = []
-        for i in xrange(1, num_commits + 1):
+        for i in range(1, num_commits + 1):
             c = [i]
             if i > 1:
                 c.append(i - 1)
@@ -122,7 +122,7 @@ class WalkerTest(TestCase):
         # implementation (in particular the choice of _MAX_EXTRA_COMMITS), but
         # we should at least be able to walk some history in a broken repo.
         del self.store[cs[-1].id]
-        for i in xrange(1, 11):
+        for i in range(1, 11):
             self.assertWalkYields(cs[:i], [cs[0].id], max_entries=i)
         self.assertRaises(MissingCommitError, Walker, self.store, [cs[-1].id])
 

dulwich/tests/test_web.py

@@ -18,7 +18,7 @@
 
 """Tests for the Git HTTP server."""
 
-from cStringIO import StringIO
+from io import BytesIO
 import gzip
 import re
 import os
@@ -90,7 +90,7 @@ class WebTestCase(TestCase):
                                     handlers=self._handlers())
         self._status = None
         self._headers = []
-        self._output = StringIO()
+        self._output = BytesIO()
 
     def _start_response(self, status, headers):
         self._status = status
@@ -122,7 +122,7 @@ class DumbHandlersTestCase(WebTestCase):
         self.assertEqual(HTTP_NOT_FOUND, self._status)
 
     def test_send_file(self):
-        f = StringIO('foobar')
+        f = BytesIO('foobar')
         output = ''.join(send_file(self._req, f, 'some/thing'))
         self.assertEqual('foobar', output)
         self.assertEqual(HTTP_OK, self._status)
@@ -132,7 +132,7 @@ class DumbHandlersTestCase(WebTestCase):
     def test_send_file_buffered(self):
         bufsize = 10240
         xs = 'x' * bufsize
-        f = StringIO(2 * xs)
+        f = BytesIO(2 * xs)
         self.assertEqual([xs, xs],
                           list(send_file(self._req, f, 'some/thing')))
         self.assertEqual(HTTP_OK, self._status)
@@ -263,7 +263,7 @@ class DumbHandlersTestCase(WebTestCase):
             def __init__(self, sha):
                 self.data = TestPackData(sha)
 
-        packs = [TestPack(str(i) * 40) for i in xrange(1, 4)]
+        packs = [TestPack(str(i) * 40) for i in range(1, 4)]
 
         class TestObjectStore(MemoryObjectStore):
             # property must be overridden, can't be assigned
@@ -311,7 +311,7 @@ class SmartHandlersTestCase(WebTestCase):
         self.assertFalse(self._req.cached)
 
     def _run_handle_service_request(self, content_length=None):
-        self._environ['wsgi.input'] = StringIO('foo')
+        self._environ['wsgi.input'] = BytesIO('foo')
         if content_length is not None:
             self._environ['CONTENT_LENGTH'] = content_length
         mat = re.search('.*', '/git-upload-pack')
@@ -342,7 +342,7 @@ class SmartHandlersTestCase(WebTestCase):
         self.assertFalse(self._req.cached)
 
     def test_get_info_refs(self):
-        self._environ['wsgi.input'] = StringIO('foo')
+        self._environ['wsgi.input'] = BytesIO('foo')
         self._environ['QUERY_STRING'] = 'service=git-upload-pack'
 
         mat = re.search('.*', '/git-upload-pack')
@@ -361,16 +361,16 @@ class SmartHandlersTestCase(WebTestCase):
 
 class LengthLimitedFileTestCase(TestCase):
     def test_no_cutoff(self):
-        f = _LengthLimitedFile(StringIO('foobar'), 1024)
+        f = _LengthLimitedFile(BytesIO('foobar'), 1024)
         self.assertEqual('foobar', f.read())
 
     def test_cutoff(self):
-        f = _LengthLimitedFile(StringIO('foobar'), 3)
+        f = _LengthLimitedFile(BytesIO('foobar'), 3)
         self.assertEqual('foo', f.read())
         self.assertEqual('', f.read())
 
     def test_multiple_reads(self):
-        f = _LengthLimitedFile(StringIO('foobar'), 3)
+        f = _LengthLimitedFile(BytesIO('foobar'), 3)
         self.assertEqual('fo', f.read(2))
         self.assertEqual('o', f.read(2))
         self.assertEqual('', f.read())
@@ -454,9 +454,10 @@ class HTTPGitApplicationTestCase(TestCase):
 
 
 class GunzipTestCase(HTTPGitApplicationTestCase):
-    """TestCase for testing the GunzipFilter, ensuring the wsgi.input
+    __doc__ = """TestCase for testing the GunzipFilter, ensuring the wsgi.input
     is correctly decompressed and headers are corrected.
     """
+    example_text = __doc__
 
     def setUp(self):
         super(GunzipTestCase, self).setUp()
@@ -465,18 +466,16 @@ class GunzipTestCase(HTTPGitApplicationTestCase):
         self._environ['REQUEST_METHOD'] = 'POST'
 
     def _get_zstream(self, text):
-        zstream = StringIO()
+        zstream = BytesIO()
         zfile = gzip.GzipFile(fileobj=zstream, mode='w')
         zfile.write(text)
         zfile.close()
-        return zstream
-
-    def test_call(self):
-        self._add_handler(self._app.app)
-        orig = self.__class__.__doc__
-        zstream = self._get_zstream(orig)
         zlength = zstream.tell()
         zstream.seek(0)
+        return zstream, zlength
+
+    def _test_call(self, orig, zstream, zlength):
+        self._add_handler(self._app.app)
         self.assertLess(zlength, len(orig))
         self.assertEqual(self._environ['HTTP_CONTENT_ENCODING'], 'gzip')
         self._environ['CONTENT_LENGTH'] = zlength
@@ -488,3 +487,32 @@ class GunzipTestCase(HTTPGitApplicationTestCase):
         self.assertEqual(orig, buf.read())
         self.assertIs(None, self._environ.get('CONTENT_LENGTH'))
         self.assertNotIn('HTTP_CONTENT_ENCODING', self._environ)
+
+    def test_call(self):
+        self._test_call(
+            self.example_text,
+            *self._get_zstream(self.example_text)
+        )
+
+    def test_call_no_seek(self):
+        """
+        This ensures that the gunzipping code doesn't require any methods on
+        'wsgi.input' except for '.read()'.  (In particular, it shouldn't
+        require '.seek()'. See https://github.com/jelmer/dulwich/issues/140.)
+        """
+        class MinimalistWSGIInputStream(object):
+            def __init__(self, data):
+                self.data = data
+                self.pos = 0
+
+            def read(self, howmuch):
+                start = self.pos
+                end = self.pos + howmuch
+                if start >= len(self.data):
+                    return ''
+                self.pos = end
+                return self.data[start:end]
+
+        zstream, zlength = self._get_zstream(self.example_text)
+        self._test_call(self.example_text,
+            MinimalistWSGIInputStream(zstream.read()), zlength)

dulwich/tests/utils.py

@@ -51,7 +51,7 @@ from dulwich.tests import (
     )
 
 # Plain files are very frequently used in tests, so let the mode be very short.
-F = 0100644  # Shorthand mode for Files.
+F = 0o100644  # Shorthand mode for Files.
 
 
 def open_repo(name):
@@ -229,7 +229,7 @@ def build_pack(f, objects_spec, store=None):
         crc32s[i] = crc32
 
     expected = []
-    for i in xrange(num_objects):
+    for i in range(num_objects):
         type_num, data, sha = full_objects[i]
         assert len(sha) == 20
         expected.append((offsets[i], type_num, data, sha, crc32s[i]))
@@ -280,7 +280,7 @@ def build_commit_graph(object_store, commit_spec, trees=None, attrs=None):
         commit_num = commit[0]
         try:
             parent_ids = [nums[pn] for pn in commit[1:]]
-        except KeyError, e:
+        except KeyError as e:
             missing_parent, = e.args
             raise ValueError('Unknown parent %i' % missing_parent)
 

dulwich/walk.py

@@ -303,7 +303,7 @@ class Walker(object):
     def _next(self):
         max_entries = self.max_entries
         while max_entries is None or self._num_entries < max_entries:
-            entry = self._queue.next()
+            entry = next(self._queue)
             if entry is not None:
                 self._out_queue.append(entry)
             if entry is None or len(self._out_queue) > _MAX_EXTRA_COMMITS:

dulwich/web.py

@@ -19,7 +19,9 @@
 
 """HTTP server for dulwich that implements the git smart HTTP protocol."""
 
-from cStringIO import StringIO
+from io import BytesIO
+import shutil
+import tempfile
 import gzip
 import os
 import re
@@ -168,7 +170,7 @@ def get_info_refs(req, backend, mat):
             return
         req.nocache()
         write = req.respond(HTTP_OK, 'application/x-%s-advertisement' % service)
-        proto = ReceivableProtocol(StringIO().read, write)
+        proto = ReceivableProtocol(BytesIO().read, write)
         handler = handler_cls(backend, [url_prefix(mat)], proto,
                               http_req=req, advertise_refs=True)
         handler.proto.write_pkt_line('# service=%s\n' % service)
@@ -358,11 +360,22 @@ class GunzipFilter(object):
 
     def __call__(self, environ, start_response):
         if environ.get('HTTP_CONTENT_ENCODING', '') == 'gzip':
-            environ.pop('HTTP_CONTENT_ENCODING')
+            if hasattr(environ['wsgi.input'], 'seek'):
+                wsgi_input = environ['wsgi.input']
+            else:
+                # The gzip implementation in the standard library of Python 2.x
+                # requires the '.seek()' and '.tell()' methods to be available
+                # on the input stream.  Read the data into a temporary file to
+                # work around this limitation.
+                wsgi_input = tempfile.SpooledTemporaryFile(16 * 1024 * 1024)
+                shutil.copyfileobj(environ['wsgi.input'], wsgi_input)
+                wsgi_input.seek(0)
+
+            environ['wsgi.input'] = gzip.GzipFile(filename=None, fileobj=wsgi_input, mode='r')
+            del environ['HTTP_CONTENT_ENCODING']
             if 'CONTENT_LENGTH' in environ:
                 del environ['CONTENT_LENGTH']
-            environ['wsgi.input'] = gzip.GzipFile(filename=None,
-                fileobj=environ['wsgi.input'], mode='r')
+
         return self.app(environ, start_response)
 
 

examples/clone.py

@@ -13,7 +13,7 @@ opts, args = getopt(sys.argv, "", [])
 opts = dict(opts)
 
 if len(args) < 2:
-    print "usage: %s host:path path" % (args[0], )
+    print("usage: %s host:path path" % (args[0], ))
     sys.exit(1)
 
 # Connect to the remote repository

examples/config.py

@@ -9,5 +9,5 @@ from dulwich.repo import Repo
 repo = Repo(".")
 config = repo.get_config()
 
-print config.get("core", "filemode")
-print config.get(("remote", "origin"), "url")
+print(config.get("core", "filemode"))
+print(config.get(("remote", "origin"), "url"))

examples/latest_change.py

@@ -6,16 +6,16 @@ import time
 from dulwich.repo import Repo
 
 if len(sys.argv) < 2:
-    print "usage: %s filename" % (sys.argv[0], )
+    print("usage: %s filename" % (sys.argv[0], ))
     sys.exit(1)
 
 r = Repo(".")
 
 w = r.get_walker(paths=[sys.argv[1]], max_entries=1)
 try:
-    c = iter(w).next().commit
+    c = next(iter(w)).commit
 except StopIteration:
-    print "No file %s anywhere in history." % sys.argv[1]
+    print("No file %s anywhere in history." % sys.argv[1])
 else:
-    print "%s was last changed at %s by %s (commit %s)" % (
-        sys.argv[1], c.author, time.ctime(c.author_time), c.id)
+    print("%s was last changed at %s by %s (commit %s)" % (
+        sys.argv[1], c.author, time.ctime(c.author_time), c.id))

setup.py

@@ -10,7 +10,7 @@ except ImportError:
     has_setuptools = False
 from distutils.core import Distribution
 
-dulwich_version_string = '0.9.5'
+dulwich_version_string = '0.9.6'
 
 include_dirs = []
 # Windows MSVC support