Imported Upstream version 0.14.0

.travis.yml

@@ -5,13 +5,13 @@ env:
 matrix:
   include:
     - python: "2.7"
-      env: TEST_REQUIRE="gevent geventhttpclient fastimport"
+      env: TEST_REQUIRE="gevent greenlet geventhttpclient fastimport"
     - python: "pypy"
       env: TEST_REQUIRE="fastimport"
     - python: "3.4"
-      env: TEST_REQUIRE="fastimport"
+      env: TEST_REQUIRE="gevent greenlet geventhttpclient fastimport"
     - python: "3.5"
-      env: TEST_REQUIRE="fastimport"
+      env: TEST_REQUIRE="gevent greenlet geventhttpclient fastimport"
 cache:
   directories:
     - $HOME/.cache/pip

MANIFEST.in

@@ -1,6 +1,7 @@
 include NEWS
 include AUTHORS
 include README.md
+include README.swift
 include Makefile
 include COPYING
 include HACKING
@@ -16,3 +17,4 @@ include dulwich.cfg
 include appveyor.yml
 include .testr.conf
 include .travis.yml
+include relicensing-apachev2.txt

NEWS

@@ -1,3 +1,22 @@
+0.14.0	2016-07-03
+
+ BUG FIXES
+
+  * Fix ShaFile.id after modification of a copied ShaFile.
+    (Félix Mattrat, Jelmer Vernooij)
+
+  * Support removing refs from porcelain.push.
+    (Jelmer Vernooij, #437)
+
+  * Stop magic protocol ref `capabilities^{}` from leaking out
+    to clients. (Jelmer Vernooij, #254)
+
+ IMPROVEMENTS
+
+  * Add `dulwich.config.parse_submodules` function.
+
+  * Add `RefsContainer.follow` method. (#438)
+
 0.13.0	2016-04-24
 
  IMPROVEMENTS

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: dulwich
-Version: 0.13.0
+Version: 0.14.0
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij

README.md

@@ -22,7 +22,7 @@ If you don't want to install the C bindings, specify the --pure argument to setu
 
     $ python setup.py --pure install
 
-or if you are installing from pip:
+or if you are installing from pip::
 
     $ pip install dulwich --global-option="--pure"
 

README.swift

@@ -0,0 +1,133 @@
+Openstack Swift as backend for Dulwich
+======================================
+Fabien Boucher <fabien.boucher@enovance.com>
+
+The module dulwich/contrib/swift.py implements dulwich.repo.BaseRepo
+in order to being compatible with Openstack Swift.
+We can then use Dulwich as server (Git server) and instead of using
+a regular POSIX file system to store repository objects we use the
+object storage Swift via its own API.
+
+c Git client <---> Dulwich server <---> Openstack Swift API
+
+This implementation is still a work in progress and we can say that
+is a Beta version so you need to be prepared to find bugs.
+
+Configuration file
+------------------
+
+We need to provide some configuration values in order to let Dulwich
+talk and authenticate against Swift. The following config file must
+be used as template:
+
+    [swift]
+    # Authentication URL (Keystone or Swift)
+    auth_url = http://127.0.0.1:5000/v2.0
+    # Authentication version to use
+    auth_ver = 2
+    # The tenant and username separated by a semicolon
+    username = admin;admin
+    # The user password
+    password = pass
+    # The Object storage region to use (auth v2) (Default RegionOne)
+    region_name = RegionOne
+    # The Object storage endpoint URL to use (auth v2) (Default internalURL)
+    endpoint_type = internalURL
+    # Concurrency to use for parallel tasks (Default 10)
+    concurrency = 10
+    # Size of the HTTP pool (Default 10)
+    http_pool_length = 10
+    # Timeout delay for HTTP connections (Default 20)
+    http_timeout = 20
+    # Chunk size to read from pack (Bytes) (Default 12228)
+    chunk_length = 12228
+    # Cache size (MBytes) (Default 20)
+    cache_length = 20
+
+
+Note that for now we use the same tenant to perform the requests
+against Swift. Therefor there is only one Swift account used
+for storing repositories. Each repository will be contained in
+a Swift container.
+
+How to start unittest
+---------------------
+
+There is no need to have a Swift cluster running to run the unitests.
+Just run the following command in the Dulwich source directory:
+
+    $ PYTHONPATH=. python -m dulwich.contrib.test_swift
+
+How to start functional tests
+-----------------------------
+
+We provide some basic tests to perform smoke tests against a real Swift
+cluster. To run those functional tests you need a properly configured
+configuration file. The tests can be run as follow:
+
+    $ DULWICH_SWIFT_CFG=/etc/swift-dul.conf PYTHONPATH=. python -m dulwich.contrib.test_swift_smoke
+
+How to install
+--------------
+
+Install the Dulwich library via the setup.py. The dependencies will be
+automatically retrieved from pypi:
+
+    $ python ./setup.py install
+
+How to run the server
+---------------------
+
+Start the server using the following command:
+
+    $ python -m dulwich.contrib.swift daemon -c /etc/swift-dul.conf -l 127.0.0.1
+
+Note that a lot of request will be performed against the Swift
+cluster so it is better to start the Dulwich server as close
+as possible of the Swift proxy. The best solution is to run
+the server on the Swift proxy node to reduce the latency.
+
+How to use
+----------
+
+Once you have validated that the functional tests is working as expected and
+the server is running we can init a bare repository. Run this
+command with the name of the repository to create:
+
+    $ python -m dulwich.contrib.swift init -c /etc/swift-dul.conf edeploy
+
+The repository name will be the container that will contain all the Git
+objects for the repository. Then standard c Git client can be used to
+perform operations againt this repository.
+
+As an example we can clone the previously empty bare repository:
+
+    $ git clone git://localhost/edeploy
+
+Then push an existing project in it:
+
+    $ git clone https://github.com/enovance/edeploy.git edeployclone
+    $ cd edeployclone
+    $ git remote add alt git://localhost/edeploy
+    $ git push alt master
+    $ git ls-remote alt
+    9dc50a9a9bff1e232a74e365707f22a62492183e        HEAD
+    9dc50a9a9bff1e232a74e365707f22a62492183e        refs/heads/master
+
+The other Git commands can be used the way you do usually against
+a regular repository.
+
+Note the daemon subcommands starts a Git server listening for the
+Git protocol. Therefor there is no authentication or encryption
+at all between the cGIT client and the GIT server (Dulwich).
+
+Note on the .info file for pack object
+--------------------------------------
+
+The Swift interface of Dulwich relies only on the pack format
+to store Git objects. Instead of using only an index (pack-sha.idx)
+along with the pack, we add a second file (pack-sha.info). This file
+is automatically created when a client pushes some references on the
+repository. The purpose of this file is to speed up pack creation
+server side when a client fetches some references. Currently this
+.info format is not optimized and may change in future.

docs/protocol.txt

@@ -24,7 +24,7 @@ remote program.
 A common way of sending a unit of information is a pkt_line. This is a 4 byte
 size as human encoded hex (i.e. totally underusing the 4 bytes...) that tells
 you the size of the payload, followed by the payload. The size includes the 4
-byes used by the size itself.
+bytes used by the size itself.
 
     0009ABCD\n
 

docs/tutorial/porcelain.txt

@@ -30,5 +30,5 @@ Commit changes
 
   >>> r = porcelain.init("testrepo")
   >>> open("testrepo/testfile", "w").write("data")
-  >>> porcelain.add(r, "testrepo/testfile")
-  >>> porcelain.commit(r, "A sample commit")
+  >>> porcelain.add(r, "testfile")
+  >>> porcelain.commit(r, b"A sample commit")

dulwich.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: dulwich
-Version: 0.13.0
+Version: 0.14.0
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij

dulwich.egg-info/SOURCES.txt

@@ -7,9 +7,11 @@ MANIFEST.in
 Makefile
 NEWS
 README.md
+README.swift
 TODO
 appveyor.yml
 dulwich.cfg
+relicensing-apachev2.txt
 setup.cfg
 setup.py
 tox.ini

dulwich/__init__.py

@@ -21,4 +21,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 13, 0)
+__version__ = (0, 14, 0)

dulwich/client.py

@@ -70,6 +70,7 @@ from dulwich.protocol import (
     CAPABILITY_REPORT_STATUS,
     CAPABILITY_SIDE_BAND_64K,
     CAPABILITY_THIN_PACK,
+    CAPABILITIES_REF,
     COMMAND_DONE,
     COMMAND_HAVE,
     COMMAND_WANT,
@@ -175,6 +176,8 @@ def read_pkt_refs(proto):
 
     if len(refs) == 0:
         return None, set([])
+    if refs == {CAPABILITIES_REF: ZERO_SHA}:
+        refs = {}
     return refs, set(server_capabilities)
 
 
@@ -218,6 +221,8 @@ class GitClient(object):
         :raises SendPackError: if server rejects the pack data
         :raises UpdateRefsError: if the server supports report-status
                                  and rejects ref updates
+        :return: new_refs dictionary containing the changes that were made
+            {refname: new_ref}, including deleted refs.
         """
         raise NotImplementedError(self.send_pack)
 
@@ -349,8 +354,16 @@ class GitClient(object):
 
         all_refs = set(new_refs.keys()).union(set(old_refs.keys()))
         for refname in all_refs:
+            if not isinstance(refname, bytes):
+                raise TypeError('refname is not a bytestring: %r' % refname)
             old_sha1 = old_refs.get(refname, ZERO_SHA)
+            if not isinstance(old_sha1, bytes):
+                raise TypeError('old sha1 for %s is not a bytestring: %r' %
+                        (refname, old_sha1))
             new_sha1 = new_refs.get(refname, ZERO_SHA)
+            if not isinstance(new_sha1, bytes):
+                raise TypeError('old sha1 for %s is not a bytestring %r' %
+                        (refname, new_sha1))
 
             if old_sha1 != new_sha1:
                 if sent_capabilities:
@@ -489,6 +502,8 @@ class TraditionalGitClient(GitClient):
         :raises SendPackError: if server rejects the pack data
         :raises UpdateRefsError: if the server supports report-status
                                  and rejects ref updates
+        :return: new_refs dictionary containing the changes that were made
+            {refname: new_ref}, including deleted refs.
         """
         proto, unused_can_read = self._connect(b'receive-pack', path)
         with proto:
@@ -761,7 +776,11 @@ class LocalGitClient(GitClient):
         :raises SendPackError: if server rejects the pack data
         :raises UpdateRefsError: if the server supports report-status
                                  and rejects ref updates
+        :return: new_refs dictionary containing the changes that were made
+            {refname: new_ref}, including deleted refs.
         """
+        if not progress:
+            progress = lambda x: None
         from dulwich.repo import Repo
 
         with closing(Repo(path)) as target:
@@ -770,20 +789,23 @@ class LocalGitClient(GitClient):
 
             have = [sha1 for sha1 in old_refs.values() if sha1 != ZERO_SHA]
             want = []
-            all_refs = set(new_refs.keys()).union(set(old_refs.keys()))
-            for refname in all_refs:
-                old_sha1 = old_refs.get(refname, ZERO_SHA)
-                new_sha1 = new_refs.get(refname, ZERO_SHA)
-                if new_sha1 not in have and new_sha1 != ZERO_SHA:
+            for refname, new_sha1 in new_refs.items():
+                if new_sha1 not in have and not new_sha1 in want and new_sha1 != ZERO_SHA:
                     want.append(new_sha1)
 
-            if not want and old_refs == new_refs:
+            if not want and set(new_refs.items()).issubset(set(old_refs.items())):
                 return new_refs
 
             target.object_store.add_objects(generate_pack_contents(have, want))
 
-            for name, sha in new_refs.items():
-                target.refs[name] = sha
+            for refname, new_sha1 in new_refs.items():
+                old_sha1 = old_refs.get(refname, ZERO_SHA)
+                if new_sha1 != ZERO_SHA:
+                    if not target.refs.set_if_equals(refname, old_sha1, new_sha1):
+                        progress('unable to set %s to %s' % (refname, new_sha1))
+                else:
+                    if not target.refs.remove_if_equals(refname, old_sha1):
+                        progress('unable to remove %s' % refname)
 
         return new_refs
 
@@ -1034,6 +1056,8 @@ class HttpGitClient(GitClient):
         :raises SendPackError: if server rejects the pack data
         :raises UpdateRefsError: if the server supports report-status
                                  and rejects ref updates
+        :return: new_refs dictionary containing the changes that were made
+            {refname: new_ref}, including deleted refs.
         """
         url = self._get_url(path)
         old_refs, server_capabilities = self._discover_references(
@@ -1045,6 +1069,7 @@ class HttpGitClient(GitClient):
 
         new_refs = determine_wants(dict(old_refs))
         if new_refs is None:
+            # Determine wants function is aborting the push.
             return old_refs
         if self.dumb:
             raise NotImplementedError(self.fetch_pack)

dulwich/config.py

@@ -219,14 +219,6 @@ def _parse_string(value):
     return bytes(ret)
 
 
-def _unescape_value(value):
-    """Unescape a value."""
-    ret = bytearray()
-    i = 0
-
-    return ret
-
-
 def _escape_value(value):
     """Escape a value."""
     return value.replace(b"\\", b"\\\\").replace(b"\n", b"\\n").replace(b"\t", b"\\t").replace(b"\"", b"\\\"")
@@ -380,12 +372,19 @@ class StackedConfig(Config):
     def default_backends(cls):
         """Retrieve the default configuration.
 
-        This will look in the users' home directory and the system
-        configuration.
+        See git-config(1) for details on the files searched.
         """
         paths = []
         paths.append(os.path.expanduser("~/.gitconfig"))
-        paths.append("/etc/gitconfig")
+
+        xdg_config_home = os.environ.get(
+            "XDG_CONFIG_HOME", os.path.expanduser("~/.config/"),
+        )
+        paths.append(os.path.join(xdg_config_home, "git", "config"))
+
+        if "GIT_CONFIG_NOSYSTEM" not in os.environ:
+            paths.append("/etc/gitconfig")
+
         backends = []
         for path in paths:
             try:
@@ -410,3 +409,18 @@ class StackedConfig(Config):
         if self.writable is None:
             raise NotImplementedError(self.set)
         return self.writable.set(section, name, value)
+
+
+def parse_submodules(config):
+    """Parse a gitmodules GitConfig file, returning submodules.
+
+   :param config: A `ConfigFile`
+   :return: list of tuples (submodule path, url, name),
+       where name is quoted part of the section's name.
+    """
+    for section in config.keys():
+        section_kind, section_name = section
+        if section_kind == b'submodule':
+            sm_path = config.get(section, b'path')
+            sm_url = config.get(section, b'url')
+            yield (sm_path, sm_url, section_name)

dulwich/contrib/paramiko_vendor.py

@@ -30,7 +30,6 @@ This implementation is experimental and does not have any tests.
 
 import paramiko
 import paramiko.client
-import subprocess
 import threading
 
 class _ParamikoWrapper(object):

dulwich/index.py

@@ -389,8 +389,12 @@ def changes_from_tree(names, lookup_entry, object_store, tree,
 
     # Mention added files
     for name in other_names:
-        (other_sha, other_mode) = lookup_entry(name)
-        yield ((None, name), (None, other_mode), (None, other_sha))
+        try:
+            (other_sha, other_mode) = lookup_entry(name)
+        except KeyError:
+            pass
+        else:
+            yield ((None, name), (None, other_mode), (None, other_sha))
 
 
 def index_entry_from_stat(stat_val, hex_sha, flags, mode=None):

dulwich/objects.py

@@ -141,11 +141,9 @@ def serializable_property(name, docstring=None):
     """A property that helps tracking whether serialization is necessary.
     """
     def set(obj, value):
-        obj._ensure_parsed()
         setattr(obj, "_"+name, value)
         obj._needs_serialization = True
     def get(obj):
-        obj._ensure_parsed()
         return getattr(obj, "_"+name)
     return property(get, set, doc=docstring)
 
@@ -218,8 +216,7 @@ class FixedSha(object):
 class ShaFile(object):
     """A git SHA file."""
 
-    __slots__ = ('_needs_parsing', '_chunked_text', '_sha',
-                 '_needs_serialization')
+    __slots__ = ('_chunked_text', '_sha', '_needs_serialization')
 
     @staticmethod
     def _parse_legacy_object_header(magic, f):
@@ -272,9 +269,8 @@ class ShaFile(object):
 
         :return: List of strings, not necessarily one per line
         """
-        if self._needs_parsing:
-            self._ensure_parsed()
-        elif self._needs_serialization:
+        if self._needs_serialization:
+            self._sha = None
             self._chunked_text = self._serialize()
             self._needs_serialization = False
         return self._chunked_text
@@ -298,13 +294,6 @@ class ShaFile(object):
         """Return a string representing this object, fit for display."""
         return self.as_raw_string()
 
-    def _ensure_parsed(self):
-        if self._needs_parsing:
-            if not self._chunked_text:
-                raise AssertionError("ShaFile needs chunked text")
-            self._deserialize(self._chunked_text)
-            self._needs_parsing = False
-
     def set_raw_string(self, text, sha=None):
         """Set the contents of this object from a serialized string."""
         if not isinstance(text, bytes):
@@ -319,7 +308,6 @@ class ShaFile(object):
             self._sha = None
         else:
             self._sha = FixedSha(sha)
-        self._needs_parsing = False
         self._needs_serialization = False
 
     @staticmethod
@@ -365,7 +353,6 @@ class ShaFile(object):
         """Don't call this directly"""
         self._sha = None
         self._chunked_text = []
-        self._needs_parsing = False
         self._needs_serialization = True
 
     def _deserialize(self, chunks):
@@ -463,13 +450,6 @@ class ShaFile(object):
             ret += len(chunk)
         return ret
 
-    def _make_sha(self):
-        ret = sha1()
-        ret.update(self._header())
-        for chunk in self.as_raw_chunks():
-            ret.update(chunk)
-        return ret
-
     def sha(self):
         """The SHA1 object that is the name of this object."""
         if self._sha is None or self._needs_serialization:
@@ -546,7 +526,6 @@ class Blob(ShaFile):
     def __init__(self):
         super(Blob, self).__init__()
         self._chunked_text = []
-        self._needs_parsing = False
         self._needs_serialization = False
 
     def _get_data(self):
@@ -559,15 +538,12 @@ class Blob(ShaFile):
                     "The text contained within the blob object.")
 
     def _get_chunked(self):
-        self._ensure_parsed()
         return self._chunked_text
 
     def _set_chunked(self, chunks):
         self._chunked_text = chunks
 
     def _serialize(self):
-        if not self._chunked_text:
-            self._ensure_parsed()
         return self._chunked_text
 
     def _deserialize(self, chunks):
@@ -746,11 +722,9 @@ class Tag(ShaFile):
 
         :return: tuple of (object class, sha).
         """
-        self._ensure_parsed()
         return (self._object_class, self._object_sha)
 
     def _set_object(self, value):
-        self._ensure_parsed()
         (self._object_class, self._object_sha) = value
         self._needs_serialization = True
 
@@ -871,11 +845,9 @@ class Tree(ShaFile):
         return tree
 
     def __contains__(self, name):
-        self._ensure_parsed()
         return name in self._entries
 
     def __getitem__(self, name):
-        self._ensure_parsed()
         return self._entries[name]
 
     def __setitem__(self, name, value):
@@ -887,21 +859,17 @@ class Tree(ShaFile):
             a string.
         """
         mode, hexsha = value
-        self._ensure_parsed()
         self._entries[name] = (mode, hexsha)
         self._needs_serialization = True
 
     def __delitem__(self, name):
-        self._ensure_parsed()
         del self._entries[name]
         self._needs_serialization = True
 
     def __len__(self):
-        self._ensure_parsed()
         return len(self._entries)
 
     def __iter__(self):
-        self._ensure_parsed()
         return iter(self._entries)
 
     def add(self, name, mode, hexsha):
@@ -917,7 +885,6 @@ class Tree(ShaFile):
             warnings.warn(
                 "Please use Tree.add(name, mode, hexsha)",
                 category=DeprecationWarning, stacklevel=2)
-        self._ensure_parsed()
         self._entries[name] = mode, hexsha
         self._needs_serialization = True
 
@@ -928,7 +895,6 @@ class Tree(ShaFile):
             order.
         :return: Iterator over (name, mode, sha) tuples
         """
-        self._ensure_parsed()
         return sorted_tree_items(self._entries, name_order)
 
     def items(self):
@@ -1216,12 +1182,10 @@ class Commit(ShaFile):
 
     def _get_parents(self):
         """Return a list of parents of this commit."""
-        self._ensure_parsed()
         return self._parents
 
     def _set_parents(self, value):
         """Set a list of parents of this commit."""
-        self._ensure_parsed()
         self._needs_serialization = True
         self._parents = value
 
@@ -1230,7 +1194,6 @@ class Commit(ShaFile):
 
     def _get_extra(self):
         """Return extra settings of this commit."""
-        self._ensure_parsed()
         return self._extra
 
     extra = property(_get_extra,

dulwich/objectspec.py

@@ -80,7 +80,7 @@ def parse_reftuple(lh_container, rh_container, refspec):
         (lh, rh) = refspec.split(b":")
     else:
         lh = rh = refspec
-    if rh == b"":
+    if lh == b"":
         lh = None
     else:
         lh = parse_ref(lh_container, lh)

dulwich/porcelain.py

@@ -82,7 +82,10 @@ from dulwich.pack import (
     write_pack_objects,
     )
 from dulwich.patch import write_tree_diff
-from dulwich.protocol import Protocol
+from dulwich.protocol import (
+    Protocol,
+    ZERO_SHA,
+    )
 from dulwich.repo import (BaseRepo, Repo)
 from dulwich.server import (
     FileSystemBackend,
@@ -489,9 +492,9 @@ def tag_create(repo, tag, author=None, message=None, annotated=False,
             tag_obj.message = message
             tag_obj.name = tag
             tag_obj.object = (type(object), object.id)
-            tag_obj.tag_time = tag_time
             if tag_time is None:
                 tag_time = int(time.time())
+            tag_obj.tag_time = tag_time
             if tag_timezone is None:
                 # TODO(jelmer) Use current user timezone rather than UTC
                 tag_timezone = 0
@@ -577,15 +580,16 @@ def push(repo, remote_location, refspecs=None,
 
         def update_refs(refs):
             selected_refs.extend(parse_reftuples(r.refs, refs, refspecs))
+            new_refs = {}
             # TODO: Handle selected_refs == {None: None}
             for (lh, rh, force) in selected_refs:
                 if lh is None:
-                    del refs[rh]
+                    new_refs[rh] = ZERO_SHA
                 else:
-                    refs[rh] = r.refs[lh]
-            return refs
+                    new_refs[rh] = r.refs[lh]
+            return new_refs
 
-        err_encoding = getattr(errstream, 'encoding', 'utf-8')
+        err_encoding = getattr(errstream, 'encoding', None) or 'utf-8'
         remote_location_bytes = remote_location.encode(err_encoding)
         try:
             client.send_pack(path, update_refs,

dulwich/protocol.py

@@ -60,6 +60,10 @@ CAPABILITY_SIDE_BAND_64K = b'side-band-64k'
 CAPABILITY_THIN_PACK = b'thin-pack'
 CAPABILITY_AGENT = b'agent'
 
+# Magic ref that is used to attach capabilities to when
+# there are no refs. Should always be ste to ZERO_SHA.
+CAPABILITIES_REF = b'capabilities^{}'
+
 
 def agent_string():
     return ('dulwich/%d.%d.%d' % dulwich.__version__).encode('ascii')

dulwich/refs.py

@@ -32,6 +32,7 @@ from dulwich.errors import (
 from dulwich.objects import (
     git_line,
     valid_hexsha,
+    ZERO_SHA,
     )
 from dulwich.file import (
     GitFile,
@@ -196,23 +197,35 @@ class RefsContainer(object):
         """
         raise NotImplementedError(self.read_loose_ref)
 
-    def _follow(self, name):
+    def follow(self, name):
         """Follow a reference name.
 
-        :return: a tuple of (refname, sha), where refname is the name of the
-            last reference in the symbolic reference chain
+        :return: a tuple of (refnames, sha), wheres refnames are the names of
+            references in the chain
         """
         contents = SYMREF + name
         depth = 0
+        refnames = []
         while contents.startswith(SYMREF):
             refname = contents[len(SYMREF):]
+            refnames.append(refname)
             contents = self.read_ref(refname)
             if not contents:
                 break
             depth += 1
             if depth > 5:
                 raise KeyError(name)
-        return refname, contents
+        return refnames, contents
+
+    def _follow(self, name):
+        import warnings
+        warnings.warn(
+            "RefsContainer._follow is deprecated. Use RefsContainer.follow instead.",
+            DeprecationWarning)
+        refnames, contents = self.follow(name)
+        if not refnames:
+            return (None, contents)
+        return (refnames[-1], contents)
 
     def __contains__(self, refname):
         if self.read_ref(refname):
@@ -224,7 +237,7 @@ class RefsContainer(object):
 
         This method follows all symbolic references.
         """
-        _, sha = self._follow(name)
+        _, sha = self.follow(name)
         if sha is None:
             raise KeyError(name)
         return sha
@@ -315,11 +328,12 @@ class DictRefsContainer(RefsContainer):
         self._refs[name] = SYMREF + other
 
     def set_if_equals(self, name, old_ref, new_ref):
-        if old_ref is not None and self._refs.get(name, None) != old_ref:
+        if old_ref is not None and self._refs.get(name, ZERO_SHA) != old_ref:
             return False
-        realname, _ = self._follow(name)
-        self._check_refname(realname)
-        self._refs[realname] = new_ref
+        realnames, _ = self.follow(name)
+        for realname in realnames:
+            self._check_refname(realname)
+            self._refs[realname] = new_ref
         return True
 
     def add_if_new(self, name, ref):
@@ -329,9 +343,12 @@ class DictRefsContainer(RefsContainer):
         return True
 
     def remove_if_equals(self, name, old_ref):
-        if old_ref is not None and self._refs.get(name, None) != old_ref:
+        if old_ref is not None and self._refs.get(name, ZERO_SHA) != old_ref:
             return False
-        del self._refs[name]
+        try:
+            del self._refs[name]
+        except KeyError:
+            pass
         return True
 
     def get_peeled(self, name):
@@ -567,8 +584,9 @@ class DiskRefsContainer(RefsContainer):
         """
         self._check_refname(name)
         try:
-            realname, _ = self._follow(name)
-        except KeyError:
+            realnames, _ = self.follow(name)
+            realname = realnames[-1]
+        except (KeyError, IndexError):
             realname = name
         filename = self.refpath(realname)
         ensure_dir_exists(os.path.dirname(filename))
@@ -578,7 +596,7 @@ class DiskRefsContainer(RefsContainer):
                     # read again while holding the lock
                     orig_ref = self.read_loose_ref(realname)
                     if orig_ref is None:
-                        orig_ref = self.get_packed_refs().get(realname, None)
+                        orig_ref = self.get_packed_refs().get(realname, ZERO_SHA)
                     if orig_ref != old_ref:
                         f.abort()
                         return False
@@ -603,10 +621,11 @@ class DiskRefsContainer(RefsContainer):
         :return: True if the add was successful, False otherwise.
         """
         try:
-            realname, contents = self._follow(name)
+            realnames, contents = self.follow(name)
             if contents is not None:
                 return False
-        except KeyError:
+            realname = realnames[-1]
+        except (KeyError, IndexError):
             realname = name
         self._check_refname(realname)
         filename = self.refpath(realname)
@@ -641,7 +660,7 @@ class DiskRefsContainer(RefsContainer):
             if old_ref is not None:
                 orig_ref = self.read_loose_ref(name)
                 if orig_ref is None:
-                    orig_ref = self.get_packed_refs().get(name, None)
+                    orig_ref = self.get_packed_refs().get(name, ZERO_SHA)
                 if orig_ref != old_ref:
                     return False
             # may only be packed

dulwich/repo.py

@@ -833,9 +833,9 @@ class Repo(BaseRepo):
             pass
 
         # Update target head
-        head, head_sha = self.refs._follow(b'HEAD')
-        if head is not None and head_sha is not None:
-            target.refs.set_symbolic_ref(b'HEAD', head)
+        head_chain, head_sha = self.refs.follow(b'HEAD')
+        if head_chain and head_sha is not None:
+            target.refs.set_symbolic_ref(b'HEAD', head_chain[-1])
             target[b'HEAD'] = head_sha
 
             if not bare:

dulwich/server.py

@@ -69,6 +69,7 @@ from dulwich.pack import (
 from dulwich.protocol import (
     BufferedPktLineWriter,
     capability_agent,
+    CAPABILITIES_REF,
     CAPABILITY_DELETE_REFS,
     CAPABILITY_INCLUDE_TAG,
     CAPABILITY_MULTI_ACK_DETAILED,
@@ -472,7 +473,6 @@ def _all_wants_satisfied(store, haves, wants):
         earliest = min([store[h].commit_time for h in haves])
     else:
         earliest = 0
-    unsatisfied_wants = set()
     for want in wants:
         if not _want_satisfied(store, haves, want, earliest):
             return False
@@ -891,12 +891,12 @@ class ReceivePackHandler(PackHandler):
                           'Attempted to delete refs without delete-refs '
                           'capability.')
                     try:
-                        del self.repo.refs[ref]
+                        self.repo.refs.remove_if_equals(ref, oldsha)
                     except all_exceptions:
                         ref_status = b'failed to delete'
                 else:
                     try:
-                        self.repo.refs[ref] = sha
+                        self.repo.refs.set_if_equals(ref, oldsha, sha)
                     except all_exceptions:
                         ref_status = b'failed to write'
             except KeyError as e:
@@ -932,16 +932,14 @@ class ReceivePackHandler(PackHandler):
         if self.advertise_refs or not self.http_req:
             refs = sorted(self.repo.get_refs().items())
 
-            if refs:
-                self.proto.write_pkt_line(
-                  refs[0][1] + b' ' + refs[0][0] + b'\0' +
-                  self.capability_line() + b'\n')
-                for i in range(1, len(refs)):
-                    ref = refs[i]
-                    self.proto.write_pkt_line(ref[1] + b' ' + ref[0] + b'\n')
-            else:
-                self.proto.write_pkt_line(ZERO_SHA + b" capabilities^{}\0" +
-                    self.capability_line())
+            if not refs:
+                refs = [(CAPABILITIES_REF, ZERO_SHA)]
+            self.proto.write_pkt_line(
+              refs[0][1] + b' ' + refs[0][0] + b'\0' +
+              self.capability_line() + b'\n')
+            for i in range(1, len(refs)):
+                ref = refs[i]
+                self.proto.write_pkt_line(ref[1] + b' ' + ref[0] + b'\n')
 
             self.proto.write_pkt_line(None)
             if self.advertise_refs:

dulwich/tests/compat/test_pack.py

@@ -134,7 +134,6 @@ class TestPack(PackTests):
         # two copy operations in git's binary delta format.
         raise SkipTest('skipping slow, large test')
         with self.get_pack(pack1_sha) as orig_pack:
-            orig_blob = orig_pack[a_sha]
             new_blob = Blob()
             new_blob.data = 'big blob' + ('x' * 2 ** 25)
             new_blob_2 = Blob()

dulwich/tests/test_client.py

@@ -107,7 +107,23 @@ class GitClientTests(TestCase):
     def test_fetch_empty(self):
         self.rin.write(b'0000')
         self.rin.seek(0)
-        self.client.fetch_pack(b'/', lambda heads: [], None, None)
+        def check_heads(heads):
+            self.assertIs(heads, None)
+            return []
+        self.client.fetch_pack(b'/', check_heads, None, None)
+
+    def test_fetch_pack_ignores_magic_ref(self):
+        self.rin.write(
+            b'00000000000000000000000000000000000000000000 capabilities^{}\x00 multi_ack '
+            b'thin-pack side-band side-band-64k ofs-delta shallow no-progress '
+            b'include-tag\n'
+            b'0000')
+        self.rin.seek(0)
+        def check_heads(heads):
+            self.assertEquals({}, heads)
+            return []
+        self.client.fetch_pack(b'bla', check_heads, None, None, None)
+        self.assertEqual(self.rout.getvalue(), b'0000')
 
     def test_fetch_pack_none(self):
         self.rin.write(
@@ -656,6 +672,7 @@ class LocalGitClientTests(TestCase):
         self.assertDictEqual(local.refs.as_dict(), refs)
 
     def send_and_verify(self, branch, local, target):
+        """Send a branch from local to remote repository and verify it worked."""
         client = LocalGitClient()
         ref_name = b"refs/heads/" + branch
         new_refs = client.send_pack(target.path,
@@ -664,9 +681,6 @@ class LocalGitClientTests(TestCase):
 
         self.assertEqual(local.refs[ref_name], new_refs[ref_name])
 
-        for name, sha in new_refs.items():
-            self.assertEqual(new_refs[name], target.refs[name])
-
         obj_local = local.get_object(new_refs[ref_name])
         obj_target = target.get_object(new_refs[ref_name])
         self.assertEqual(obj_local, obj_target)

dulwich/tests/test_config.py

@@ -28,7 +28,7 @@ from dulwich.config import (
     _format_string,
     _escape_value,
     _parse_string,
-    _unescape_value,
+    parse_submodules,
     )
 from dulwich.tests import (
     TestCase,
@@ -292,3 +292,16 @@ class CheckSectionNameTests(TestCase):
         self.assertTrue(_check_section_name(b"foo"))
         self.assertTrue(_check_section_name(b"foo-bar"))
         self.assertTrue(_check_section_name(b"bar.bar"))
+
+
+class SubmodulesTests(TestCase):
+
+    def testSubmodules(self):
+        cf = ConfigFile.from_file(BytesIO(b"""\
+[submodule "core/lib"]
+	path = core/lib
+	url = https://github.com/phhusson/QuasselC.git
+"""))
+        got = list(parse_submodules(cf))
+        self.assertEqual([
+            (b'core/lib', b'https://github.com/phhusson/QuasselC.git', b'core/lib')], got)

dulwich/tests/test_greenthreads.py

@@ -51,14 +51,14 @@ if gevent_support:
 skipmsg = "Gevent library is not installed"
 
 def create_commit(marker=None):
-    blob = Blob.from_string('The blob content %s' % marker)
+    blob = Blob.from_string(b'The blob content ' + marker)
     tree = Tree()
-    tree.add("thefile %s" % marker, 0o100644, blob.id)
+    tree.add(b"thefile " + marker, 0o100644, blob.id)
     cmt = Commit()
     cmt.tree = tree.id
-    cmt.author = cmt.committer = "John Doe <john@doe.net>"
-    cmt.message = "%s" % marker
-    tz = parse_timezone('-0200')[0]
+    cmt.author = cmt.committer = b"John Doe <john@doe.net>"
+    cmt.message = marker
+    tz = parse_timezone(b'-0200')[0]
     cmt.commit_time = cmt.author_time = int(time.time())
     cmt.commit_timezone = cmt.author_timezone = tz
     return cmt, tree, blob
@@ -67,7 +67,7 @@ def create_commit(marker=None):
 def init_store(store, count=1):
     ret = []
     for i in range(0, count):
-        objs = create_commit(marker=i)
+        objs = create_commit(marker=("%d" % i).encode('ascii'))
         for obj in objs:
             ret.append(obj)
             store.add_object(obj)
@@ -127,7 +127,7 @@ class TestGreenThreadsMissingObjectFinder(TestCase):
         self.assertEqual(len(finder.objects_to_send), self.cmt_amount)
 
         finder = GreenThreadsMissingObjectFinder(self.store,
-                                             wants[0:self.cmt_amount/2],
+                                             wants[0:int(self.cmt_amount/2)],
                                              wants)
         # sha_done will contains commit id and sha of blob refered in tree
         self.assertEqual(len(finder.sha_done), (self.cmt_amount/2)*2)

dulwich/tests/test_hooks.py

@@ -20,7 +20,6 @@
 import os
 import stat
 import shutil
-import sys
 import tempfile
 
 from dulwich import errors

dulwich/tests/test_objects.py

@@ -231,7 +231,7 @@ class BlobReadTests(TestCase):
         c = make_commit(id=sha, message=b'foo')
         self.assertTrue(isinstance(c, Commit))
         self.assertEqual(sha, c.id)
-        self.assertNotEqual(sha, c._make_sha())
+        self.assertNotEqual(sha, c.sha())
 
 
 class ShaFileCheckTests(TestCase):
@@ -937,6 +937,20 @@ class TagParseTests(ShaFileCheckTests):
             else:
                 self.assertCheckFails(Tag, text)
 
+    def test_tree_copy_after_update(self):
+        """Check Tree.id is correctly updated when the tree is copied after updated.
+        """
+        shas = []
+        tree = Tree()
+        shas.append(tree.id)
+        tree.add(b'data', 0o644, Blob().id)
+        copied = tree.copy()
+        shas.append(tree.id)
+        shas.append(copied.id)
+
+        self.assertNotIn(shas[0], shas[1:])
+        self.assertEqual(shas[1], shas[2])
+
 
 class CheckTests(TestCase):
 

dulwich/tests/test_objectspec.py

@@ -162,6 +162,16 @@ class ParseReftupleTests(TestCase):
         self.assertEqual((b"refs/heads/foo", b"refs/heads/foo", False),
             parse_reftuple(r, r, b"refs/heads/foo"))
 
+    def test_no_left_ref(self):
+        r = {b"refs/heads/foo": "bla"}
+        self.assertEqual((None, b"refs/heads/foo", False),
+            parse_reftuple(r, r, b":refs/heads/foo"))
+
+    def test_no_right_ref(self):
+        r = {b"refs/heads/foo": "bla"}
+        self.assertEqual((b"refs/heads/foo", None, False),
+            parse_reftuple(r, r, b"refs/heads/foo:"))
+
 
 class ParseReftuplesTests(TestCase):
 

dulwich/tests/test_porcelain.py

@@ -28,6 +28,7 @@ import os
 import shutil
 import tarfile
 import tempfile
+import time
 
 from dulwich import porcelain
 from dulwich.diff_tree import tree_changes
@@ -35,6 +36,7 @@ from dulwich.objects import (
     Blob,
     Tag,
     Tree,
+    ZERO_SHA,
     )
 from dulwich.repo import Repo
 from dulwich.tests import (
@@ -375,6 +377,7 @@ class TagCreateTests(PorcelainTestCase):
         self.assertTrue(isinstance(tag, Tag))
         self.assertEqual(b"foo <foo@bar.com>", tag.tagger)
         self.assertEqual(b"bar", tag.message)
+        self.assertLess(time.time() - tag.tag_time, 5)
 
     def test_unannotated(self):
         c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1],
@@ -456,7 +459,10 @@ class PushTests(PorcelainTestCase):
         self.addCleanup(shutil.rmtree, clone_path)
         target_repo = porcelain.clone(self.repo.path, target=clone_path,
             errstream=errstream)
-        target_repo.close()
+        try:
+            self.assertEqual(target_repo[b'HEAD'], self.repo[b'HEAD'])
+        finally:
+            target_repo.close()
 
         # create a second file to be pushed back to origin
         handle, fullpath = tempfile.mkstemp(dir=clone_path)
@@ -467,7 +473,9 @@ class PushTests(PorcelainTestCase):
 
         # Setup a non-checked out branch in the remote
         refs_path = b"refs/heads/foo"
-        self.repo.refs[refs_path] = self.repo[b'HEAD'].id
+        new_id = self.repo[b'HEAD'].id
+        self.assertNotEqual(new_id, ZERO_SHA)
+        self.repo.refs[refs_path] = new_id
 
         # Push to the remote
         porcelain.push(clone_path, self.repo.path, b"HEAD:" + refs_path, outstream=outstream,
@@ -475,7 +483,12 @@ class PushTests(PorcelainTestCase):
 
         # Check that the target and source
         with closing(Repo(clone_path)) as r_clone:
-            self.assertEqual(r_clone[b'HEAD'].id, self.repo[refs_path].id)
+            self.assertEqual({
+                b'HEAD': new_id,
+                b'refs/heads/foo': r_clone[b'HEAD'].id,
+                b'refs/heads/master': new_id,
+                }, self.repo.get_refs())
+            self.assertEqual(r_clone[b'HEAD'].id, self.repo.refs[refs_path])
 
             # Get the change in the target repo corresponding to the add
             # this will be in the foo branch.
@@ -484,6 +497,38 @@ class PushTests(PorcelainTestCase):
             self.assertEqual(os.path.basename(fullpath),
                 change.new.path.decode('ascii'))
 
+    def test_delete(self):
+        """Basic test of porcelain push, removing a branch.
+        """
+        outstream = BytesIO()
+        errstream = BytesIO()
+
+        porcelain.commit(repo=self.repo.path, message=b'init',
+            author=b'', committer=b'')
+
+        # Setup target repo cloned from temp test repo
+        clone_path = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, clone_path)
+        target_repo = porcelain.clone(self.repo.path, target=clone_path,
+            errstream=errstream)
+        target_repo.close()
+
+        # Setup a non-checked out branch in the remote
+        refs_path = b"refs/heads/foo"
+        new_id = self.repo[b'HEAD'].id
+        self.assertNotEqual(new_id, ZERO_SHA)
+        self.repo.refs[refs_path] = new_id
+
+        # Push to the remote
+        porcelain.push(clone_path, self.repo.path, b":" + refs_path, outstream=outstream,
+            errstream=errstream)
+
+        self.assertEqual({
+            b'HEAD': new_id,
+            b'refs/heads/master': new_id,
+            }, self.repo.get_refs())
+
+
 
 class PullTests(PorcelainTestCase):
 

dulwich/tests/test_refs.py

@@ -29,6 +29,7 @@ from dulwich import errors
 from dulwich.file import (
     GitFile,
     )
+from dulwich.objects import ZERO_SHA
 from dulwich.refs import (
     DictRefsContainer,
     InfoRefsContainer,
@@ -203,6 +204,10 @@ class RefsContainerTests(object):
                                                  nines))
         self.assertEqual(nines, self._refs[b'refs/heads/master'])
 
+        self.assertTrue(self._refs.set_if_equals(
+            b'refs/heads/nonexistant', ZERO_SHA, nines))
+        self.assertEqual(nines, self._refs[b'refs/heads/nonexistant'])
+
     def test_add_if_new(self):
         nines = b'9' * 40
         self.assertFalse(self._refs.add_if_new(b'refs/heads/master', nines))
@@ -259,10 +264,11 @@ class RefsContainerTests(object):
                          self._refs[b'HEAD'])
         self.assertTrue(self._refs.remove_if_equals(
             b'refs/tags/refs-0.2', b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8'))
+        self.assertTrue(self._refs.remove_if_equals(
+            b'refs/tags/refs-0.2', ZERO_SHA))
         self.assertFalse(b'refs/tags/refs-0.2' in self._refs)
 
 
-
 class DictRefsContainerTests(RefsContainerTests, TestCase):
 
     def setUp(self):
@@ -367,13 +373,13 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
         self.assertEqual(nines, refs[b'refs/heads/master'])
 
     def test_follow(self):
-        self.assertEqual((b'refs/heads/master',
+        self.assertEqual(([b'HEAD', b'refs/heads/master'],
                           b'42d06bd4b77fed026b154d16493e5deab78f02ec'),
-                         self._refs._follow(b'HEAD'))
-        self.assertEqual((b'refs/heads/master',
+                         self._refs.follow(b'HEAD'))
+        self.assertEqual(([b'refs/heads/master'],
                           b'42d06bd4b77fed026b154d16493e5deab78f02ec'),
-                         self._refs._follow(b'refs/heads/master'))
-        self.assertRaises(KeyError, self._refs._follow, b'refs/heads/loop')
+                         self._refs.follow(b'refs/heads/master'))
+        self.assertRaises(KeyError, self._refs.follow, b'refs/heads/loop')
 
     def test_delitem(self):
         RefsContainerTests.test_delitem(self)
@@ -441,7 +447,7 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
     def test_non_ascii(self):
         try:
             encoded_ref = u'refs/tags/schön'.encode(sys.getfilesystemencoding())
-        except UnicodeDecodeError:
+        except UnicodeEncodeError:
             raise SkipTest("filesystem encoding doesn't support special character")
         p = os.path.join(self._repo.path, 'refs', 'tags', 'schön')
         with open(p, 'w') as f:

dulwich/tests/utils.py

@@ -23,7 +23,6 @@
 import datetime
 import os
 import shutil
-import sys
 import tempfile
 import time
 import types

relicensing-apachev2.txt

@@ -0,0 +1,23 @@
+At the moment, Dulwich is licensed under the GNU General Public License,
+version 2 or later.
+
+We'd like to relicense Dulwich under the Apache v2 (or later) license, as
+the GPL is problematic for many free software Python projects that are under
+BSD-style licenses. See also https://github.com/jelmer/dulwich/issues/153
+
+For reference, a full copy of the Apachev2 license can be found here:
+https://www.apache.org/licenses/LICENSE-2.0
+
+New contributions to Dulwich should be dual licensed under the GNU GPLv2 (or
+later) and the Apachev2 (or later) licenses.
+
+Contributions made prior were contributed under the GPLv2 (or later) license
+alone. Code written by the following contributors has not (yet) been relicensed
+under dual Apachev2/GPLv2:
+
+Artem Tikhomirov <artem.tikhomirov@syntevo.com>
+Risto Kankkunen <risto.kankkunen@f-secure.com> <risto.kankkunen@iki.fi>
+
+If your name is in this list and you'd be happy to relicense your contribution
+under dual GPLv2/Apachev2, then please send me an e-mail (jelmer@jelmer.uk) or
+a pull request on GitHub removing your name from the list above.

setup.py

@@ -8,7 +8,7 @@ except ImportError:
     from distutils.core import setup, Extension
 from distutils.core import Distribution
 
-dulwich_version_string = '0.13.0'
+dulwich_version_string = '0.14.0'
 
 include_dirs = []
 # Windows MSVC support
@@ -46,15 +46,10 @@ if sys.platform == 'darwin' and os.path.exists('/usr/bin/xcodebuild'):
         if l.startswith('Xcode') and int(l.split()[1].split('.')[0]) >= 4:
             os.environ['ARCHFLAGS'] = ''
 
-if sys.version_info[0] == 2:
-    tests_require = ['fastimport']
-    if not '__pypy__' in sys.modules and not sys.platform == 'win32':
-        tests_require.extend([
-            'gevent', 'geventhttpclient', 'mock', 'setuptools>=17.1'])
-else:
-    # fastimport, gevent, geventhttpclient are not available for PY3
-    # mock only used for test_swift, which requires gevent/geventhttpclient
-    tests_require = []
+tests_require = ['fastimport']
+if not '__pypy__' in sys.modules and not sys.platform == 'win32':
+    tests_require.extend([
+        'gevent', 'geventhttpclient', 'mock', 'setuptools>=17.1'])
 
 if sys.version_info[0] > 2 and sys.platform == 'win32':
     # C Modules don't build for python3 windows, and prevent tests from running