Imported Upstream version 0.14.1

NEWS

@@ -1,3 +1,13 @@
+0.14.1	2016-07-05
+
+ BUG FIXES
+
+  * Fix regression removing untouched refs when pushing over SSH.
+    (Jelmer Vernooij #441)
+
+  * Skip Python3 tests for SWIFT contrib module, as it has not yet
+    been ported.
+
 0.14.0	2016-07-03
 
  BUG FIXES

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: dulwich
-Version: 0.14.0
+Version: 0.14.1
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij

dulwich.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: dulwich
-Version: 0.14.0
+Version: 0.14.1
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij

dulwich/__init__.py

@@ -21,4 +21,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 14, 0)
+__version__ = (0, 14, 1)

dulwich/client.py

@@ -345,15 +345,14 @@ class GitClient(object):
         :param proto: Protocol object to read from
         :param capabilities: List of negotiated capabilities
         :param old_refs: Old refs, as received from the server
-        :param new_refs: New refs
+        :param new_refs: Refs to change
         :return: (have, want) tuple
         """
         want = []
         have = [x for x in old_refs.values() if not x == ZERO_SHA]
         sent_capabilities = False
 
-        all_refs = set(new_refs.keys()).union(set(old_refs.keys()))
-        for refname in all_refs:
+        for refname in new_refs:
             if not isinstance(refname, bytes):
                 raise TypeError('refname is not a bytestring: %r' % refname)
             old_sha1 = old_refs.get(refname, ZERO_SHA)
@@ -544,7 +543,7 @@ class TraditionalGitClient(GitClient):
 
             (have, want) = self._handle_receive_pack_head(
                 proto, negotiated_capabilities, old_refs, new_refs)
-            if not want and old_refs == new_refs:
+            if not want and set(new_refs.items()).issubset(set(old_refs.items())):
                 return new_refs
             objects = generate_pack_contents(have, want)
 
@@ -1077,7 +1076,7 @@ class HttpGitClient(GitClient):
         req_proto = Protocol(None, req_data.write)
         (have, want) = self._handle_receive_pack_head(
             req_proto, negotiated_capabilities, old_refs, new_refs)
-        if not want and old_refs == new_refs:
+        if not want and set(new_refs.items()).issubset(set(old_refs.items())):
             return new_refs
         objects = generate_pack_contents(have, want)
         if len(objects) > 0:

dulwich/contrib/swift.py

@@ -31,9 +31,16 @@ import zlib
 import tempfile
 import posixpath
 
-from urlparse import urlparse
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
+
 from io import BytesIO
-from ConfigParser import ConfigParser
+try:
+    from ConfigParser import ConfigParser
+except ImportError:
+    from configparser import ConfigParser
 from geventhttpclient import HTTPClient
 
 from dulwich.greenthreads import (
@@ -204,7 +211,7 @@ def pack_info_create(pack_data, pack_index):
         # Tree
         elif obj.type_num == Tree.type_num:
             shas = [(s, n, not stat.S_ISDIR(m)) for
-                    n, m, s in obj.iteritems() if not S_ISGITLINK(m)]
+                    n, m, s in obj.items() if not S_ISGITLINK(m)]
             info[obj.id] = (obj.type_num, shas)
         # Blob
         elif obj.type_num == Blob.type_num:
@@ -274,8 +281,8 @@ class SwiftConnector(object):
                                 connection_timeout=self.http_timeout,
                                 network_timeout=self.http_timeout,
                                 headers=token_header)
-        self.base_path = str(posixpath.join(urlparse(self.storage_url).path,
-                             self.root))
+        self.base_path = str(
+            posixpath.join(urlparse.urlparse(self.storage_url).path, self.root))
 
     def swift_auth_v1(self):
         self.user = self.user.replace(";", ":")
@@ -286,7 +293,7 @@ class SwiftConnector(object):
             )
         headers = {'X-Auth-User': self.user,
                    'X-Auth-Key': self.password}
-        path = urlparse(self.auth_url).path
+        path = urlparse.urlparse(self.auth_url).path
 
         ret = auth_httpclient.request('GET', path, headers=headers)
 
@@ -318,7 +325,7 @@ class SwiftConnector(object):
             connection_timeout=self.http_timeout,
             network_timeout=self.http_timeout,
             )
-        path = urlparse(self.auth_url).path
+        path = urlparse.urlparse(self.auth_url).path
         if not path.endswith('tokens'):
             path = posixpath.join(path, 'tokens')
         ret = auth_httpclient.request('POST', path,
@@ -397,7 +404,7 @@ class SwiftConnector(object):
             raise SwiftException('HEAD request failed with error code %s'
                                  % ret.status_code)
         resp_headers = {}
-        for header, value in ret.iteritems():
+        for header, value in ret.items():
             resp_headers[header.lower()] = value
         return resp_headers
 
@@ -502,7 +509,7 @@ class SwiftPackReader(object):
         self.pack_length = pack_length
         self.offset = 0
         self.base_offset = 0
-        self.buff = ''
+        self.buff = b''
         self.buff_length = self.scon.chunk_length
 
     def _read(self, more=False):
@@ -523,16 +530,14 @@ class SwiftPackReader(object):
         if self.base_offset + end > self.pack_length:
             data = self.buff[self.offset:]
             self.offset = end
-            return "".join(data)
-        try:
-            self.buff[end]
-        except IndexError:
+            return data
+        if end > len(self.buff):
             # Need to read more from swift
             self._read(more=True)
             return self.read(length)
         data = self.buff[self.offset:end]
         self.offset = end
-        return "".join(data)
+        return data
 
     def seek(self, offset):
         """Seek to a specified offset
@@ -579,8 +584,6 @@ class SwiftPackData(PackData):
     def get_object_at(self, offset):
         if offset in self._offset_cache:
             return self._offset_cache[offset]
-        assert isinstance(offset, long) or isinstance(offset, int),\
-            'offset was %r' % offset
         assert offset >= self._header_size
         pack_reader = SwiftPackReader(self.scon, self._filename,
                                       self.pack_length)
@@ -803,7 +806,8 @@ class SwiftObjectStore(PackBasedObjectStore):
         # Move the pack in.
         entries.sort()
         pack_base_name = posixpath.join(
-            self.pack_dir, 'pack-' + iter_sha1(e[0] for e in entries))
+            self.pack_dir,
+            'pack-' + iter_sha1(e[0] for e in entries).decode(sys.getfilesystemencoding()))
         self.scon.put_object(pack_base_name + '.pack', f)
 
         # Write the index.
@@ -842,7 +846,7 @@ class SwiftInfoRefsContainer(InfoRefsContainer):
         self.store = store
         f = self.scon.get_object(self.filename)
         if not f:
-            f = BytesIO('')
+            f = BytesIO(b'')
         super(SwiftInfoRefsContainer, self).__init__(f)
 
     def _load_check_ref(self, name, old_ref):
@@ -944,7 +948,7 @@ class SwiftRepo(BaseRepo):
         scon.create_root()
         for obj in [posixpath.join(OBJECTDIR, PACKDIR),
                     posixpath.join(INFODIR, 'refs')]:
-            scon.put_object(obj, BytesIO(''))
+            scon.put_object(obj, BytesIO(b''))
         ret = cls(scon.root, conf)
         ret._init_files(True)
         return ret

dulwich/contrib/test_swift.py

@@ -25,6 +25,12 @@ import posixpath
 
 from time import time
 from io import BytesIO
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+import sys
 from unittest import skipIf
 
 from dulwich.tests import (
@@ -74,6 +80,8 @@ except ImportError:
 
 skipmsg = "Required libraries are not installed (%r)" % missing_libs
 
+skipIfPY3 = skipIf(sys.version_info[0] == 3, "SWIFT module not yet ported to python3.")
+
 if not missing_libs:
     from dulwich.contrib import swift
 
@@ -119,11 +127,7 @@ class Response(object):
         return self.headers[key]
 
     def items(self):
-        return self.headers
-
-    def iteritems(self):
-        for k, v in self.headers.iteritems():
-            yield k, v
+        return self.headers.items()
 
     def read(self):
         return self.content
@@ -161,37 +165,37 @@ def fake_auth_request_v2(*args, **kwargs):
     return ret
 
 
-def create_commit(data, marker='Default', blob=None):
+def create_commit(data, marker=b'Default', blob=None):
     if not blob:
-        blob = Blob.from_string('The blob content %s' % marker)
+        blob = Blob.from_string(b'The blob content ' + marker)
     tree = Tree()
-    tree.add("thefile_%s" % marker, 0o100644, blob.id)
+    tree.add(b"thefile_" + marker, 0o100644, blob.id)
     cmt = Commit()
     if data:
         assert isinstance(data[-1], Commit)
         cmt.parents = [data[-1].id]
     cmt.tree = tree.id
-    author = "John Doe %s <john@doe.net>" % marker
+    author = b"John Doe " + marker + b" <john@doe.net>"
     cmt.author = cmt.committer = author
-    tz = parse_timezone('-0200')[0]
+    tz = parse_timezone(b'-0200')[0]
     cmt.commit_time = cmt.author_time = int(time())
     cmt.commit_timezone = cmt.author_timezone = tz
-    cmt.encoding = "UTF-8"
-    cmt.message = "The commit message %s" % marker
+    cmt.encoding = b"UTF-8"
+    cmt.message = b"The commit message " + marker
     tag = Tag()
-    tag.tagger = "john@doe.net"
-    tag.message = "Annotated tag"
-    tag.tag_timezone = parse_timezone('-0200')[0]
+    tag.tagger = b"john@doe.net"
+    tag.message = b"Annotated tag"
+    tag.tag_timezone = parse_timezone(b'-0200')[0]
     tag.tag_time = cmt.author_time
     tag.object = (Commit, cmt.id)
-    tag.name = "v_%s_0.1" % marker
+    tag.name = b"v_" + marker + b"_0.1"
     return blob, tree, tag, cmt
 
 
-def create_commits(length=1, marker='Default'):
+def create_commits(length=1, marker=b'Default'):
     data = []
     for i in range(0, length):
-        _marker = "%s_%s" % (marker, i)
+        _marker = ("%s_%s" % (marker, i)).encode()
         blob, tree, tag, cmt = create_commit(data, _marker)
         data.extend([blob, tree, tag, cmt])
     return data
@@ -253,11 +257,12 @@ class FakeSwiftConnector(object):
 
 
 @skipIf(missing_libs, skipmsg)
+@skipIfPY3
 class TestSwiftObjectStore(TestCase):
 
     def setUp(self):
         super(TestSwiftObjectStore, self).setUp()
-        self.conf = swift.load_conf(file=BytesIO(config_file %
+        self.conf = swift.load_conf(file=StringIO(config_file %
                                                   def_config_file))
         self.fsc = FakeSwiftConnector('fakerepo', conf=self.conf)
 
@@ -376,7 +381,7 @@ class TestSwiftRepo(TestCase):
 
     def setUp(self):
         super(TestSwiftRepo, self).setUp()
-        self.conf = swift.load_conf(file=BytesIO(config_file %
+        self.conf = swift.load_conf(file=StringIO(config_file %
                                                   def_config_file))
 
     def test_init(self):
@@ -406,7 +411,7 @@ class TestSwiftRepo(TestCase):
                    new_callable=create_swift_connector,
                    store=store):
             repo = swift.SwiftRepo('fakerepo', conf=self.conf)
-            desc = 'Fake repo'
+            desc = b'Fake repo'
             repo._put_named_file('description', desc)
         self.assertEqual(repo.scon.store['fakerepo/description'],
                          desc)
@@ -423,9 +428,10 @@ class TestSwiftRepo(TestCase):
 
 
 @skipIf(missing_libs, skipmsg)
+@skipIfPY3
 class TestPackInfoLoadDump(TestCase):
     def setUp(self):
-        conf = swift.load_conf(file=BytesIO(config_file %
+        conf = swift.load_conf(file=StringIO(config_file %
                                              def_config_file))
         sos = swift.SwiftObjectStore(
             FakeSwiftConnector('fakerepo', conf=conf))
@@ -470,10 +476,10 @@ class TestSwiftInfoRefsContainer(TestCase):
     def setUp(self):
         super(TestSwiftInfoRefsContainer, self).setUp()
         content = \
-            "22effb216e3a82f97da599b8885a6cadb488b4c5\trefs/heads/master\n" + \
-            "cca703b0e1399008b53a1a236d6b4584737649e4\trefs/heads/dev"
+            b"22effb216e3a82f97da599b8885a6cadb488b4c5\trefs/heads/master\n" + \
+            b"cca703b0e1399008b53a1a236d6b4584737649e4\trefs/heads/dev"
         self.store = {'fakerepo/info/refs': content}
-        self.conf = swift.load_conf(file=BytesIO(config_file %
+        self.conf = swift.load_conf(file=StringIO(config_file %
                                                   def_config_file))
         self.fsc = FakeSwiftConnector('fakerepo', conf=self.conf)
         self.object_store = {}
@@ -484,22 +490,22 @@ class TestSwiftInfoRefsContainer(TestCase):
         self.assertEqual(len(irc._refs), 0)
         self.fsc.store = self.store
         irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store)
-        self.assertIn('refs/heads/dev', irc.allkeys())
-        self.assertIn('refs/heads/master', irc.allkeys())
+        self.assertIn(b'refs/heads/dev', irc.allkeys())
+        self.assertIn(b'refs/heads/master', irc.allkeys())
 
     def test_set_if_equals(self):
         self.fsc.store = self.store
         irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store)
-        irc.set_if_equals('refs/heads/dev',
-                          "cca703b0e1399008b53a1a236d6b4584737649e4", '1'*40)
-        self.assertEqual(irc['refs/heads/dev'], '1'*40)
+        irc.set_if_equals(b'refs/heads/dev',
+                          b"cca703b0e1399008b53a1a236d6b4584737649e4", b'1'*40)
+        self.assertEqual(irc[b'refs/heads/dev'], b'1'*40)
 
     def test_remove_if_equals(self):
         self.fsc.store = self.store
         irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store)
-        irc.remove_if_equals('refs/heads/dev',
-                             "cca703b0e1399008b53a1a236d6b4584737649e4")
-        self.assertNotIn('refs/heads/dev', irc.allkeys())
+        irc.remove_if_equals(b'refs/heads/dev',
+                             b"cca703b0e1399008b53a1a236d6b4584737649e4")
+        self.assertNotIn(b'refs/heads/dev', irc.allkeys())
 
 
 @skipIf(missing_libs, skipmsg)
@@ -507,7 +513,7 @@ class TestSwiftConnector(TestCase):
 
     def setUp(self):
         super(TestSwiftConnector, self).setUp()
-        self.conf = swift.load_conf(file=BytesIO(config_file %
+        self.conf = swift.load_conf(file=StringIO(config_file %
                                                   def_config_file))
         with patch('geventhttpclient.HTTPClient.request',
                    fake_auth_request_v1):
@@ -591,7 +597,7 @@ class TestSwiftConnector(TestCase):
     def test_put_object(self):
         with patch('geventhttpclient.HTTPClient.request',
                    lambda *args, **kwargs: Response()):
-            self.assertEqual(self.conn.put_object('a', BytesIO('content')),
+            self.assertEqual(self.conn.put_object('a', BytesIO(b'content')),
                              None)
 
     def test_put_object_fails(self):
@@ -599,15 +605,15 @@ class TestSwiftConnector(TestCase):
                    lambda *args, **kwargs: Response(status=400)):
             self.assertRaises(swift.SwiftException,
                               lambda: self.conn.put_object(
-                                  'a', BytesIO('content')))
+                                  'a', BytesIO(b'content')))
 
     def test_get_object(self):
         with patch('geventhttpclient.HTTPClient.request',
-                   lambda *args, **kwargs: Response(content='content')):
-            self.assertEqual(self.conn.get_object('a').read(), 'content')
+                   lambda *args, **kwargs: Response(content=b'content')):
+            self.assertEqual(self.conn.get_object('a').read(), b'content')
         with patch('geventhttpclient.HTTPClient.request',
-                   lambda *args, **kwargs: Response(content='content')):
-            self.assertEqual(self.conn.get_object('a', range='0-6'), 'content')
+                   lambda *args, **kwargs: Response(content=b'content')):
+            self.assertEqual(self.conn.get_object('a', range='0-6'), b'content')
 
     def test_get_object_fails(self):
         with patch('geventhttpclient.HTTPClient.request',
@@ -635,7 +641,7 @@ class SwiftObjectStoreTests(ObjectStoreTests, TestCase):
 
     def setUp(self):
         TestCase.setUp(self)
-        conf = swift.load_conf(file=BytesIO(config_file %
+        conf = swift.load_conf(file=StringIO(config_file %
                                def_config_file))
         fsc = FakeSwiftConnector('fakerepo', conf=conf)
         self.store = swift.SwiftObjectStore(fsc)

dulwich/refs.py

@@ -756,7 +756,7 @@ def write_packed_refs(f, packed_refs, peeled_refs=None):
 def read_info_refs(f):
     ret = {}
     for l in f.readlines():
-        (sha, name) = l.rstrip("\r\n").split("\t", 1)
+        (sha, name) = l.rstrip(b"\r\n").split(b"\t", 1)
         ret[name] = sha
     return ret
 

dulwich/tests/test_client.py

@@ -190,6 +190,32 @@ class GitClientTests(TestCase):
         self.client.send_pack(b'/', determine_wants, generate_pack_contents)
         self.assertEqual(self.rout.getvalue(), b'0000')
 
+    def test_send_pack_keep_and_delete(self):
+        self.rin.write(
+            b'0063310ca9477129b8586fa2afc779c1f57cf64bba6c '
+            b'refs/heads/master\x00report-status delete-refs ofs-delta\n'
+            b'003f310ca9477129b8586fa2afc779c1f57cf64bba6c refs/heads/keepme\n'
+            b'0000000eunpack ok\n'
+            b'0019ok refs/heads/master\n'
+            b'0000')
+        self.rin.seek(0)
+
+        def determine_wants(refs):
+            return {b'refs/heads/master': b'0' * 40}
+
+        def generate_pack_contents(have, want):
+            return {}
+
+        self.client.send_pack(b'/', determine_wants, generate_pack_contents)
+        self.assertIn(
+            self.rout.getvalue(),
+            [b'007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
+             b'0000000000000000000000000000000000000000 '
+             b'refs/heads/master\x00report-status ofs-delta0000',
+             b'007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
+             b'0000000000000000000000000000000000000000 '
+             b'refs/heads/master\x00ofs-delta report-status0000'])
+
     def test_send_pack_delete_only(self):
         self.rin.write(
             b'0063310ca9477129b8586fa2afc779c1f57cf64bba6c '

setup.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # Setup file for dulwich
-# Copyright (C) 2008-2011 Jelmer Vernooij <jelmer@jelmer.uk>
+# Copyright (C) 2008-2016 Jelmer Vernooij <jelmer@jelmer.uk>
 
 try:
     from setuptools import setup, Extension
@@ -8,7 +8,7 @@ except ImportError:
     from distutils.core import setup, Extension
 from distutils.core import Distribution
 
-dulwich_version_string = '0.14.0'
+dulwich_version_string = '0.14.1'
 
 include_dirs = []
 # Windows MSVC support