2
0
Эх сурвалжийг харах

Merge commit '183a43059fe40949ef7674626ac2cd2f8eb0a635'

Jelmer Vernooij 5 жил өмнө
parent
commit
59ba47f046

+ 3 - 0
NEWS

@@ -6,6 +6,9 @@
  * Decode URL paths in HttpGitClient using utf-8 rather than file system
    encoding. (Manuel Jacob)
 
+ * Fix pushing from a shallow clone.
+   (Brecht Machiels, #705)
+
 0.19.16	2020-04-17
 
  * Don't send "deepen None" to server if graph walker

+ 2 - 3
dulwich/client.py

@@ -30,10 +30,10 @@ The Dulwich client supports the following capabilities:
  * quiet
  * report-status
  * delete-refs
+ * shallow
 
 Known capabilities that are not supported:
 
- * shallow
  * no-progress
  * include-tag
 """
@@ -1125,7 +1125,6 @@ class LocalGitClient(GitClient):
           update_refs: Function to determine changes to remote refs.
         Receive dict with existing remote refs, returns dict with
         changed refs (name -> sha, where sha=ZERO_SHA for deletions)
-          generate_pack_data: Function that can return a tuple
         with number of items and pack data to upload.
           progress: Optional progress function
 
@@ -1676,7 +1675,7 @@ class HttpGitClient(GitClient):
         Args:
           path: Repository path (as bytestring)
           update_refs: Function to determine changes to remote refs.
-        Receive dict with existing remote refs, returns dict with
+        Receives dict with existing remote refs, returns dict with
         changed refs (name -> sha, where sha=ZERO_SHA for deletions)
           generate_pack_data: Function that can return a tuple
         with number of elements and pack data to upload.

+ 14 - 21
dulwich/contrib/test_swift_smoke.py

@@ -140,9 +140,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         swift.SwiftRepo.init_bare(self.scon, self.conf)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack(self.fakerepo,
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack(self.fakerepo, determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo("fakerepo", self.conf)
         remote_sha = swift_repo.refs.read_loose_ref('refs/heads/master')
         self.assertEqual(sha, remote_sha)
@@ -160,9 +159,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         swift.SwiftRepo.init_bare(self.scon, self.conf)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack("/fakerepo",
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack("/fakerepo", determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo(self.fakerepo, self.conf)
         remote_sha = swift_repo.refs.read_loose_ref('refs/heads/mybranch')
         self.assertEqual(sha, remote_sha)
@@ -187,9 +185,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         swift.SwiftRepo.init_bare(self.scon, self.conf)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack(self.fakerepo,
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack(self.fakerepo, determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo("fakerepo", self.conf)
         for branch in ('master', 'mybranch', 'pullr-108'):
             remote_shas[branch] = swift_repo.refs.read_loose_ref(
@@ -212,9 +209,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         swift.SwiftRepo.init_bare(self.scon, self.conf)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack(self.fakerepo,
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack(self.fakerepo, determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo("fakerepo", self.conf)
         commit_sha = swift_repo.refs.read_loose_ref('refs/heads/master')
         otype, data = swift_repo.object_store.get_raw(commit_sha)
@@ -259,9 +255,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         local_repo.stage(files)
         local_repo.do_commit('Test commit', 'fbo@localhost',
                              ref='refs/heads/master')
-        tcp_client.send_pack("/fakerepo",
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack("/fakerepo", determine_wants,
+                             local_repo.generate_pack_data)
 
     def test_push_remove_branch(self):
         def determine_wants(*args):
@@ -275,9 +270,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         local_repo = repo.Repo(self.temp_d)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack(self.fakerepo,
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack(self.fakerepo, determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo("fakerepo", self.conf)
         self.assertNotIn('refs/heads/pullr-108', swift_repo.refs.allkeys())
 
@@ -302,9 +296,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         swift.SwiftRepo.init_bare(self.scon, self.conf)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack(self.fakerepo,
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack(self.fakerepo, determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo(self.fakerepo, self.conf)
         tag_sha = swift_repo.refs.read_loose_ref('refs/tags/v1.0')
         otype, data = swift_repo.object_store.get_raw(tag_sha)

+ 21 - 11
dulwich/object_store.py

@@ -201,7 +201,7 @@ class BaseObjectStore(object):
                  not stat.S_ISDIR(entry.mode)) or include_trees):
                 yield entry
 
-    def find_missing_objects(self, haves, wants, progress=None,
+    def find_missing_objects(self, haves, wants, shallow=None, progress=None,
                              get_tagged=None,
                              get_parents=lambda commit: commit.parents,
                              depth=None):
@@ -210,6 +210,7 @@ class BaseObjectStore(object):
         Args:
           haves: Iterable over SHAs already in common.
           wants: Iterable over SHAs of objects to fetch.
+          shallow: Set of shallow commit SHA1s to skip
           progress: Simple progress function that will be called with
             updated progress strings.
           get_tagged: Function that returns a dict of pointed-to sha ->
@@ -218,8 +219,8 @@ class BaseObjectStore(object):
             commit.
         Returns: Iterator over (sha, path) pairs.
         """
-        finder = MissingObjectFinder(self, haves, wants, progress, get_tagged,
-                                     get_parents=get_parents)
+        finder = MissingObjectFinder(self, haves, wants, shallow, progress,
+                                     get_tagged, get_parents=get_parents)
         return iter(finder.next, None)
 
     def find_common_revisions(self, graphwalker):
@@ -238,28 +239,32 @@ class BaseObjectStore(object):
             sha = next(graphwalker)
         return haves
 
-    def generate_pack_contents(self, have, want, progress=None):
+    def generate_pack_contents(self, have, want, shallow=None, progress=None):
         """Iterate over the contents of a pack file.
 
         Args:
           have: List of SHA1s of objects that should not be sent
           want: List of SHA1s of objects that should be sent
+          shallow: Set of shallow commit SHA1s to skip
           progress: Optional progress reporting method
         """
-        return self.iter_shas(self.find_missing_objects(have, want, progress))
+        missing = self.find_missing_objects(have, want, shallow, progress)
+        return self.iter_shas(missing)
 
-    def generate_pack_data(self, have, want, progress=None, ofs_delta=True):
+    def generate_pack_data(self, have, want, shallow=None, progress=None,
+                           ofs_delta=True):
         """Generate pack data objects for a set of wants/haves.
 
         Args:
           have: List of SHA1s of objects that should not be sent
           want: List of SHA1s of objects that should be sent
+          shallow: Set of shallow commit SHA1s to skip
           ofs_delta: Whether OFS deltas can be included
           progress: Optional progress reporting method
         """
         # TODO(jelmer): More efficient implementation
         return pack_objects_to_data(
-            self.generate_pack_contents(have, want, progress))
+            self.generate_pack_contents(have, want, shallow, progress))
 
     def peel_sha(self, sha):
         """Peel all tags from a SHA.
@@ -277,7 +282,7 @@ class BaseObjectStore(object):
             obj = self[sha]
         return obj
 
-    def _collect_ancestors(self, heads, common=set(),
+    def _collect_ancestors(self, heads, common=set(), shallow=set(),
                            get_parents=lambda commit: commit.parents):
         """Collect all ancestors of heads up to (excluding) those in common.
 
@@ -301,6 +306,8 @@ class BaseObjectStore(object):
                 bases.add(e)
             elif e not in commits:
                 commits.add(e)
+                if e in shallow:
+                    continue
                 cmt = self[e]
                 queue.extend(get_parents(cmt))
         return (commits, bases)
@@ -1162,9 +1169,11 @@ class MissingObjectFinder(object):
       tagged: dict of pointed-to sha -> tag sha for including tags
     """
 
-    def __init__(self, object_store, haves, wants, progress=None,
+    def __init__(self, object_store, haves, wants, shallow=None, progress=None,
                  get_tagged=None, get_parents=lambda commit: commit.parents):
         self.object_store = object_store
+        if shallow is None:
+            shallow = set()
         self._get_parents = get_parents
         # process Commits and Tags differently
         # Note, while haves may list commits/tags not available locally,
@@ -1178,12 +1187,13 @@ class MissingObjectFinder(object):
         # all_ancestors is a set of commits that shall not be sent
         # (complete repository up to 'haves')
         all_ancestors = object_store._collect_ancestors(
-            have_commits, get_parents=self._get_parents)[0]
+            have_commits, shallow=shallow, get_parents=self._get_parents)[0]
         # all_missing - complete set of commits between haves and wants
         # common - commits from all_ancestors we hit into while
         # traversing parent hierarchy of wants
         missing_commits, common_commits = object_store._collect_ancestors(
-            want_commits, all_ancestors, get_parents=self._get_parents)
+            want_commits, all_ancestors, shallow=shallow,
+            get_parents=self._get_parents)
         self.sha_done = set()
         # Now, fill sha_done with commits and revisions of
         # files and directories known to be both locally

+ 1 - 1
dulwich/porcelain.py

@@ -912,7 +912,7 @@ def push(repo, remote_location, refspecs,
         try:
             client.send_pack(
                 path, update_refs,
-                generate_pack_data=r.object_store.generate_pack_data,
+                generate_pack_data=r.generate_pack_data,
                 progress=errstream.write)
             errstream.write(
                 b"Push to " + remote_location_bytes + b" successful.\n")

+ 15 - 2
dulwich/repo.py

@@ -468,10 +468,23 @@ class BaseRepo(object):
 
         return self.object_store.iter_shas(
           self.object_store.find_missing_objects(
-              haves, wants, progress,
-              get_tagged,
+              haves, wants, self.get_shallow(),
+              progress, get_tagged,
               get_parents=get_parents))
 
+    def generate_pack_data(self, have, want, progress=None, ofs_delta=None):
+        """Generate pack data objects for a set of wants/haves.
+
+        Args:
+          have: List of SHA1s of objects that should not be sent
+          want: List of SHA1s of objects that should be sent
+          ofs_delta: Whether OFS deltas can be included
+          progress: Optional progress reporting method
+        """
+        return self.object_store.generate_pack_data(
+            have, want, shallow=self.get_shallow(),
+            progress=progress, ofs_delta=ofs_delta)
+
     def get_graph_walker(self, heads=None):
         """Retrieve a graph walker.
 

+ 7 - 8
dulwich/tests/compat/test_client.py

@@ -25,6 +25,7 @@ from io import BytesIO
 import os
 import select
 import signal
+import stat
 import subprocess
 import sys
 import tarfile
@@ -106,7 +107,7 @@ class DulwichClientTestBase(object):
             sendrefs = dict(src.get_refs())
             del sendrefs[b'HEAD']
             c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
-                        src.object_store.generate_pack_data)
+                        src.generate_pack_data)
 
     def test_send_pack(self):
         self._do_send_pack()
@@ -128,8 +129,6 @@ class DulwichClientTestBase(object):
         repo.object_store.add_object(tree)
         return tree.id
 
-    # Pushing from a shallow clone currently fails. See #705
-    @unittest.expectedFailure
     def test_send_pack_from_shallow_clone(self):
         c = self._client()
         server_new_path = os.path.join(self.gitroot, 'server_new.export')
@@ -153,7 +152,7 @@ class DulwichClientTestBase(object):
             sendrefs = dict(local.get_refs())
             del sendrefs[b'HEAD']
             c.send_pack(remote_path, lambda _: sendrefs,
-                        local.object_store.generate_pack_data)
+                        local.generate_pack_data)
         with repo.Repo(server_new_path) as remote:
             self.assertEqual(remote.head(), commit_id)
 
@@ -165,7 +164,7 @@ class DulwichClientTestBase(object):
             sendrefs = dict(src.get_refs())
             del sendrefs[b'HEAD']
             c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
-                        src.object_store.generate_pack_data)
+                        src.generate_pack_data)
             self.assertDestEqualsSrc()
 
     def make_dummy_commit(self, dest):
@@ -192,7 +191,7 @@ class DulwichClientTestBase(object):
     def compute_send(self, src):
         sendrefs = dict(src.get_refs())
         del sendrefs[b'HEAD']
-        return sendrefs, src.object_store.generate_pack_data
+        return sendrefs, src.generate_pack_data
 
     def test_send_pack_one_error(self):
         dest, dummy_commit = self.disable_ff_and_make_dummy_commit()
@@ -202,8 +201,8 @@ class DulwichClientTestBase(object):
             sendrefs, gen_pack = self.compute_send(src)
             c = self._client()
             try:
-                c.send_pack(self._build_path('/dest'),
-                            lambda _: sendrefs, gen_pack)
+                c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
+                            gen_pack)
             except errors.UpdateRefsError as e:
                 self.assertEqual('refs/heads/master failed to update',
                                  e.args[0])

+ 2 - 2
dulwich/tests/test_client.py

@@ -247,7 +247,7 @@ class GitClientTests(TestCase):
         def generate_pack_data(have, want, ofs_delta=False):
             return 0, []
 
-        self.client.send_pack(b'/', update_refs, generate_pack_data)
+        self.client.send_pack(b'/', update_refs, set(), generate_pack_data)
         self.assertEqual(self.rout.getvalue(), b'0000')
 
     def test_send_pack_keep_and_delete(self):
@@ -873,7 +873,7 @@ class LocalGitClientTests(TestCase):
         ref_name = b"refs/heads/" + branch
         new_refs = client.send_pack(target.path,
                                     lambda _: {ref_name: local.refs[ref_name]},
-                                    local.object_store.generate_pack_data)
+                                    local.generate_pack_data)
 
         self.assertEqual(local.refs[ref_name], new_refs[ref_name])
 

+ 2 - 2
dulwich/tests/test_missing_obj_finder.py

@@ -43,7 +43,7 @@ class MissingObjectFinderTest(TestCase):
         return self.commits[n-1]
 
     def assertMissingMatch(self, haves, wants, expected):
-        for sha, path in self.store.find_missing_objects(haves, wants):
+        for sha, path in self.store.find_missing_objects(haves, wants, set()):
             self.assertTrue(
                     sha in expected,
                     "(%s,%s) erroneously reported as missing" % (sha, path))
@@ -112,7 +112,7 @@ class MOFLinearRepoTest(MissingObjectFinderTest):
         haves = [self.cmt(1).id]
         wants = [self.cmt(3).id, bogus_sha]
         self.assertRaises(
-                KeyError, self.store.find_missing_objects, haves, wants)
+                KeyError, self.store.find_missing_objects, haves, wants, set())
 
     def test_no_changes(self):
         self.assertMissingMatch([self.cmt(3).id], [self.cmt(3).id], [])