dumb: Add fallback when HEAD is missing (#2030)

Some really old git repositories still using the dumb transfer protocol
do not serve the HEAD file leading to errors when attempting to clone
them with dulwich.

As a workaround set HEAD value to the first ref in the info/refs file.

Examples of such repositories: 
- http://git.1wt.eu/git/patches-2.4-hf.git
- http://git.1wt.eu/git/linux-2.4-upstream.git
- http://git.1wt.eu/git/alix-leds.git

dulwich/client.py

@@ -1165,11 +1165,28 @@ class GitClient:
                     head = None
             else:
                 _set_origin_head(target.refs, origin.encode("utf-8"), origin_head)
+
+                # If origin_head is None (missing HEAD), fall back to configured default branch
+                default_branch: bytes | None = None
+                if origin_head is None:
+                    target_config = target.get_config()
+                    try:
+                        default_branch_name = target_config.get(
+                            (b"init",), b"defaultBranch"
+                        )
+                    except KeyError:
+                        # Git's default is "master"
+                        default_branch_name = b"master"
+
+                    default_ref = Ref(b"refs/remotes/origin/" + default_branch_name)
+                    if default_ref in target.refs:
+                        default_branch = default_branch_name
+
                 head_ref = _set_default_branch(
                     target.refs,
                     origin.encode("utf-8"),
                     origin_head,
-                    branch.encode("utf-8") if branch is not None else None,
+                    (branch.encode("utf-8") if branch is not None else default_branch),
                     ref_message,
                 )
 
@@ -4096,7 +4113,9 @@ class AbstractHttpGitClient(GitClient):
                 )
             )
 
-            symrefs[HEADREF] = dumb_repo.get_head()
+            head = dumb_repo.get_head()
+            if head is not None:
+                symrefs[HEADREF] = head
 
             # Write pack data
             if pack_data_list:

dulwich/dumb.py

@@ -430,19 +430,27 @@ class DumbRemoteHTTPRepo:
 
         return dict(self._refs)
 
-    def get_head(self) -> Ref:
+    def get_head(self) -> Ref | None:
         """Get the current HEAD reference.
 
         Returns:
           HEAD reference name or commit ID
         """
-        head_resp_bytes = self._fetch_url("HEAD")
-        head_split = head_resp_bytes.replace(b"\n", b"").split(b" ")
-        head_target_bytes = head_split[1] if len(head_split) > 1 else head_split[0]
-        # handle HEAD legacy format containing a commit id instead of a ref name
-        for ref_name, ret_target in self.get_refs().items():
-            if ret_target == head_target_bytes:
-                return ref_name
+        try:
+            head_resp_bytes = self._fetch_url("HEAD")
+        except OSError as e:
+            if "HTTP error 429" not in str(e):
+                return None
+            else:
+                # rate-limit reached so raise exception
+                raise
+        else:
+            head_split = head_resp_bytes.replace(b"\n", b"").split(b" ")
+            head_target_bytes = head_split[1] if len(head_split) > 1 else head_split[0]
+            # handle HEAD legacy format containing a commit id instead of a ref name
+            for ref_name, ret_target in self.get_refs().items():
+                if ret_target == head_target_bytes:
+                    return ref_name
         return Ref(head_target_bytes)
 
     def get_peeled(self, ref: Ref) -> ObjectID:

tests/compat/test_dumb.py

@@ -110,6 +110,7 @@ class DumbHTTPClientNoPackTests(CompatTestCase):
     """Tests for dumb HTTP client against real git repositories."""
 
     with_pack = False
+    with_missing_remote_head = False
 
     def setUp(self):
         super().setUp()
@@ -152,6 +153,9 @@ class DumbHTTPClientNoPackTests(CompatTestCase):
         # Update server info for dumb HTTP
         run_git_or_fail(["update-server-info"], cwd=self.origin_path)
 
+        if self.with_missing_remote_head:
+            os.remove(os.path.join(self.origin_path, "HEAD"))
+
         # Start HTTP server
         self.server = DumbHTTPGitServer(self.origin_path)
         self.server.start()
@@ -314,3 +318,16 @@ class DumbHTTPClientNoPackTests(CompatTestCase):
 
 class DumbHTTPClientWithPackTests(DumbHTTPClientNoPackTests):
     with_pack = True
+
+
+class DumbHTTPClientWithMissingRemoteHEAD(DumbHTTPClientNoPackTests):
+    with_missing_remote_head = True
+
+    # we only want to test clone operation as removing the HEAD file
+    # prevents any push operation used in tests below
+
+    def test_fetch_from_dumb_http_with_tags(self):
+        pass
+
+    def test_fetch_new_commit_from_dumb_http(self):
+        pass

tests/test_dumb.py

@@ -308,3 +308,9 @@ fedcba9876543210fedcba9876543210fedcba98\trefs/tags/v1.0
     def test_object_store_property(self) -> None:
         self.assertIsInstance(self.repo.object_store, DumbHTTPObjectStore)
         self.assertEqual(self.base_url, self.repo.object_store.base_url)
+
+    def test_fetch_pack_data_missing_head(self) -> None:
+        refs_content = b"0123456789abcdef0123456789abcdef01234567\trefs/heads/master\n"
+        self._add_response("info/refs", refs_content)
+        self._add_response("HEAD", b"", status=404)
+        assert self.repo.get_head() is None