dumb: Add fallback when HEAD is missing

Some really old git repositories still using the dumb transfer
protocol do not serve the HEAD file leading to errors when
attempting to clone them with dulwich.

Do as official git C client when encountering that edge case:
create a HEAD at the end of clone operation that targets the
configured default branch.

dulwich/client.py

@@ -1164,12 +1164,22 @@ class GitClient:
                 else:
                     head = None
             else:
+                from dulwich import porcelain
+
                 _set_origin_head(target.refs, origin.encode("utf-8"), origin_head)
+
+                default_branch_name = porcelain.var(
+                    target, variable="GIT_DEFAULT_BRANCH"
+                ).encode("utf-8")
+                default_branch: bytes | None = default_branch_name
+                default_ref = Ref(b"refs/remotes/origin/" + default_branch_name)
+                if default_ref not in target.refs:
+                    default_branch = None
                 head_ref = _set_default_branch(
                     target.refs,
                     origin.encode("utf-8"),
                     origin_head,
-                    branch.encode("utf-8") if branch is not None else None,
+                    (branch.encode("utf-8") if branch is not None else default_branch),
                     ref_message,
                 )
 
@@ -4096,7 +4106,9 @@ class AbstractHttpGitClient(GitClient):
                 )
             )
 
-            symrefs[HEADREF] = dumb_repo.get_head()
+            head = dumb_repo.get_head()
+            if head is not None:
+                symrefs[HEADREF] = head
 
             # Write pack data
             if pack_data_list:

dulwich/dumb.py

@@ -430,19 +430,27 @@ class DumbRemoteHTTPRepo:
 
         return dict(self._refs)
 
-    def get_head(self) -> Ref:
+    def get_head(self) -> Ref | None:
         """Get the current HEAD reference.
 
         Returns:
           HEAD reference name or commit ID
         """
-        head_resp_bytes = self._fetch_url("HEAD")
-        head_split = head_resp_bytes.replace(b"\n", b"").split(b" ")
-        head_target_bytes = head_split[1] if len(head_split) > 1 else head_split[0]
-        # handle HEAD legacy format containing a commit id instead of a ref name
-        for ref_name, ret_target in self.get_refs().items():
-            if ret_target == head_target_bytes:
-                return ref_name
+        try:
+            head_resp_bytes = self._fetch_url("HEAD")
+        except OSError as e:
+            if "HTTP error 429" not in str(e):
+                return None
+            else:
+                # rate-limit reached so raise exception
+                raise
+        else:
+            head_split = head_resp_bytes.replace(b"\n", b"").split(b" ")
+            head_target_bytes = head_split[1] if len(head_split) > 1 else head_split[0]
+            # handle HEAD legacy format containing a commit id instead of a ref name
+            for ref_name, ret_target in self.get_refs().items():
+                if ret_target == head_target_bytes:
+                    return ref_name
         return Ref(head_target_bytes)
 
     def get_peeled(self, ref: Ref) -> ObjectID:

tests/compat/test_dumb.py

@@ -110,6 +110,7 @@ class DumbHTTPClientNoPackTests(CompatTestCase):
     """Tests for dumb HTTP client against real git repositories."""
 
     with_pack = False
+    with_missing_remote_head = False
 
     def setUp(self):
         super().setUp()
@@ -152,6 +153,9 @@ class DumbHTTPClientNoPackTests(CompatTestCase):
         # Update server info for dumb HTTP
         run_git_or_fail(["update-server-info"], cwd=self.origin_path)
 
+        if self.with_missing_remote_head:
+            os.remove(os.path.join(self.origin_path, "HEAD"))
+
         # Start HTTP server
         self.server = DumbHTTPGitServer(self.origin_path)
         self.server.start()
@@ -314,3 +318,16 @@ class DumbHTTPClientNoPackTests(CompatTestCase):
 
 class DumbHTTPClientWithPackTests(DumbHTTPClientNoPackTests):
     with_pack = True
+
+
+class DumbHTTPClientWithMissingRemoteHEAD(DumbHTTPClientNoPackTests):
+    with_missing_remote_head = True
+
+    # we only want to test clone operation as removing the HEAD file
+    # prevents any push operation used in tests below
+
+    def test_fetch_from_dumb_http_with_tags(self):
+        pass
+
+    def test_fetch_new_commit_from_dumb_http(self):
+        pass

tests/test_dumb.py

@@ -308,3 +308,9 @@ fedcba9876543210fedcba9876543210fedcba98\trefs/tags/v1.0
     def test_object_store_property(self) -> None:
         self.assertIsInstance(self.repo.object_store, DumbHTTPObjectStore)
         self.assertEqual(self.base_url, self.repo.object_store.base_url)
+
+    def test_fetch_pack_data_missing_head(self) -> None:
+        refs_content = b"0123456789abcdef0123456789abcdef01234567\trefs/heads/master\n"
+        self._add_response("info/refs", refs_content)
+        self._add_response("HEAD", b"", status=404)
+        assert self.repo.get_head() is None