server.py 59 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775
  1. # server.py -- Implementation of the server side git protocols
  2. # Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
  3. # Copyright(C) 2011-2012 Jelmer Vernooij <jelmer@jelmer.uk>
  4. #
  5. # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
  6. # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
  7. # General Public License as published by the Free Software Foundation; version 2.0
  8. # or (at your option) any later version. You can redistribute it and/or
  9. # modify it under the terms of either of these two licenses.
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. #
  17. # You should have received a copy of the licenses; if not, see
  18. # <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
  19. # and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
  20. # License, Version 2.0.
  21. #
  22. """Git smart network protocol server implementation.
  23. For more detailed implementation on the network protocol, see the
  24. Documentation/technical directory in the cgit distribution, and in particular:
  25. * Documentation/technical/protocol-capabilities.txt
  26. * Documentation/technical/pack-protocol.txt
  27. Currently supported capabilities:
  28. * include-tag
  29. * thin-pack
  30. * multi_ack_detailed
  31. * multi_ack
  32. * side-band-64k
  33. * ofs-delta
  34. * no-progress
  35. * report-status
  36. * delete-refs
  37. * shallow
  38. * symref
  39. """
  40. __all__ = [
  41. "DEFAULT_HANDLERS",
  42. "AckGraphWalkerImpl",
  43. "Backend",
  44. "BackendRepo",
  45. "DictBackend",
  46. "FileSystemBackend",
  47. "Handler",
  48. "MultiAckDetailedGraphWalkerImpl",
  49. "MultiAckGraphWalkerImpl",
  50. "PackHandler",
  51. "ReceivePackHandler",
  52. "SingleAckGraphWalkerImpl",
  53. "TCPGitRequestHandler",
  54. "TCPGitServer",
  55. "UploadArchiveHandler",
  56. "UploadPackHandler",
  57. "generate_info_refs",
  58. "generate_objects_info_packs",
  59. "main",
  60. "serve_command",
  61. "update_server_info",
  62. ]
  63. import os
  64. import socket
  65. import socketserver
  66. import sys
  67. import time
  68. import zlib
  69. from collections import deque
  70. from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
  71. from collections.abc import Set as AbstractSet
  72. from functools import partial
  73. from typing import IO, TYPE_CHECKING
  74. from typing import Protocol as TypingProtocol
  75. if TYPE_CHECKING:
  76. from .object_format import ObjectFormat
  77. from .object_store import BaseObjectStore
  78. from .repo import BaseRepo
  79. from dulwich import log_utils
  80. from .archive import tar_stream
  81. from .errors import (
  82. ApplyDeltaError,
  83. ChecksumMismatch,
  84. GitProtocolError,
  85. HookError,
  86. NotGitRepository,
  87. ObjectFormatException,
  88. UnexpectedCommandError,
  89. )
  90. from .object_store import MissingObjectFinder, PackBasedObjectStore, find_shallow
  91. from .objects import Commit, ObjectID, Tree, valid_hexsha
  92. from .pack import ObjectContainer, write_pack_from_container
  93. from .protocol import (
  94. CAPABILITIES_REF,
  95. CAPABILITY_AGENT,
  96. CAPABILITY_DELETE_REFS,
  97. CAPABILITY_INCLUDE_TAG,
  98. CAPABILITY_MULTI_ACK,
  99. CAPABILITY_MULTI_ACK_DETAILED,
  100. CAPABILITY_NO_DONE,
  101. CAPABILITY_NO_PROGRESS,
  102. CAPABILITY_OFS_DELTA,
  103. CAPABILITY_QUIET,
  104. CAPABILITY_REPORT_STATUS,
  105. CAPABILITY_SHALLOW,
  106. CAPABILITY_SIDE_BAND_64K,
  107. CAPABILITY_THIN_PACK,
  108. COMMAND_DEEPEN,
  109. COMMAND_DONE,
  110. COMMAND_HAVE,
  111. COMMAND_SHALLOW,
  112. COMMAND_UNSHALLOW,
  113. COMMAND_WANT,
  114. MULTI_ACK,
  115. MULTI_ACK_DETAILED,
  116. NAK_LINE,
  117. PEELED_TAG_SUFFIX,
  118. SIDE_BAND_CHANNEL_DATA,
  119. SIDE_BAND_CHANNEL_FATAL,
  120. SIDE_BAND_CHANNEL_PROGRESS,
  121. SINGLE_ACK,
  122. TCP_GIT_PORT,
  123. BufferedPktLineWriter,
  124. Protocol,
  125. ReceivableProtocol,
  126. ack_type,
  127. capability_agent,
  128. capability_object_format,
  129. extract_capabilities,
  130. extract_want_line_capabilities,
  131. format_ack_line,
  132. format_ref_line,
  133. format_shallow_line,
  134. format_unshallow_line,
  135. symref_capabilities,
  136. write_info_refs,
  137. )
  138. from .refs import Ref, RefsContainer
  139. from .repo import Repo
  140. logger = log_utils.getLogger(__name__)
  141. class Backend:
  142. """A backend for the Git smart server implementation."""
  143. def open_repository(self, path: str) -> "BackendRepo":
  144. """Open the repository at a path.
  145. Args:
  146. path: Path to the repository
  147. Raises:
  148. NotGitRepository: no git repository was found at path
  149. Returns: Instance of BackendRepo
  150. """
  151. raise NotImplementedError(self.open_repository)
  152. class BackendRepo(TypingProtocol):
  153. """Repository abstraction used by the Git server.
  154. The methods required here are a subset of those provided by
  155. dulwich.repo.Repo.
  156. """
  157. object_store: PackBasedObjectStore
  158. refs: RefsContainer
  159. object_format: "ObjectFormat"
  160. def get_refs(self) -> dict[bytes, bytes]:
  161. """Get all the refs in the repository.
  162. Returns: dict of name -> sha
  163. """
  164. raise NotImplementedError
  165. def get_peeled(self, name: bytes) -> ObjectID | None:
  166. """Return the cached peeled value of a ref, if available.
  167. Args:
  168. name: Name of the ref to peel
  169. Returns: The peeled value of the ref. If the ref is known not point to
  170. a tag, this will be the SHA the ref refers to. If no cached
  171. information about a tag is available, this method may return None,
  172. but it should attempt to peel the tag if possible.
  173. """
  174. return None
  175. def find_missing_objects(
  176. self,
  177. determine_wants: Callable[[Mapping[Ref, ObjectID], int | None], list[ObjectID]],
  178. graph_walker: "_ProtocolGraphWalker",
  179. progress: Callable[[bytes], None] | None,
  180. *,
  181. get_tagged: Callable[[], dict[ObjectID, ObjectID]] | None = None,
  182. depth: int | None = None,
  183. ) -> "MissingObjectFinder | None":
  184. """Yield the objects required for a list of commits.
  185. Args:
  186. determine_wants: Function to determine which objects the client wants
  187. graph_walker: Object used to walk the commit graph
  188. progress: is a callback to send progress messages to the client
  189. get_tagged: Function that returns a dict of pointed-to sha ->
  190. tag sha for including tags.
  191. depth: Maximum depth of commits to fetch for shallow clones
  192. """
  193. raise NotImplementedError
  194. class DictBackend(Backend):
  195. """Trivial backend that looks up Git repositories in a dictionary."""
  196. def __init__(
  197. self, repos: dict[bytes, "BackendRepo"] | dict[str, "BackendRepo"]
  198. ) -> None:
  199. """Initialize a DictBackend.
  200. Args:
  201. repos: Dictionary mapping repository paths to BackendRepo instances
  202. """
  203. self.repos = repos
  204. def open_repository(self, path: str) -> BackendRepo:
  205. """Open repository at given path.
  206. Args:
  207. path: Path to the repository
  208. Returns:
  209. Repository object
  210. Raises:
  211. NotGitRepository: If no repository found at path
  212. """
  213. logger.debug("Opening repository at %s", path)
  214. # Handle both str and bytes keys for backward compatibility
  215. if path in self.repos:
  216. return self.repos[path] # type: ignore
  217. # Try converting between str and bytes
  218. if isinstance(path, bytes):
  219. try:
  220. alt_path = path.decode("utf-8")
  221. if alt_path in self.repos:
  222. return self.repos[alt_path]
  223. except UnicodeDecodeError:
  224. pass
  225. else:
  226. alt_path_bytes = path.encode("utf-8")
  227. if alt_path_bytes in self.repos:
  228. return self.repos[alt_path_bytes] # type: ignore
  229. raise NotGitRepository(
  230. "No git repository was found at {path}".format(**dict(path=path))
  231. )
  232. class FileSystemBackend(Backend):
  233. """Simple backend looking up Git repositories in the local file system."""
  234. def __init__(self, root: str = os.sep) -> None:
  235. """Initialize a FileSystemBackend.
  236. Args:
  237. root: Root directory to serve repositories from
  238. """
  239. super().__init__()
  240. self.root = (os.path.abspath(root) + os.sep).replace(os.sep * 2, os.sep)
  241. def open_repository(self, path: str) -> BackendRepo:
  242. """Open a repository from the filesystem.
  243. Args:
  244. path: Path to the repository relative to the root
  245. Returns: Repo instance
  246. Raises:
  247. NotGitRepository: If path is outside the root or not a git repository
  248. """
  249. logger.debug("opening repository at %s", path)
  250. # Ensure path is a string to avoid TypeError when joining with self.root
  251. path = os.fspath(path)
  252. if isinstance(path, bytes):
  253. path = os.fsdecode(path)
  254. abspath = os.path.abspath(os.path.join(self.root, path)) + os.sep
  255. normcase_abspath = os.path.normcase(abspath)
  256. normcase_root = os.path.normcase(self.root)
  257. if not normcase_abspath.startswith(normcase_root):
  258. raise NotGitRepository(f"Path {path!r} not inside root {self.root!r}")
  259. return Repo(abspath) # type: ignore[return-value]
  260. class Handler:
  261. """Smart protocol command handler base class."""
  262. def __init__(
  263. self, backend: Backend, proto: Protocol, stateless_rpc: bool = False
  264. ) -> None:
  265. """Initialize a Handler.
  266. Args:
  267. backend: Backend instance for repository access
  268. proto: Protocol instance for communication
  269. stateless_rpc: Whether this is a stateless RPC session
  270. """
  271. self.backend = backend
  272. self.proto = proto
  273. self.stateless_rpc = stateless_rpc
  274. def handle(self) -> None:
  275. """Handle a request."""
  276. raise NotImplementedError(self.handle)
  277. class PackHandler(Handler):
  278. """Protocol handler for packs."""
  279. def __init__(
  280. self, backend: Backend, proto: Protocol, stateless_rpc: bool = False
  281. ) -> None:
  282. """Initialize a PackHandler.
  283. Args:
  284. backend: Backend instance for repository access
  285. proto: Protocol instance for communication
  286. stateless_rpc: Whether this is a stateless RPC session
  287. """
  288. super().__init__(backend, proto, stateless_rpc)
  289. self._client_capabilities: set[bytes] | None = None
  290. # Flags needed for the no-done capability
  291. self._done_received = False
  292. self.advertise_refs = False
  293. def capabilities(self) -> Iterable[bytes]:
  294. """Return a list of capabilities supported by this handler."""
  295. raise NotImplementedError(self.capabilities)
  296. @classmethod
  297. def innocuous_capabilities(cls) -> Iterable[bytes]:
  298. """Return capabilities that don't affect protocol behavior.
  299. Returns:
  300. List of innocuous capability names
  301. """
  302. return [
  303. CAPABILITY_INCLUDE_TAG,
  304. CAPABILITY_THIN_PACK,
  305. CAPABILITY_NO_PROGRESS,
  306. CAPABILITY_OFS_DELTA,
  307. capability_agent(),
  308. ]
  309. @classmethod
  310. def required_capabilities(cls) -> Iterable[bytes]:
  311. """Return a list of capabilities that we require the client to have."""
  312. return []
  313. def set_client_capabilities(self, caps: Iterable[bytes]) -> None:
  314. """Set the client capabilities and validate them.
  315. Args:
  316. caps: List of capabilities requested by the client
  317. Raises:
  318. GitProtocolError: If client requests unsupported capability or lacks required ones
  319. """
  320. allowable_caps = set(self.innocuous_capabilities())
  321. allowable_caps.update(self.capabilities())
  322. for cap in caps:
  323. if cap.startswith(CAPABILITY_AGENT + b"="):
  324. continue
  325. if cap not in allowable_caps:
  326. raise GitProtocolError(
  327. f"Client asked for capability {cap!r} that was not advertised."
  328. )
  329. for cap in self.required_capabilities():
  330. if cap not in caps:
  331. raise GitProtocolError(
  332. f"Client does not support required capability {cap!r}."
  333. )
  334. self._client_capabilities = set(caps)
  335. logger.info("Client capabilities: %s", caps)
  336. def has_capability(self, cap: bytes) -> bool:
  337. """Check if the client supports a specific capability.
  338. Args:
  339. cap: Capability name to check
  340. Returns: True if the client supports the capability
  341. Raises:
  342. GitProtocolError: If called before client capabilities are set
  343. """
  344. if self._client_capabilities is None:
  345. raise GitProtocolError(
  346. f"Server attempted to access capability {cap!r} before asking client"
  347. )
  348. return cap in self._client_capabilities
  349. def notify_done(self) -> None:
  350. """Notify that the 'done' command has been received from the client."""
  351. self._done_received = True
  352. class UploadPackHandler(PackHandler):
  353. """Protocol handler for uploading a pack to the client."""
  354. def __init__(
  355. self,
  356. backend: Backend,
  357. args: Sequence[str],
  358. proto: Protocol,
  359. stateless_rpc: bool = False,
  360. advertise_refs: bool = False,
  361. ) -> None:
  362. """Initialize an UploadPackHandler.
  363. Args:
  364. backend: Backend instance for repository access
  365. args: Command arguments (first arg is repository path)
  366. proto: Protocol instance for communication
  367. stateless_rpc: Whether this is a stateless RPC session
  368. advertise_refs: Whether to advertise refs
  369. """
  370. super().__init__(backend, proto, stateless_rpc=stateless_rpc)
  371. self.repo = backend.open_repository(args[0])
  372. self._graph_walker = None
  373. self.advertise_refs = advertise_refs
  374. # A state variable for denoting that the have list is still
  375. # being processed, and the client is not accepting any other
  376. # data (such as side-band, see the progress method here).
  377. self._processing_have_lines = False
  378. def capabilities(self) -> list[bytes]:
  379. """Return the list of capabilities supported by upload-pack.
  380. Returns:
  381. List of capabilities including object-format for the repository
  382. """
  383. return [
  384. CAPABILITY_MULTI_ACK_DETAILED,
  385. CAPABILITY_MULTI_ACK,
  386. CAPABILITY_SIDE_BAND_64K,
  387. CAPABILITY_THIN_PACK,
  388. CAPABILITY_OFS_DELTA,
  389. CAPABILITY_NO_PROGRESS,
  390. CAPABILITY_INCLUDE_TAG,
  391. CAPABILITY_SHALLOW,
  392. CAPABILITY_NO_DONE,
  393. capability_object_format(self.repo.object_format.name),
  394. ]
  395. @classmethod
  396. def required_capabilities(cls) -> tuple[bytes, ...]:
  397. """Return the list of capabilities required for upload-pack."""
  398. return (
  399. CAPABILITY_SIDE_BAND_64K,
  400. CAPABILITY_THIN_PACK,
  401. CAPABILITY_OFS_DELTA,
  402. )
  403. def progress(self, message: bytes) -> None:
  404. """Send a progress message to the client.
  405. Args:
  406. message: Progress message to send
  407. """
  408. def _start_pack_send_phase(self) -> None:
  409. """Start the pack sending phase, setting up sideband if supported."""
  410. if self.has_capability(CAPABILITY_SIDE_BAND_64K):
  411. # The provided haves are processed, and it is safe to send side-
  412. # band data now.
  413. if not self.has_capability(CAPABILITY_NO_PROGRESS):
  414. self.progress = partial( # type: ignore
  415. self.proto.write_sideband, SIDE_BAND_CHANNEL_PROGRESS
  416. )
  417. self.write_pack_data: Callable[[bytes], None] = partial(
  418. self.proto.write_sideband, SIDE_BAND_CHANNEL_DATA
  419. )
  420. else:
  421. # proto.write returns Optional[int], but we need to treat it as returning None
  422. # for compatibility with write_pack_from_container
  423. def write_data(data: bytes) -> None:
  424. self.proto.write(data)
  425. self.write_pack_data = write_data
  426. def get_tagged(
  427. self,
  428. refs: Mapping[bytes, bytes] | None = None,
  429. repo: BackendRepo | None = None,
  430. ) -> dict[ObjectID, ObjectID]:
  431. """Get a dict of peeled values of tags to their original tag shas.
  432. Args:
  433. refs: dict of refname -> sha of possible tags; defaults to all
  434. of the backend's refs.
  435. repo: optional Repo instance for getting peeled refs; defaults
  436. to the backend's repo, if available
  437. Returns: dict of peeled_sha -> tag_sha, where tag_sha is the sha of a
  438. tag whose peeled value is peeled_sha.
  439. """
  440. if not self.has_capability(CAPABILITY_INCLUDE_TAG):
  441. return {}
  442. if refs is None:
  443. refs = self.repo.get_refs()
  444. if repo is None:
  445. repo = getattr(self.repo, "repo", None)
  446. if repo is None:
  447. # Bail if we don't have a Repo available; this is ok since
  448. # clients must be able to handle if the server doesn't include
  449. # all relevant tags.
  450. # TODO: fix behavior when missing
  451. return {}
  452. # TODO(jelmer): Integrate this with the refs logic in
  453. # Repo.find_missing_objects
  454. tagged = {}
  455. for name, sha in refs.items():
  456. peeled_sha = repo.get_peeled(name)
  457. if peeled_sha is not None and peeled_sha != ObjectID(sha):
  458. tagged[peeled_sha] = ObjectID(sha)
  459. return tagged
  460. def handle(self) -> None:
  461. """Handle an upload-pack request.
  462. This method processes the client's wants and haves, determines which
  463. objects to send, and writes the pack data to the client.
  464. """
  465. # Note the fact that client is only processing responses related
  466. # to the have lines it sent, and any other data (including side-
  467. # band) will be be considered a fatal error.
  468. self._processing_have_lines = True
  469. graph_walker = _ProtocolGraphWalker(
  470. self,
  471. self.repo.object_store,
  472. self.repo.get_peeled,
  473. self.repo.refs.get_symrefs,
  474. )
  475. wants: list[ObjectID] = []
  476. def wants_wrapper(
  477. refs: Mapping[Ref, ObjectID], depth: int | None = None
  478. ) -> list[ObjectID]:
  479. wants.extend(graph_walker.determine_wants(refs, depth))
  480. return wants
  481. missing_objects = self.repo.find_missing_objects(
  482. wants_wrapper,
  483. graph_walker,
  484. self.progress,
  485. get_tagged=self.get_tagged,
  486. )
  487. # Did the process short-circuit (e.g. in a stateless RPC call)? Note
  488. # that the client still expects a 0-object pack in most cases.
  489. # Also, if it also happens that the object_iter is instantiated
  490. # with a graph walker with an implementation that talks over the
  491. # wire (which is this instance of this class) this will actually
  492. # iterate through everything and write things out to the wire.
  493. if len(wants) == 0:
  494. return
  495. # Handle shallow clone case where missing_objects can be None
  496. if missing_objects is None:
  497. return
  498. object_ids = list(missing_objects)
  499. if not graph_walker.handle_done(
  500. not self.has_capability(CAPABILITY_NO_DONE), self._done_received
  501. ):
  502. return
  503. self._start_pack_send_phase()
  504. self.progress((f"counting objects: {len(object_ids)}, done.\n").encode("ascii"))
  505. write_pack_from_container(
  506. self.write_pack_data,
  507. self.repo.object_store,
  508. object_ids,
  509. object_format=self.repo.object_format,
  510. )
  511. # we are done
  512. self.proto.write_pkt_line(None)
  513. def _split_proto_line(
  514. line: bytes | None, allowed: Iterable[bytes | None] | None
  515. ) -> tuple[bytes | None, bytes | int | None]:
  516. """Split a line read from the wire.
  517. Args:
  518. line: The line read from the wire.
  519. allowed: An iterable of command names that should be allowed.
  520. Command names not listed below as possible return values will be
  521. ignored. If None, any commands from the possible return values are
  522. allowed.
  523. Returns: a tuple having one of the following forms:
  524. ('want', obj_id)
  525. ('have', obj_id)
  526. ('done', None)
  527. (None, None) (for a flush-pkt)
  528. Raises:
  529. UnexpectedCommandError: if the line cannot be parsed into one of the
  530. allowed return values.
  531. """
  532. if not line:
  533. fields: list[bytes | None] = [None]
  534. else:
  535. fields = list(line.rstrip(b"\n").split(b" ", 1))
  536. command = fields[0]
  537. if allowed is not None and command not in allowed:
  538. raise UnexpectedCommandError(command.decode("utf-8") if command else None)
  539. if len(fields) == 1 and command in (COMMAND_DONE, None):
  540. return (command, None)
  541. elif len(fields) == 2:
  542. if command in (
  543. COMMAND_WANT,
  544. COMMAND_HAVE,
  545. COMMAND_SHALLOW,
  546. COMMAND_UNSHALLOW,
  547. ):
  548. assert fields[1] is not None
  549. if not valid_hexsha(fields[1]):
  550. raise GitProtocolError("Invalid sha")
  551. return (command, fields[1])
  552. elif command == COMMAND_DEEPEN:
  553. assert fields[1] is not None
  554. return command, int(fields[1])
  555. raise GitProtocolError(f"Received invalid line from client: {line!r}")
  556. def _want_satisfied(
  557. store: ObjectContainer, haves: set[ObjectID], want: ObjectID, earliest: int
  558. ) -> bool:
  559. """Check if a specific want is satisfied by a set of haves.
  560. Args:
  561. store: Object store to retrieve objects from
  562. haves: Set of commit IDs the client has
  563. want: Commit ID the client wants
  564. earliest: Earliest commit time to consider
  565. Returns: True if the want is satisfied by the haves
  566. """
  567. o = store[want]
  568. pending = deque([o])
  569. known = {want}
  570. while pending:
  571. commit = pending.popleft()
  572. if commit.id in haves:
  573. return True
  574. if not isinstance(commit, Commit):
  575. # non-commit wants are assumed to be satisfied
  576. continue
  577. for parent in commit.parents:
  578. if parent in known:
  579. continue
  580. known.add(parent)
  581. parent_obj = store[parent]
  582. assert isinstance(parent_obj, Commit)
  583. # TODO: handle parents with later commit times than children
  584. if parent_obj.commit_time >= earliest:
  585. pending.append(parent_obj)
  586. return False
  587. def _all_wants_satisfied(
  588. store: ObjectContainer, haves: AbstractSet[ObjectID], wants: set[ObjectID]
  589. ) -> bool:
  590. """Check whether all the current wants are satisfied by a set of haves.
  591. Args:
  592. store: Object store to retrieve objects from
  593. haves: A set of commits we know the client has.
  594. wants: A set of commits the client wants
  595. Note: Wants are specified with set_wants rather than passed in since
  596. in the current interface they are determined outside this class.
  597. """
  598. haves = set(haves)
  599. if haves:
  600. have_objs = [store[h] for h in haves]
  601. earliest = min([h.commit_time for h in have_objs if isinstance(h, Commit)])
  602. else:
  603. earliest = 0
  604. for want in wants:
  605. if not _want_satisfied(store, haves, want, earliest):
  606. return False
  607. return True
  608. class AckGraphWalkerImpl:
  609. """Base class for acknowledgment graph walker implementations."""
  610. def __init__(self, graph_walker: "_ProtocolGraphWalker") -> None:
  611. """Initialize acknowledgment graph walker.
  612. Args:
  613. graph_walker: Graph walker to wrap
  614. """
  615. raise NotImplementedError
  616. def ack(self, have_ref: ObjectID) -> None:
  617. """Acknowledge a have reference.
  618. Args:
  619. have_ref: Object ID to acknowledge
  620. """
  621. raise NotImplementedError
  622. def handle_done(self, done_required: bool, done_received: bool) -> bool:
  623. """Handle 'done' packet from client."""
  624. raise NotImplementedError
  625. class _ProtocolGraphWalker:
  626. """A graph walker that knows the git protocol.
  627. As a graph walker, this class implements ack(), next(), and reset(). It
  628. also contains some base methods for interacting with the wire and walking
  629. the commit tree.
  630. The work of determining which acks to send is passed on to the
  631. implementation instance stored in _impl. The reason for this is that we do
  632. not know at object creation time what ack level the protocol requires. A
  633. call to set_ack_type() is required to set up the implementation, before
  634. any calls to next() or ack() are made.
  635. """
  636. def __init__(
  637. self,
  638. handler: PackHandler,
  639. object_store: ObjectContainer,
  640. get_peeled: Callable[[bytes], ObjectID | None],
  641. get_symrefs: Callable[[], dict[Ref, Ref]],
  642. ) -> None:
  643. """Initialize a ProtocolGraphWalker.
  644. Args:
  645. handler: Protocol handler instance
  646. object_store: Object store for retrieving objects
  647. get_peeled: Function to get peeled refs
  648. get_symrefs: Function to get symbolic refs
  649. """
  650. self.handler = handler
  651. self.store: ObjectContainer = object_store
  652. self.get_peeled = get_peeled
  653. self.get_symrefs = get_symrefs
  654. self.proto = handler.proto
  655. self.stateless_rpc = handler.stateless_rpc
  656. self.advertise_refs = handler.advertise_refs
  657. self._wants: list[ObjectID] = []
  658. self.shallow: set[ObjectID] = set()
  659. self.client_shallow: set[ObjectID] = set()
  660. self.unshallow: set[ObjectID] = set()
  661. self._cached = False
  662. self._cache: list[ObjectID] = []
  663. self._cache_index = 0
  664. self._impl: AckGraphWalkerImpl | None = None
  665. def determine_wants(
  666. self, heads: Mapping[Ref, ObjectID], depth: int | None = None
  667. ) -> list[ObjectID]:
  668. """Determine the wants for a set of heads.
  669. The given heads are advertised to the client, who then specifies which
  670. refs they want using 'want' lines. This portion of the protocol is the
  671. same regardless of ack type, and in fact is used to set the ack type of
  672. the ProtocolGraphWalker.
  673. If the client has the 'shallow' capability, this method also reads and
  674. responds to the 'shallow' and 'deepen' lines from the client. These are
  675. not part of the wants per se, but they set up necessary state for
  676. walking the graph. Additionally, later code depends on this method
  677. consuming everything up to the first 'have' line.
  678. Args:
  679. heads: a dict of refname->SHA1 to advertise
  680. depth: Maximum depth for shallow clones
  681. Returns: a list of SHA1s requested by the client
  682. """
  683. symrefs = self.get_symrefs()
  684. values = set(heads.values())
  685. if self.advertise_refs or not self.stateless_rpc:
  686. for i, (ref, sha) in enumerate(sorted(heads.items())):
  687. try:
  688. peeled_sha = self.get_peeled(ref)
  689. except KeyError:
  690. # Skip refs that are inaccessible
  691. # TODO(jelmer): Integrate with Repo.find_missing_objects refs
  692. # logic.
  693. continue
  694. if i == 0:
  695. logger.info("Sending capabilities: %s", self.handler.capabilities())
  696. line = format_ref_line(
  697. ref,
  698. sha,
  699. list(self.handler.capabilities())
  700. + symref_capabilities(symrefs.items()),
  701. )
  702. else:
  703. line = format_ref_line(ref, sha)
  704. self.proto.write_pkt_line(line)
  705. if peeled_sha is not None and peeled_sha != sha:
  706. self.proto.write_pkt_line(
  707. format_ref_line(ref + PEELED_TAG_SUFFIX, peeled_sha)
  708. )
  709. # i'm done..
  710. self.proto.write_pkt_line(None)
  711. if self.advertise_refs:
  712. return []
  713. # Now client will sending want want want commands
  714. want = self.proto.read_pkt_line()
  715. if not want:
  716. return []
  717. line, caps = extract_want_line_capabilities(want)
  718. self.handler.set_client_capabilities(caps)
  719. self.set_ack_type(ack_type(caps))
  720. allowed = (COMMAND_WANT, COMMAND_SHALLOW, COMMAND_DEEPEN, None)
  721. command, sha_result = _split_proto_line(line, allowed)
  722. want_revs: list[ObjectID] = []
  723. while command == COMMAND_WANT:
  724. assert isinstance(sha_result, bytes)
  725. if sha_result not in values:
  726. raise GitProtocolError(f"Client wants invalid object {sha_result!r}")
  727. want_revs.append(ObjectID(sha_result))
  728. command, sha_result = self.read_proto_line(allowed)
  729. self.set_wants(want_revs)
  730. if command in (COMMAND_SHALLOW, COMMAND_DEEPEN):
  731. assert sha_result is not None
  732. self.unread_proto_line(command, sha_result)
  733. self._handle_shallow_request(want_revs)
  734. if self.stateless_rpc and self.proto.eof():
  735. # The client may close the socket at this point, expecting a
  736. # flush-pkt from the server. We might be ready to send a packfile
  737. # at this point, so we need to explicitly short-circuit in this
  738. # case.
  739. return []
  740. return want_revs
  741. def unread_proto_line(self, command: bytes, value: bytes | int) -> None:
  742. """Push a command back to be read again.
  743. Args:
  744. command: Command name
  745. value: Command value
  746. """
  747. if isinstance(value, int):
  748. value = str(value).encode("ascii")
  749. self.proto.unread_pkt_line(command + b" " + value)
  750. def nak(self) -> None:
  751. """Send a NAK response."""
  752. def ack(self, have_ref: ObjectID) -> None:
  753. """Acknowledge a have reference.
  754. Args:
  755. have_ref: SHA to acknowledge (40 bytes hex for SHA-1, 64 bytes for SHA-256)
  756. Raises:
  757. ValueError: If have_ref is not a valid length
  758. """
  759. if len(have_ref) not in (40, 64):
  760. raise ValueError(f"invalid sha {have_ref!r}")
  761. assert self._impl is not None
  762. return self._impl.ack(have_ref)
  763. def reset(self) -> None:
  764. """Reset the graph walker cache."""
  765. self._cached = True
  766. self._cache_index = 0
  767. def next(self) -> ObjectID | None:
  768. """Get the next SHA from the graph walker.
  769. Returns: Next SHA or None if done
  770. """
  771. if not self._cached:
  772. if not self._impl and self.stateless_rpc:
  773. return None
  774. assert self._impl is not None
  775. return next(self._impl) # type: ignore[call-overload, no-any-return]
  776. self._cache_index += 1
  777. if self._cache_index > len(self._cache):
  778. return None
  779. return self._cache[self._cache_index]
  780. __next__ = next
  781. def read_proto_line(
  782. self, allowed: Iterable[bytes | None] | None
  783. ) -> tuple[bytes | None, bytes | int | None]:
  784. """Read a line from the wire.
  785. Args:
  786. allowed: An iterable of command names that should be allowed.
  787. Returns: A tuple of (command, value); see _split_proto_line.
  788. Raises:
  789. UnexpectedCommandError: If an error occurred reading the line.
  790. """
  791. return _split_proto_line(self.proto.read_pkt_line(), allowed)
  792. def _handle_shallow_request(self, wants: Sequence[ObjectID]) -> None:
  793. """Handle shallow clone requests from the client.
  794. Args:
  795. wants: List of wanted object SHAs
  796. """
  797. while True:
  798. command, val = self.read_proto_line((COMMAND_DEEPEN, COMMAND_SHALLOW))
  799. if command == COMMAND_DEEPEN:
  800. assert isinstance(val, int)
  801. depth = val
  802. break
  803. assert isinstance(val, bytes)
  804. self.client_shallow.add(ObjectID(val))
  805. self.read_proto_line((None,)) # consume client's flush-pkt
  806. shallow, not_shallow = find_shallow(self.store, wants, depth)
  807. # Update self.shallow instead of reassigning it since we passed a
  808. # reference to it before this method was called.
  809. self.shallow.update(shallow - not_shallow)
  810. new_shallow = self.shallow - self.client_shallow
  811. unshallow = self.unshallow = not_shallow & self.client_shallow
  812. self.update_shallow(new_shallow, unshallow)
  813. def update_shallow(
  814. self, new_shallow: AbstractSet[ObjectID], unshallow: AbstractSet[ObjectID]
  815. ) -> None:
  816. """Update shallow/unshallow information to the client.
  817. Args:
  818. new_shallow: Set of newly shallow commits
  819. unshallow: Set of commits to unshallow
  820. """
  821. for sha in sorted(new_shallow):
  822. self.proto.write_pkt_line(format_shallow_line(sha))
  823. for sha in sorted(unshallow):
  824. self.proto.write_pkt_line(format_unshallow_line(sha))
  825. self.proto.write_pkt_line(None)
  826. def notify_done(self) -> None:
  827. """Notify that the client sent 'done'."""
  828. # relay the message down to the handler.
  829. self.handler.notify_done()
  830. def send_ack(self, sha: ObjectID, ack_type: bytes = b"") -> None:
  831. """Send an ACK to the client.
  832. Args:
  833. sha: SHA to acknowledge
  834. ack_type: Type of ACK (e.g., b'continue', b'ready')
  835. """
  836. self.proto.write_pkt_line(format_ack_line(sha, ack_type))
  837. def send_nak(self) -> None:
  838. """Send a NAK to the client."""
  839. self.proto.write_pkt_line(NAK_LINE)
  840. def handle_done(self, done_required: bool, done_received: bool) -> bool:
  841. """Handle the 'done' command.
  842. Args:
  843. done_required: Whether done is required
  844. done_received: Whether done was received
  845. Returns: True if done handling succeeded
  846. """
  847. # Delegate this to the implementation.
  848. assert self._impl is not None
  849. return self._impl.handle_done(done_required, done_received)
  850. def set_wants(self, wants: list[ObjectID]) -> None:
  851. """Set the list of wanted objects.
  852. Args:
  853. wants: List of wanted object SHAs
  854. """
  855. self._wants = wants
  856. def all_wants_satisfied(self, haves: AbstractSet[ObjectID]) -> bool:
  857. """Check whether all the current wants are satisfied by a set of haves.
  858. Args:
  859. haves: A set of commits we know the client has.
  860. Note: Wants are specified with set_wants rather than passed in since
  861. in the current interface they are determined outside this class.
  862. """
  863. return _all_wants_satisfied(self.store, haves, set(self._wants))
  864. def set_ack_type(self, ack_type: int) -> None:
  865. """Set the acknowledgment type for the graph walker.
  866. Args:
  867. ack_type: One of SINGLE_ACK, MULTI_ACK, or MULTI_ACK_DETAILED
  868. """
  869. impl_classes: dict[int, type[AckGraphWalkerImpl]] = {
  870. MULTI_ACK: MultiAckGraphWalkerImpl,
  871. MULTI_ACK_DETAILED: MultiAckDetailedGraphWalkerImpl,
  872. SINGLE_ACK: SingleAckGraphWalkerImpl,
  873. }
  874. self._impl = impl_classes[ack_type](self)
  875. _GRAPH_WALKER_COMMANDS = (COMMAND_HAVE, COMMAND_DONE, None)
  876. class SingleAckGraphWalkerImpl(AckGraphWalkerImpl):
  877. """Graph walker implementation that speaks the single-ack protocol."""
  878. def __init__(self, walker: "_ProtocolGraphWalker") -> None:
  879. """Initialize a SingleAckGraphWalkerImpl.
  880. Args:
  881. walker: Parent ProtocolGraphWalker instance
  882. """
  883. self.walker = walker
  884. self._common: list[ObjectID] = []
  885. def ack(self, have_ref: ObjectID) -> None:
  886. """Acknowledge a have reference.
  887. Args:
  888. have_ref: Object ID to acknowledge
  889. """
  890. if not self._common:
  891. self.walker.send_ack(have_ref)
  892. self._common.append(have_ref)
  893. def next(self) -> ObjectID | None:
  894. """Get next SHA from graph walker.
  895. Returns:
  896. SHA bytes or None if done
  897. """
  898. command, sha = self.walker.read_proto_line(_GRAPH_WALKER_COMMANDS)
  899. if command in (None, COMMAND_DONE):
  900. # defer the handling of done
  901. self.walker.notify_done()
  902. return None
  903. elif command == COMMAND_HAVE:
  904. assert isinstance(sha, bytes)
  905. return ObjectID(sha)
  906. return None
  907. __next__ = next
  908. def handle_done(self, done_required: bool, done_received: bool) -> bool:
  909. """Handle done command.
  910. Args:
  911. done_required: Whether done is required
  912. done_received: Whether done was received
  913. Returns:
  914. True if handling completed successfully
  915. """
  916. if not self._common:
  917. self.walker.send_nak()
  918. if done_required and not done_received:
  919. # we are not done, especially when done is required; skip
  920. # the pack for this request and especially do not handle
  921. # the done.
  922. return False
  923. if not done_received and not self._common:
  924. # Okay we are not actually done then since the walker picked
  925. # up no haves. This is usually triggered when client attempts
  926. # to pull from a source that has no common base_commit.
  927. # See: test_server.MultiAckDetailedGraphWalkerImplTestCase.\
  928. # test_multi_ack_stateless_nodone
  929. return False
  930. return True
  931. class MultiAckGraphWalkerImpl(AckGraphWalkerImpl):
  932. """Graph walker implementation that speaks the multi-ack protocol."""
  933. def __init__(self, walker: "_ProtocolGraphWalker") -> None:
  934. """Initialize multi-ack graph walker.
  935. Args:
  936. walker: Parent ProtocolGraphWalker instance
  937. """
  938. self.walker = walker
  939. self._found_base = False
  940. self._common: list[ObjectID] = []
  941. def ack(self, have_ref: ObjectID) -> None:
  942. """Acknowledge a have reference.
  943. Args:
  944. have_ref: Object ID to acknowledge
  945. """
  946. self._common.append(have_ref)
  947. if not self._found_base:
  948. self.walker.send_ack(have_ref, b"continue")
  949. if self.walker.all_wants_satisfied(set(self._common)):
  950. self._found_base = True
  951. # else we blind ack within next
  952. def next(self) -> ObjectID | None:
  953. """Get next SHA from graph walker.
  954. Returns:
  955. SHA bytes or None if done
  956. """
  957. while True:
  958. command, sha = self.walker.read_proto_line(_GRAPH_WALKER_COMMANDS)
  959. if command is None:
  960. self.walker.send_nak()
  961. # in multi-ack mode, a flush-pkt indicates the client wants to
  962. # flush but more have lines are still coming
  963. continue
  964. elif command == COMMAND_DONE:
  965. self.walker.notify_done()
  966. return None
  967. elif command == COMMAND_HAVE:
  968. assert isinstance(sha, bytes)
  969. sha_id = ObjectID(sha)
  970. if self._found_base:
  971. # blind ack
  972. self.walker.send_ack(sha_id, b"continue")
  973. return sha_id
  974. __next__ = next
  975. def handle_done(self, done_required: bool, done_received: bool) -> bool:
  976. """Handle done command.
  977. Args:
  978. done_required: Whether done is required
  979. done_received: Whether done was received
  980. Returns:
  981. True if handling completed successfully
  982. """
  983. if done_required and not done_received:
  984. # we are not done, especially when done is required; skip
  985. # the pack for this request and especially do not handle
  986. # the done.
  987. return False
  988. if not done_received and not self._common:
  989. # Okay we are not actually done then since the walker picked
  990. # up no haves. This is usually triggered when client attempts
  991. # to pull from a source that has no common base_commit.
  992. # See: test_server.MultiAckDetailedGraphWalkerImplTestCase.\
  993. # test_multi_ack_stateless_nodone
  994. return False
  995. # don't nak unless no common commits were found, even if not
  996. # everything is satisfied
  997. if self._common:
  998. self.walker.send_ack(self._common[-1])
  999. else:
  1000. self.walker.send_nak()
  1001. return True
  1002. class MultiAckDetailedGraphWalkerImpl(AckGraphWalkerImpl):
  1003. """Graph walker implementation speaking the multi-ack-detailed protocol."""
  1004. def __init__(self, walker: "_ProtocolGraphWalker") -> None:
  1005. """Initialize multi-ack-detailed graph walker.
  1006. Args:
  1007. walker: Parent ProtocolGraphWalker instance
  1008. """
  1009. self.walker = walker
  1010. self._common: list[ObjectID] = []
  1011. def ack(self, have_ref: ObjectID) -> None:
  1012. """Acknowledge a have reference.
  1013. Args:
  1014. have_ref: Object ID to acknowledge
  1015. """
  1016. # Should only be called iff have_ref is common
  1017. self._common.append(have_ref)
  1018. self.walker.send_ack(have_ref, b"common")
  1019. def next(self) -> ObjectID | None:
  1020. """Get next SHA from graph walker.
  1021. Returns:
  1022. SHA bytes or None if done
  1023. """
  1024. while True:
  1025. command, sha = self.walker.read_proto_line(_GRAPH_WALKER_COMMANDS)
  1026. if command is None:
  1027. if self.walker.all_wants_satisfied(set(self._common)):
  1028. self.walker.send_ack(self._common[-1], b"ready")
  1029. self.walker.send_nak()
  1030. if self.walker.stateless_rpc:
  1031. # The HTTP version of this request a flush-pkt always
  1032. # signifies an end of request, so we also return
  1033. # nothing here as if we are done (but not really, as
  1034. # it depends on whether no-done capability was
  1035. # specified and that's handled in handle_done which
  1036. # may or may not call post_nodone_check depending on
  1037. # that).
  1038. return None
  1039. elif command == COMMAND_DONE:
  1040. # Let the walker know that we got a done.
  1041. self.walker.notify_done()
  1042. break
  1043. elif command == COMMAND_HAVE:
  1044. # return the sha and let the caller ACK it with the
  1045. # above ack method.
  1046. assert isinstance(sha, bytes)
  1047. return ObjectID(sha)
  1048. # don't nak unless no common commits were found, even if not
  1049. # everything is satisfied
  1050. return None
  1051. __next__ = next
  1052. def handle_done(self, done_required: bool, done_received: bool) -> bool:
  1053. """Handle done command.
  1054. Args:
  1055. done_required: Whether done is required
  1056. done_received: Whether done was received
  1057. Returns:
  1058. True if handling completed successfully
  1059. """
  1060. if done_required and not done_received:
  1061. # we are not done, especially when done is required; skip
  1062. # the pack for this request and especially do not handle
  1063. # the done.
  1064. return False
  1065. if not done_received and not self._common:
  1066. # Okay we are not actually done then since the walker picked
  1067. # up no haves. This is usually triggered when client attempts
  1068. # to pull from a source that has no common base_commit.
  1069. # See: test_server.MultiAckDetailedGraphWalkerImplTestCase.\
  1070. # test_multi_ack_stateless_nodone
  1071. return False
  1072. # don't nak unless no common commits were found, even if not
  1073. # everything is satisfied
  1074. if self._common:
  1075. self.walker.send_ack(self._common[-1])
  1076. else:
  1077. self.walker.send_nak()
  1078. return True
  1079. class ReceivePackHandler(PackHandler):
  1080. """Protocol handler for downloading a pack from the client."""
  1081. def __init__(
  1082. self,
  1083. backend: Backend,
  1084. args: Sequence[str],
  1085. proto: Protocol,
  1086. stateless_rpc: bool = False,
  1087. advertise_refs: bool = False,
  1088. ) -> None:
  1089. """Initialize receive-pack handler.
  1090. Args:
  1091. backend: Backend instance
  1092. args: Command arguments
  1093. proto: Protocol instance
  1094. stateless_rpc: Whether to use stateless RPC
  1095. advertise_refs: Whether to advertise refs
  1096. """
  1097. super().__init__(backend, proto, stateless_rpc=stateless_rpc)
  1098. self.repo = backend.open_repository(args[0])
  1099. self.advertise_refs = advertise_refs
  1100. def capabilities(self) -> Iterable[bytes]:
  1101. """Return supported capabilities.
  1102. Returns:
  1103. List of capability names including object-format for the repository
  1104. """
  1105. return [
  1106. CAPABILITY_REPORT_STATUS,
  1107. CAPABILITY_DELETE_REFS,
  1108. CAPABILITY_QUIET,
  1109. CAPABILITY_OFS_DELTA,
  1110. CAPABILITY_SIDE_BAND_64K,
  1111. CAPABILITY_NO_DONE,
  1112. capability_object_format(self.repo.object_format.name),
  1113. ]
  1114. def _apply_pack(
  1115. self, refs: list[tuple[ObjectID, ObjectID, Ref]]
  1116. ) -> Iterator[tuple[bytes, bytes]]:
  1117. """Apply received pack to repository.
  1118. Args:
  1119. refs: List of (old_sha, new_sha, ref_name) tuples
  1120. Yields:
  1121. Tuples of (ref_name, error_message) for any errors
  1122. """
  1123. all_exceptions = (
  1124. IOError,
  1125. OSError,
  1126. ChecksumMismatch,
  1127. ApplyDeltaError,
  1128. AssertionError,
  1129. socket.error,
  1130. zlib.error,
  1131. ObjectFormatException,
  1132. )
  1133. will_send_pack = False
  1134. zero_sha = ObjectID(b"0" * self.repo.object_format.hex_length)
  1135. for command in refs:
  1136. if command[1] != zero_sha:
  1137. will_send_pack = True
  1138. if will_send_pack:
  1139. # TODO: more informative error messages than just the exception
  1140. # string
  1141. try:
  1142. recv = getattr(self.proto, "recv", None)
  1143. self.repo.object_store.add_thin_pack(self.proto.read, recv) # type: ignore[attr-defined]
  1144. yield (b"unpack", b"ok")
  1145. except all_exceptions as e:
  1146. yield (b"unpack", str(e).replace("\n", "").encode("utf-8"))
  1147. # The pack may still have been moved in, but it may contain
  1148. # broken objects. We trust a later GC to clean it up.
  1149. else:
  1150. # The git protocol want to find a status entry related to unpack
  1151. # process even if no pack data has been sent.
  1152. yield (b"unpack", b"ok")
  1153. for oldsha, sha, ref in refs:
  1154. ref_status = b"ok"
  1155. try:
  1156. if sha == zero_sha:
  1157. if CAPABILITY_DELETE_REFS not in self.capabilities():
  1158. raise GitProtocolError(
  1159. "Attempted to delete refs without delete-refs capability."
  1160. )
  1161. try:
  1162. self.repo.refs.remove_if_equals(ref, oldsha)
  1163. except all_exceptions:
  1164. ref_status = b"failed to delete"
  1165. else:
  1166. try:
  1167. self.repo.refs.set_if_equals(ref, oldsha, sha)
  1168. except all_exceptions:
  1169. ref_status = b"failed to write"
  1170. except KeyError:
  1171. ref_status = b"bad ref"
  1172. yield (ref, ref_status)
  1173. def _report_status(self, status: Sequence[tuple[bytes, bytes]]) -> None:
  1174. """Report status to client.
  1175. Args:
  1176. status: List of (ref_name, status_message) tuples
  1177. """
  1178. if self.has_capability(CAPABILITY_SIDE_BAND_64K):
  1179. writer = BufferedPktLineWriter(
  1180. lambda d: self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, d)
  1181. )
  1182. write = writer.write
  1183. def flush() -> None:
  1184. writer.flush()
  1185. self.proto.write_pkt_line(None)
  1186. else:
  1187. write = self.proto.write_pkt_line # type: ignore[assignment]
  1188. def flush() -> None:
  1189. pass
  1190. for name, msg in status:
  1191. if name == b"unpack":
  1192. write(b"unpack " + msg + b"\n")
  1193. elif msg == b"ok":
  1194. write(b"ok " + name + b"\n")
  1195. else:
  1196. write(b"ng " + name + b" " + msg + b"\n")
  1197. write(None) # type: ignore
  1198. flush()
  1199. def _on_post_receive(self, client_refs: dict[bytes, tuple[bytes, bytes]]) -> None:
  1200. """Run post-receive hook.
  1201. Args:
  1202. client_refs: Dictionary of ref changes from client
  1203. """
  1204. hook = self.repo.hooks.get("post-receive", None) # type: ignore[attr-defined]
  1205. if not hook:
  1206. return
  1207. try:
  1208. output = hook.execute(client_refs)
  1209. if output:
  1210. self.proto.write_sideband(SIDE_BAND_CHANNEL_PROGRESS, output)
  1211. except HookError as err:
  1212. self.proto.write_sideband(SIDE_BAND_CHANNEL_FATAL, str(err).encode("utf-8"))
  1213. def handle(self) -> None:
  1214. """Handle receive-pack request."""
  1215. if self.advertise_refs or not self.stateless_rpc:
  1216. refs = sorted(self.repo.get_refs().items())
  1217. symrefs = sorted(self.repo.refs.get_symrefs().items())
  1218. if not refs:
  1219. zero_sha = ObjectID(b"0" * self.repo.object_format.hex_length)
  1220. refs = [(CAPABILITIES_REF, zero_sha)]
  1221. logger.info("Sending capabilities: %s", self.capabilities())
  1222. self.proto.write_pkt_line(
  1223. format_ref_line(
  1224. refs[0][0],
  1225. refs[0][1],
  1226. list(self.capabilities()) + symref_capabilities(symrefs),
  1227. )
  1228. )
  1229. for i in range(1, len(refs)):
  1230. ref = refs[i]
  1231. self.proto.write_pkt_line(format_ref_line(ref[0], ref[1]))
  1232. self.proto.write_pkt_line(None)
  1233. if self.advertise_refs:
  1234. return
  1235. client_refs = []
  1236. ref_line = self.proto.read_pkt_line()
  1237. # if ref is none then client doesn't want to send us anything..
  1238. if ref_line is None:
  1239. return
  1240. ref_line, caps = extract_capabilities(ref_line)
  1241. self.set_client_capabilities(caps)
  1242. # client will now send us a list of (oldsha, newsha, ref)
  1243. while ref_line:
  1244. (oldsha, newsha, ref_name) = ref_line.split()
  1245. client_refs.append((ObjectID(oldsha), ObjectID(newsha), Ref(ref_name)))
  1246. ref_line = self.proto.read_pkt_line()
  1247. # backend can now deal with this refs and read a pack using self.read
  1248. status = list(self._apply_pack(client_refs))
  1249. self._on_post_receive(client_refs) # type: ignore[arg-type]
  1250. # when we have read all the pack from the client, send a status report
  1251. # if the client asked for it
  1252. if self.has_capability(CAPABILITY_REPORT_STATUS):
  1253. self._report_status(status)
  1254. class UploadArchiveHandler(Handler):
  1255. """Handler for git-upload-archive requests."""
  1256. def __init__(
  1257. self,
  1258. backend: Backend,
  1259. args: Sequence[str],
  1260. proto: Protocol,
  1261. stateless_rpc: bool = False,
  1262. ) -> None:
  1263. """Initialize upload-archive handler.
  1264. Args:
  1265. backend: Backend instance
  1266. args: Command arguments
  1267. proto: Protocol instance
  1268. stateless_rpc: Whether to use stateless RPC
  1269. """
  1270. super().__init__(backend, proto, stateless_rpc)
  1271. self.repo = backend.open_repository(args[0])
  1272. def handle(self) -> None:
  1273. """Handle upload-archive request."""
  1274. def write(x: bytes) -> None:
  1275. self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, x)
  1276. arguments = []
  1277. for pkt in self.proto.read_pkt_seq():
  1278. (key, value) = pkt.split(b" ", 1)
  1279. if key != b"argument":
  1280. raise GitProtocolError(f"unknown command {key!r}")
  1281. arguments.append(value.rstrip(b"\n"))
  1282. prefix = b""
  1283. format = "tar"
  1284. i = 0
  1285. store: BaseObjectStore = self.repo.object_store
  1286. while i < len(arguments):
  1287. argument = arguments[i]
  1288. if argument == b"--prefix":
  1289. i += 1
  1290. prefix = arguments[i]
  1291. elif argument == b"--format":
  1292. i += 1
  1293. format = arguments[i].decode("ascii")
  1294. else:
  1295. commit_sha = self.repo.refs[Ref(argument)]
  1296. commit_obj = store[commit_sha]
  1297. assert isinstance(commit_obj, Commit)
  1298. tree_obj = store[commit_obj.tree]
  1299. assert isinstance(tree_obj, Tree)
  1300. tree = tree_obj
  1301. i += 1
  1302. self.proto.write_pkt_line(b"ACK")
  1303. self.proto.write_pkt_line(None)
  1304. for chunk in tar_stream(
  1305. store,
  1306. tree,
  1307. mtime=int(time.time()),
  1308. prefix=prefix,
  1309. format=format,
  1310. ):
  1311. write(chunk)
  1312. self.proto.write_pkt_line(None)
  1313. # Default handler classes for git services.
  1314. DEFAULT_HANDLERS = {
  1315. b"git-upload-pack": UploadPackHandler,
  1316. b"git-receive-pack": ReceivePackHandler,
  1317. b"git-upload-archive": UploadArchiveHandler,
  1318. }
  1319. class TCPGitRequestHandler(socketserver.StreamRequestHandler):
  1320. """TCP request handler for git protocol."""
  1321. def __init__(
  1322. self,
  1323. handlers: dict[bytes, type[Handler]],
  1324. request: socket.socket,
  1325. client_address: tuple[str, int],
  1326. server: socketserver.TCPServer,
  1327. ) -> None:
  1328. """Initialize TCP request handler.
  1329. Args:
  1330. handlers: Dictionary mapping commands to handler classes
  1331. request: Request socket
  1332. client_address: Client address tuple
  1333. server: Server instance
  1334. """
  1335. self.handlers = handlers
  1336. socketserver.StreamRequestHandler.__init__(
  1337. self, request, client_address, server
  1338. )
  1339. def handle(self) -> None:
  1340. """Handle TCP git request."""
  1341. proto = ReceivableProtocol(self.connection.recv, self.wfile.write)
  1342. command, args = proto.read_cmd()
  1343. logger.info("Handling %s request, args=%s", command, args)
  1344. cls = self.handlers.get(command, None)
  1345. if not callable(cls):
  1346. raise GitProtocolError(f"Invalid service {command!r}")
  1347. h = cls(self.server.backend, args, proto) # type: ignore
  1348. h.handle()
  1349. class TCPGitServer(socketserver.TCPServer):
  1350. """TCP server for git protocol."""
  1351. allow_reuse_address = True
  1352. serve = socketserver.TCPServer.serve_forever
  1353. def _make_handler(
  1354. self,
  1355. request: socket.socket,
  1356. client_address: tuple[str, int],
  1357. server: socketserver.TCPServer,
  1358. ) -> TCPGitRequestHandler:
  1359. """Create request handler instance.
  1360. Args:
  1361. request: Request socket
  1362. client_address: Client address tuple
  1363. server: Server instance
  1364. Returns:
  1365. TCPGitRequestHandler instance
  1366. """
  1367. return TCPGitRequestHandler(self.handlers, request, client_address, server)
  1368. def __init__(
  1369. self,
  1370. backend: Backend,
  1371. listen_addr: str,
  1372. port: int = TCP_GIT_PORT,
  1373. handlers: dict[bytes, type[Handler]] | None = None,
  1374. ) -> None:
  1375. """Initialize TCP git server.
  1376. Args:
  1377. backend: Backend instance
  1378. listen_addr: Address to listen on
  1379. port: Port to listen on (default: TCP_GIT_PORT)
  1380. handlers: Optional dictionary of custom handlers
  1381. """
  1382. self.handlers = dict(DEFAULT_HANDLERS)
  1383. if handlers is not None:
  1384. self.handlers.update(handlers)
  1385. self.backend = backend
  1386. logger.info("Listening for TCP connections on %s:%d", listen_addr, port)
  1387. socketserver.TCPServer.__init__(self, (listen_addr, port), self._make_handler)
  1388. def verify_request(
  1389. self,
  1390. request: socket.socket | tuple[bytes, socket.socket],
  1391. client_address: tuple[str, int] | socket.socket,
  1392. ) -> bool:
  1393. """Verify incoming request.
  1394. Args:
  1395. request: Request socket
  1396. client_address: Client address tuple
  1397. Returns:
  1398. True to accept request
  1399. """
  1400. logger.info("Handling request from %s", client_address)
  1401. return True
  1402. def handle_error(
  1403. self,
  1404. request: socket.socket | tuple[bytes, socket.socket],
  1405. client_address: tuple[str, int] | socket.socket,
  1406. ) -> None:
  1407. """Handle request processing errors.
  1408. Args:
  1409. request: Request socket
  1410. client_address: Client address tuple
  1411. """
  1412. logger.exception(
  1413. "Exception happened during processing of request from %s",
  1414. client_address,
  1415. )
  1416. def main(argv: list[str] = sys.argv) -> None:
  1417. """Entry point for starting a TCP git server."""
  1418. import optparse
  1419. parser = optparse.OptionParser()
  1420. parser.add_option(
  1421. "-l",
  1422. "--listen_address",
  1423. dest="listen_address",
  1424. default="localhost",
  1425. help="Binding IP address.",
  1426. )
  1427. parser.add_option(
  1428. "-p",
  1429. "--port",
  1430. dest="port",
  1431. type=int,
  1432. default=TCP_GIT_PORT,
  1433. help="Binding TCP port.",
  1434. )
  1435. options, args = parser.parse_args(argv)
  1436. log_utils.default_logging_config()
  1437. if len(args) > 1:
  1438. gitdir = args[1]
  1439. else:
  1440. gitdir = "."
  1441. # TODO(jelmer): Support git-daemon-export-ok and --export-all.
  1442. backend = FileSystemBackend(gitdir)
  1443. server = TCPGitServer(backend, options.listen_address, options.port)
  1444. server.serve_forever()
  1445. def serve_command(
  1446. handler_cls: type[Handler],
  1447. argv: list[str] = sys.argv,
  1448. backend: Backend | None = None,
  1449. inf: IO[bytes] | None = None,
  1450. outf: IO[bytes] | None = None,
  1451. ) -> int:
  1452. """Serve a single command.
  1453. This is mostly useful for the implementation of commands used by e.g.
  1454. git+ssh.
  1455. Args:
  1456. handler_cls: `Handler` class to use for the request
  1457. argv: execv-style command-line arguments. Defaults to sys.argv.
  1458. backend: `Backend` to use
  1459. inf: File-like object to read from, defaults to standard input.
  1460. outf: File-like object to write to, defaults to standard output.
  1461. Returns: Exit code for use with sys.exit. 0 on success, 1 on failure.
  1462. """
  1463. if backend is None:
  1464. backend = FileSystemBackend()
  1465. if inf is None:
  1466. inf = sys.stdin.buffer
  1467. if outf is None:
  1468. outf = sys.stdout.buffer
  1469. def send_fn(data: bytes) -> None:
  1470. outf.write(data)
  1471. outf.flush()
  1472. proto = Protocol(inf.read, send_fn)
  1473. handler = handler_cls(backend, argv[1:], proto) # type: ignore[arg-type]
  1474. # FIXME: Catch exceptions and write a single-line summary to outf.
  1475. handler.handle()
  1476. return 0
  1477. def generate_info_refs(repo: "BaseRepo") -> Iterator[bytes]:
  1478. """Generate an info refs file."""
  1479. refs = repo.get_refs()
  1480. return write_info_refs(refs, repo.object_store)
  1481. def generate_objects_info_packs(repo: "BaseRepo") -> Iterator[bytes]:
  1482. """Generate an index for for packs."""
  1483. for pack in repo.object_store.packs:
  1484. yield (b"P " + os.fsencode(pack.data.filename) + b"\n")
  1485. def update_server_info(repo: "BaseRepo") -> None:
  1486. """Generate server info for dumb file access.
  1487. This generates info/refs and objects/info/packs,
  1488. similar to "git update-server-info".
  1489. """
  1490. repo._put_named_file(
  1491. os.path.join("info", "refs"), b"".join(generate_info_refs(repo))
  1492. )
  1493. repo._put_named_file(
  1494. os.path.join("objects", "info", "packs"),
  1495. b"".join(generate_objects_info_packs(repo)),
  1496. )
  1497. if __name__ == "__main__":
  1498. main()