server.py 59 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767
  1. # server.py -- Implementation of the server side git protocols
  2. # Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
  3. # Copyright(C) 2011-2012 Jelmer Vernooij <jelmer@jelmer.uk>
  4. #
  5. # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
  6. # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
  7. # General Public License as published by the Free Software Foundation; version 2.0
  8. # or (at your option) any later version. You can redistribute it and/or
  9. # modify it under the terms of either of these two licenses.
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. #
  17. # You should have received a copy of the licenses; if not, see
  18. # <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
  19. # and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
  20. # License, Version 2.0.
  21. #
  22. """Git smart network protocol server implementation.
  23. For more detailed implementation on the network protocol, see the
  24. Documentation/technical directory in the cgit distribution, and in particular:
  25. * Documentation/technical/protocol-capabilities.txt
  26. * Documentation/technical/pack-protocol.txt
  27. Currently supported capabilities:
  28. * include-tag
  29. * thin-pack
  30. * multi_ack_detailed
  31. * multi_ack
  32. * side-band-64k
  33. * ofs-delta
  34. * no-progress
  35. * report-status
  36. * delete-refs
  37. * shallow
  38. * symref
  39. """
  40. __all__ = [
  41. "DEFAULT_HANDLERS",
  42. "AckGraphWalkerImpl",
  43. "Backend",
  44. "BackendRepo",
  45. "DictBackend",
  46. "FileSystemBackend",
  47. "Handler",
  48. "MultiAckDetailedGraphWalkerImpl",
  49. "MultiAckGraphWalkerImpl",
  50. "PackHandler",
  51. "ReceivePackHandler",
  52. "SingleAckGraphWalkerImpl",
  53. "TCPGitRequestHandler",
  54. "TCPGitServer",
  55. "UploadArchiveHandler",
  56. "UploadPackHandler",
  57. "generate_info_refs",
  58. "generate_objects_info_packs",
  59. "main",
  60. "serve_command",
  61. "update_server_info",
  62. ]
  63. import os
  64. import socket
  65. import socketserver
  66. import sys
  67. import time
  68. import zlib
  69. from collections import deque
  70. from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
  71. from collections.abc import Set as AbstractSet
  72. from functools import partial
  73. from typing import IO, TYPE_CHECKING
  74. from typing import Protocol as TypingProtocol
  75. if TYPE_CHECKING:
  76. from .object_store import BaseObjectStore
  77. from .repo import BaseRepo
  78. from dulwich import log_utils
  79. from .archive import tar_stream
  80. from .errors import (
  81. ApplyDeltaError,
  82. ChecksumMismatch,
  83. GitProtocolError,
  84. HookError,
  85. NotGitRepository,
  86. ObjectFormatException,
  87. UnexpectedCommandError,
  88. )
  89. from .object_store import MissingObjectFinder, PackBasedObjectStore, find_shallow
  90. from .objects import Commit, ObjectID, Tree, valid_hexsha
  91. from .pack import ObjectContainer, write_pack_from_container
  92. from .protocol import (
  93. CAPABILITIES_REF,
  94. CAPABILITY_AGENT,
  95. CAPABILITY_DELETE_REFS,
  96. CAPABILITY_INCLUDE_TAG,
  97. CAPABILITY_MULTI_ACK,
  98. CAPABILITY_MULTI_ACK_DETAILED,
  99. CAPABILITY_NO_DONE,
  100. CAPABILITY_NO_PROGRESS,
  101. CAPABILITY_OFS_DELTA,
  102. CAPABILITY_QUIET,
  103. CAPABILITY_REPORT_STATUS,
  104. CAPABILITY_SHALLOW,
  105. CAPABILITY_SIDE_BAND_64K,
  106. CAPABILITY_THIN_PACK,
  107. COMMAND_DEEPEN,
  108. COMMAND_DONE,
  109. COMMAND_HAVE,
  110. COMMAND_SHALLOW,
  111. COMMAND_UNSHALLOW,
  112. COMMAND_WANT,
  113. MULTI_ACK,
  114. MULTI_ACK_DETAILED,
  115. NAK_LINE,
  116. PEELED_TAG_SUFFIX,
  117. SIDE_BAND_CHANNEL_DATA,
  118. SIDE_BAND_CHANNEL_FATAL,
  119. SIDE_BAND_CHANNEL_PROGRESS,
  120. SINGLE_ACK,
  121. TCP_GIT_PORT,
  122. ZERO_SHA,
  123. BufferedPktLineWriter,
  124. Protocol,
  125. ReceivableProtocol,
  126. ack_type,
  127. capability_agent,
  128. extract_capabilities,
  129. extract_want_line_capabilities,
  130. format_ack_line,
  131. format_ref_line,
  132. format_shallow_line,
  133. format_unshallow_line,
  134. symref_capabilities,
  135. write_info_refs,
  136. )
  137. from .refs import Ref, RefsContainer
  138. from .repo import Repo
  139. logger = log_utils.getLogger(__name__)
  140. class Backend:
  141. """A backend for the Git smart server implementation."""
  142. def open_repository(self, path: str) -> "BackendRepo":
  143. """Open the repository at a path.
  144. Args:
  145. path: Path to the repository
  146. Raises:
  147. NotGitRepository: no git repository was found at path
  148. Returns: Instance of BackendRepo
  149. """
  150. raise NotImplementedError(self.open_repository)
  151. class BackendRepo(TypingProtocol):
  152. """Repository abstraction used by the Git server.
  153. The methods required here are a subset of those provided by
  154. dulwich.repo.Repo.
  155. """
  156. object_store: PackBasedObjectStore
  157. refs: RefsContainer
  158. def get_refs(self) -> dict[bytes, bytes]:
  159. """Get all the refs in the repository.
  160. Returns: dict of name -> sha
  161. """
  162. raise NotImplementedError
  163. def get_peeled(self, name: bytes) -> ObjectID | None:
  164. """Return the cached peeled value of a ref, if available.
  165. Args:
  166. name: Name of the ref to peel
  167. Returns: The peeled value of the ref. If the ref is known not point to
  168. a tag, this will be the SHA the ref refers to. If no cached
  169. information about a tag is available, this method may return None,
  170. but it should attempt to peel the tag if possible.
  171. """
  172. return None
  173. def find_missing_objects(
  174. self,
  175. determine_wants: Callable[[Mapping[Ref, ObjectID], int | None], list[ObjectID]],
  176. graph_walker: "_ProtocolGraphWalker",
  177. progress: Callable[[bytes], None] | None,
  178. *,
  179. get_tagged: Callable[[], dict[ObjectID, ObjectID]] | None = None,
  180. depth: int | None = None,
  181. ) -> "MissingObjectFinder | None":
  182. """Yield the objects required for a list of commits.
  183. Args:
  184. determine_wants: Function to determine which objects the client wants
  185. graph_walker: Object used to walk the commit graph
  186. progress: is a callback to send progress messages to the client
  187. get_tagged: Function that returns a dict of pointed-to sha ->
  188. tag sha for including tags.
  189. depth: Maximum depth of commits to fetch for shallow clones
  190. """
  191. raise NotImplementedError
  192. class DictBackend(Backend):
  193. """Trivial backend that looks up Git repositories in a dictionary."""
  194. def __init__(
  195. self, repos: dict[bytes, "BackendRepo"] | dict[str, "BackendRepo"]
  196. ) -> None:
  197. """Initialize a DictBackend.
  198. Args:
  199. repos: Dictionary mapping repository paths to BackendRepo instances
  200. """
  201. self.repos = repos
  202. def open_repository(self, path: str) -> BackendRepo:
  203. """Open repository at given path.
  204. Args:
  205. path: Path to the repository
  206. Returns:
  207. Repository object
  208. Raises:
  209. NotGitRepository: If no repository found at path
  210. """
  211. logger.debug("Opening repository at %s", path)
  212. # Handle both str and bytes keys for backward compatibility
  213. if path in self.repos:
  214. return self.repos[path] # type: ignore
  215. # Try converting between str and bytes
  216. if isinstance(path, bytes):
  217. try:
  218. alt_path = path.decode("utf-8")
  219. if alt_path in self.repos:
  220. return self.repos[alt_path]
  221. except UnicodeDecodeError:
  222. pass
  223. else:
  224. alt_path_bytes = path.encode("utf-8")
  225. if alt_path_bytes in self.repos:
  226. return self.repos[alt_path_bytes] # type: ignore
  227. raise NotGitRepository(
  228. "No git repository was found at {path}".format(**dict(path=path))
  229. )
  230. class FileSystemBackend(Backend):
  231. """Simple backend looking up Git repositories in the local file system."""
  232. def __init__(self, root: str = os.sep) -> None:
  233. """Initialize a FileSystemBackend.
  234. Args:
  235. root: Root directory to serve repositories from
  236. """
  237. super().__init__()
  238. self.root = (os.path.abspath(root) + os.sep).replace(os.sep * 2, os.sep)
  239. def open_repository(self, path: str) -> BackendRepo:
  240. """Open a repository from the filesystem.
  241. Args:
  242. path: Path to the repository relative to the root
  243. Returns: Repo instance
  244. Raises:
  245. NotGitRepository: If path is outside the root or not a git repository
  246. """
  247. logger.debug("opening repository at %s", path)
  248. # Ensure path is a string to avoid TypeError when joining with self.root
  249. path = os.fspath(path)
  250. if isinstance(path, bytes):
  251. path = os.fsdecode(path)
  252. abspath = os.path.abspath(os.path.join(self.root, path)) + os.sep
  253. normcase_abspath = os.path.normcase(abspath)
  254. normcase_root = os.path.normcase(self.root)
  255. if not normcase_abspath.startswith(normcase_root):
  256. raise NotGitRepository(f"Path {path!r} not inside root {self.root!r}")
  257. return Repo(abspath) # type: ignore[return-value]
  258. class Handler:
  259. """Smart protocol command handler base class."""
  260. def __init__(
  261. self, backend: Backend, proto: Protocol, stateless_rpc: bool = False
  262. ) -> None:
  263. """Initialize a Handler.
  264. Args:
  265. backend: Backend instance for repository access
  266. proto: Protocol instance for communication
  267. stateless_rpc: Whether this is a stateless RPC session
  268. """
  269. self.backend = backend
  270. self.proto = proto
  271. self.stateless_rpc = stateless_rpc
  272. def handle(self) -> None:
  273. """Handle a request."""
  274. raise NotImplementedError(self.handle)
  275. class PackHandler(Handler):
  276. """Protocol handler for packs."""
  277. def __init__(
  278. self, backend: Backend, proto: Protocol, stateless_rpc: bool = False
  279. ) -> None:
  280. """Initialize a PackHandler.
  281. Args:
  282. backend: Backend instance for repository access
  283. proto: Protocol instance for communication
  284. stateless_rpc: Whether this is a stateless RPC session
  285. """
  286. super().__init__(backend, proto, stateless_rpc)
  287. self._client_capabilities: set[bytes] | None = None
  288. # Flags needed for the no-done capability
  289. self._done_received = False
  290. self.advertise_refs = False
  291. @classmethod
  292. def capabilities(cls) -> Iterable[bytes]:
  293. """Return a list of capabilities supported by this handler."""
  294. raise NotImplementedError(cls.capabilities)
  295. @classmethod
  296. def innocuous_capabilities(cls) -> Iterable[bytes]:
  297. """Return capabilities that don't affect protocol behavior.
  298. Returns:
  299. List of innocuous capability names
  300. """
  301. return [
  302. CAPABILITY_INCLUDE_TAG,
  303. CAPABILITY_THIN_PACK,
  304. CAPABILITY_NO_PROGRESS,
  305. CAPABILITY_OFS_DELTA,
  306. capability_agent(),
  307. ]
  308. @classmethod
  309. def required_capabilities(cls) -> Iterable[bytes]:
  310. """Return a list of capabilities that we require the client to have."""
  311. return []
  312. def set_client_capabilities(self, caps: Iterable[bytes]) -> None:
  313. """Set the client capabilities and validate them.
  314. Args:
  315. caps: List of capabilities requested by the client
  316. Raises:
  317. GitProtocolError: If client requests unsupported capability or lacks required ones
  318. """
  319. allowable_caps = set(self.innocuous_capabilities())
  320. allowable_caps.update(self.capabilities())
  321. for cap in caps:
  322. if cap.startswith(CAPABILITY_AGENT + b"="):
  323. continue
  324. if cap not in allowable_caps:
  325. raise GitProtocolError(
  326. f"Client asked for capability {cap!r} that was not advertised."
  327. )
  328. for cap in self.required_capabilities():
  329. if cap not in caps:
  330. raise GitProtocolError(
  331. f"Client does not support required capability {cap!r}."
  332. )
  333. self._client_capabilities = set(caps)
  334. logger.info("Client capabilities: %s", caps)
  335. def has_capability(self, cap: bytes) -> bool:
  336. """Check if the client supports a specific capability.
  337. Args:
  338. cap: Capability name to check
  339. Returns: True if the client supports the capability
  340. Raises:
  341. GitProtocolError: If called before client capabilities are set
  342. """
  343. if self._client_capabilities is None:
  344. raise GitProtocolError(
  345. f"Server attempted to access capability {cap!r} before asking client"
  346. )
  347. return cap in self._client_capabilities
  348. def notify_done(self) -> None:
  349. """Notify that the 'done' command has been received from the client."""
  350. self._done_received = True
  351. class UploadPackHandler(PackHandler):
  352. """Protocol handler for uploading a pack to the client."""
  353. def __init__(
  354. self,
  355. backend: Backend,
  356. args: Sequence[str],
  357. proto: Protocol,
  358. stateless_rpc: bool = False,
  359. advertise_refs: bool = False,
  360. ) -> None:
  361. """Initialize an UploadPackHandler.
  362. Args:
  363. backend: Backend instance for repository access
  364. args: Command arguments (first arg is repository path)
  365. proto: Protocol instance for communication
  366. stateless_rpc: Whether this is a stateless RPC session
  367. advertise_refs: Whether to advertise refs
  368. """
  369. super().__init__(backend, proto, stateless_rpc=stateless_rpc)
  370. self.repo = backend.open_repository(args[0])
  371. self._graph_walker = None
  372. self.advertise_refs = advertise_refs
  373. # A state variable for denoting that the have list is still
  374. # being processed, and the client is not accepting any other
  375. # data (such as side-band, see the progress method here).
  376. self._processing_have_lines = False
  377. @classmethod
  378. def capabilities(cls) -> list[bytes]:
  379. """Return the list of capabilities supported by upload-pack."""
  380. return [
  381. CAPABILITY_MULTI_ACK_DETAILED,
  382. CAPABILITY_MULTI_ACK,
  383. CAPABILITY_SIDE_BAND_64K,
  384. CAPABILITY_THIN_PACK,
  385. CAPABILITY_OFS_DELTA,
  386. CAPABILITY_NO_PROGRESS,
  387. CAPABILITY_INCLUDE_TAG,
  388. CAPABILITY_SHALLOW,
  389. CAPABILITY_NO_DONE,
  390. ]
  391. @classmethod
  392. def required_capabilities(cls) -> tuple[bytes, ...]:
  393. """Return the list of capabilities required for upload-pack."""
  394. return (
  395. CAPABILITY_SIDE_BAND_64K,
  396. CAPABILITY_THIN_PACK,
  397. CAPABILITY_OFS_DELTA,
  398. )
  399. def progress(self, message: bytes) -> None:
  400. """Send a progress message to the client.
  401. Args:
  402. message: Progress message to send
  403. """
  404. def _start_pack_send_phase(self) -> None:
  405. """Start the pack sending phase, setting up sideband if supported."""
  406. if self.has_capability(CAPABILITY_SIDE_BAND_64K):
  407. # The provided haves are processed, and it is safe to send side-
  408. # band data now.
  409. if not self.has_capability(CAPABILITY_NO_PROGRESS):
  410. self.progress = partial( # type: ignore
  411. self.proto.write_sideband, SIDE_BAND_CHANNEL_PROGRESS
  412. )
  413. self.write_pack_data: Callable[[bytes], None] = partial(
  414. self.proto.write_sideband, SIDE_BAND_CHANNEL_DATA
  415. )
  416. else:
  417. # proto.write returns Optional[int], but we need to treat it as returning None
  418. # for compatibility with write_pack_from_container
  419. def write_data(data: bytes) -> None:
  420. self.proto.write(data)
  421. self.write_pack_data = write_data
  422. def get_tagged(
  423. self,
  424. refs: Mapping[bytes, bytes] | None = None,
  425. repo: BackendRepo | None = None,
  426. ) -> dict[ObjectID, ObjectID]:
  427. """Get a dict of peeled values of tags to their original tag shas.
  428. Args:
  429. refs: dict of refname -> sha of possible tags; defaults to all
  430. of the backend's refs.
  431. repo: optional Repo instance for getting peeled refs; defaults
  432. to the backend's repo, if available
  433. Returns: dict of peeled_sha -> tag_sha, where tag_sha is the sha of a
  434. tag whose peeled value is peeled_sha.
  435. """
  436. if not self.has_capability(CAPABILITY_INCLUDE_TAG):
  437. return {}
  438. if refs is None:
  439. refs = self.repo.get_refs()
  440. if repo is None:
  441. repo = getattr(self.repo, "repo", None)
  442. if repo is None:
  443. # Bail if we don't have a Repo available; this is ok since
  444. # clients must be able to handle if the server doesn't include
  445. # all relevant tags.
  446. # TODO: fix behavior when missing
  447. return {}
  448. # TODO(jelmer): Integrate this with the refs logic in
  449. # Repo.find_missing_objects
  450. tagged = {}
  451. for name, sha in refs.items():
  452. peeled_sha = repo.get_peeled(name)
  453. if peeled_sha is not None and peeled_sha != ObjectID(sha):
  454. tagged[peeled_sha] = ObjectID(sha)
  455. return tagged
  456. def handle(self) -> None:
  457. """Handle an upload-pack request.
  458. This method processes the client's wants and haves, determines which
  459. objects to send, and writes the pack data to the client.
  460. """
  461. # Note the fact that client is only processing responses related
  462. # to the have lines it sent, and any other data (including side-
  463. # band) will be be considered a fatal error.
  464. self._processing_have_lines = True
  465. graph_walker = _ProtocolGraphWalker(
  466. self,
  467. self.repo.object_store,
  468. self.repo.get_peeled,
  469. self.repo.refs.get_symrefs,
  470. )
  471. wants: list[ObjectID] = []
  472. def wants_wrapper(
  473. refs: Mapping[Ref, ObjectID], depth: int | None = None
  474. ) -> list[ObjectID]:
  475. wants.extend(graph_walker.determine_wants(refs, depth))
  476. return wants
  477. missing_objects = self.repo.find_missing_objects(
  478. wants_wrapper,
  479. graph_walker,
  480. self.progress,
  481. get_tagged=self.get_tagged,
  482. )
  483. # Did the process short-circuit (e.g. in a stateless RPC call)? Note
  484. # that the client still expects a 0-object pack in most cases.
  485. # Also, if it also happens that the object_iter is instantiated
  486. # with a graph walker with an implementation that talks over the
  487. # wire (which is this instance of this class) this will actually
  488. # iterate through everything and write things out to the wire.
  489. if len(wants) == 0:
  490. return
  491. # Handle shallow clone case where missing_objects can be None
  492. if missing_objects is None:
  493. return
  494. object_ids = list(missing_objects)
  495. if not graph_walker.handle_done(
  496. not self.has_capability(CAPABILITY_NO_DONE), self._done_received
  497. ):
  498. return
  499. self._start_pack_send_phase()
  500. self.progress((f"counting objects: {len(object_ids)}, done.\n").encode("ascii"))
  501. write_pack_from_container(
  502. self.write_pack_data,
  503. self.repo.object_store,
  504. object_ids,
  505. )
  506. # we are done
  507. self.proto.write_pkt_line(None)
  508. def _split_proto_line(
  509. line: bytes | None, allowed: Iterable[bytes | None] | None
  510. ) -> tuple[bytes | None, bytes | int | None]:
  511. """Split a line read from the wire.
  512. Args:
  513. line: The line read from the wire.
  514. allowed: An iterable of command names that should be allowed.
  515. Command names not listed below as possible return values will be
  516. ignored. If None, any commands from the possible return values are
  517. allowed.
  518. Returns: a tuple having one of the following forms:
  519. ('want', obj_id)
  520. ('have', obj_id)
  521. ('done', None)
  522. (None, None) (for a flush-pkt)
  523. Raises:
  524. UnexpectedCommandError: if the line cannot be parsed into one of the
  525. allowed return values.
  526. """
  527. if not line:
  528. fields: list[bytes | None] = [None]
  529. else:
  530. fields = list(line.rstrip(b"\n").split(b" ", 1))
  531. command = fields[0]
  532. if allowed is not None and command not in allowed:
  533. raise UnexpectedCommandError(command.decode("utf-8") if command else None)
  534. if len(fields) == 1 and command in (COMMAND_DONE, None):
  535. return (command, None)
  536. elif len(fields) == 2:
  537. if command in (
  538. COMMAND_WANT,
  539. COMMAND_HAVE,
  540. COMMAND_SHALLOW,
  541. COMMAND_UNSHALLOW,
  542. ):
  543. assert fields[1] is not None
  544. if not valid_hexsha(fields[1]):
  545. raise GitProtocolError("Invalid sha")
  546. return (command, fields[1])
  547. elif command == COMMAND_DEEPEN:
  548. assert fields[1] is not None
  549. return command, int(fields[1])
  550. raise GitProtocolError(f"Received invalid line from client: {line!r}")
  551. def _want_satisfied(
  552. store: ObjectContainer, haves: set[ObjectID], want: ObjectID, earliest: int
  553. ) -> bool:
  554. """Check if a specific want is satisfied by a set of haves.
  555. Args:
  556. store: Object store to retrieve objects from
  557. haves: Set of commit IDs the client has
  558. want: Commit ID the client wants
  559. earliest: Earliest commit time to consider
  560. Returns: True if the want is satisfied by the haves
  561. """
  562. o = store[want]
  563. pending = deque([o])
  564. known = {want}
  565. while pending:
  566. commit = pending.popleft()
  567. if commit.id in haves:
  568. return True
  569. if not isinstance(commit, Commit):
  570. # non-commit wants are assumed to be satisfied
  571. continue
  572. for parent in commit.parents:
  573. if parent in known:
  574. continue
  575. known.add(parent)
  576. parent_obj = store[parent]
  577. assert isinstance(parent_obj, Commit)
  578. # TODO: handle parents with later commit times than children
  579. if parent_obj.commit_time >= earliest:
  580. pending.append(parent_obj)
  581. return False
  582. def _all_wants_satisfied(
  583. store: ObjectContainer, haves: AbstractSet[ObjectID], wants: set[ObjectID]
  584. ) -> bool:
  585. """Check whether all the current wants are satisfied by a set of haves.
  586. Args:
  587. store: Object store to retrieve objects from
  588. haves: A set of commits we know the client has.
  589. wants: A set of commits the client wants
  590. Note: Wants are specified with set_wants rather than passed in since
  591. in the current interface they are determined outside this class.
  592. """
  593. haves = set(haves)
  594. if haves:
  595. have_objs = [store[h] for h in haves]
  596. earliest = min([h.commit_time for h in have_objs if isinstance(h, Commit)])
  597. else:
  598. earliest = 0
  599. for want in wants:
  600. if not _want_satisfied(store, haves, want, earliest):
  601. return False
  602. return True
  603. class AckGraphWalkerImpl:
  604. """Base class for acknowledgment graph walker implementations."""
  605. def __init__(self, graph_walker: "_ProtocolGraphWalker") -> None:
  606. """Initialize acknowledgment graph walker.
  607. Args:
  608. graph_walker: Graph walker to wrap
  609. """
  610. raise NotImplementedError
  611. def ack(self, have_ref: ObjectID) -> None:
  612. """Acknowledge a have reference.
  613. Args:
  614. have_ref: Object ID to acknowledge
  615. """
  616. raise NotImplementedError
  617. def handle_done(self, done_required: bool, done_received: bool) -> bool:
  618. """Handle 'done' packet from client."""
  619. raise NotImplementedError
  620. class _ProtocolGraphWalker:
  621. """A graph walker that knows the git protocol.
  622. As a graph walker, this class implements ack(), next(), and reset(). It
  623. also contains some base methods for interacting with the wire and walking
  624. the commit tree.
  625. The work of determining which acks to send is passed on to the
  626. implementation instance stored in _impl. The reason for this is that we do
  627. not know at object creation time what ack level the protocol requires. A
  628. call to set_ack_type() is required to set up the implementation, before
  629. any calls to next() or ack() are made.
  630. """
  631. def __init__(
  632. self,
  633. handler: PackHandler,
  634. object_store: ObjectContainer,
  635. get_peeled: Callable[[bytes], ObjectID | None],
  636. get_symrefs: Callable[[], dict[Ref, Ref]],
  637. ) -> None:
  638. """Initialize a ProtocolGraphWalker.
  639. Args:
  640. handler: Protocol handler instance
  641. object_store: Object store for retrieving objects
  642. get_peeled: Function to get peeled refs
  643. get_symrefs: Function to get symbolic refs
  644. """
  645. self.handler = handler
  646. self.store: ObjectContainer = object_store
  647. self.get_peeled = get_peeled
  648. self.get_symrefs = get_symrefs
  649. self.proto = handler.proto
  650. self.stateless_rpc = handler.stateless_rpc
  651. self.advertise_refs = handler.advertise_refs
  652. self._wants: list[ObjectID] = []
  653. self.shallow: set[ObjectID] = set()
  654. self.client_shallow: set[ObjectID] = set()
  655. self.unshallow: set[ObjectID] = set()
  656. self._cached = False
  657. self._cache: list[ObjectID] = []
  658. self._cache_index = 0
  659. self._impl: AckGraphWalkerImpl | None = None
  660. def determine_wants(
  661. self, heads: Mapping[Ref, ObjectID], depth: int | None = None
  662. ) -> list[ObjectID]:
  663. """Determine the wants for a set of heads.
  664. The given heads are advertised to the client, who then specifies which
  665. refs they want using 'want' lines. This portion of the protocol is the
  666. same regardless of ack type, and in fact is used to set the ack type of
  667. the ProtocolGraphWalker.
  668. If the client has the 'shallow' capability, this method also reads and
  669. responds to the 'shallow' and 'deepen' lines from the client. These are
  670. not part of the wants per se, but they set up necessary state for
  671. walking the graph. Additionally, later code depends on this method
  672. consuming everything up to the first 'have' line.
  673. Args:
  674. heads: a dict of refname->SHA1 to advertise
  675. depth: Maximum depth for shallow clones
  676. Returns: a list of SHA1s requested by the client
  677. """
  678. symrefs = self.get_symrefs()
  679. values = set(heads.values())
  680. if self.advertise_refs or not self.stateless_rpc:
  681. for i, (ref, sha) in enumerate(sorted(heads.items())):
  682. try:
  683. peeled_sha = self.get_peeled(ref)
  684. except KeyError:
  685. # Skip refs that are inaccessible
  686. # TODO(jelmer): Integrate with Repo.find_missing_objects refs
  687. # logic.
  688. continue
  689. if i == 0:
  690. logger.info("Sending capabilities: %s", self.handler.capabilities())
  691. line = format_ref_line(
  692. ref,
  693. sha,
  694. list(self.handler.capabilities())
  695. + symref_capabilities(symrefs.items()),
  696. )
  697. else:
  698. line = format_ref_line(ref, sha)
  699. self.proto.write_pkt_line(line)
  700. if peeled_sha is not None and peeled_sha != sha:
  701. self.proto.write_pkt_line(
  702. format_ref_line(ref + PEELED_TAG_SUFFIX, peeled_sha)
  703. )
  704. # i'm done..
  705. self.proto.write_pkt_line(None)
  706. if self.advertise_refs:
  707. return []
  708. # Now client will sending want want want commands
  709. want = self.proto.read_pkt_line()
  710. if not want:
  711. return []
  712. line, caps = extract_want_line_capabilities(want)
  713. self.handler.set_client_capabilities(caps)
  714. self.set_ack_type(ack_type(caps))
  715. allowed = (COMMAND_WANT, COMMAND_SHALLOW, COMMAND_DEEPEN, None)
  716. command, sha_result = _split_proto_line(line, allowed)
  717. want_revs: list[ObjectID] = []
  718. while command == COMMAND_WANT:
  719. assert isinstance(sha_result, bytes)
  720. if sha_result not in values:
  721. raise GitProtocolError(f"Client wants invalid object {sha_result!r}")
  722. want_revs.append(ObjectID(sha_result))
  723. command, sha_result = self.read_proto_line(allowed)
  724. self.set_wants(want_revs)
  725. if command in (COMMAND_SHALLOW, COMMAND_DEEPEN):
  726. assert sha_result is not None
  727. self.unread_proto_line(command, sha_result)
  728. self._handle_shallow_request(want_revs)
  729. if self.stateless_rpc and self.proto.eof():
  730. # The client may close the socket at this point, expecting a
  731. # flush-pkt from the server. We might be ready to send a packfile
  732. # at this point, so we need to explicitly short-circuit in this
  733. # case.
  734. return []
  735. return want_revs
  736. def unread_proto_line(self, command: bytes, value: bytes | int) -> None:
  737. """Push a command back to be read again.
  738. Args:
  739. command: Command name
  740. value: Command value
  741. """
  742. if isinstance(value, int):
  743. value = str(value).encode("ascii")
  744. self.proto.unread_pkt_line(command + b" " + value)
  745. def nak(self) -> None:
  746. """Send a NAK response."""
  747. def ack(self, have_ref: ObjectID) -> None:
  748. """Acknowledge a have reference.
  749. Args:
  750. have_ref: SHA to acknowledge (40 bytes hex)
  751. Raises:
  752. ValueError: If have_ref is not 40 bytes
  753. """
  754. if len(have_ref) != 40:
  755. raise ValueError(f"invalid sha {have_ref!r}")
  756. assert self._impl is not None
  757. return self._impl.ack(have_ref)
  758. def reset(self) -> None:
  759. """Reset the graph walker cache."""
  760. self._cached = True
  761. self._cache_index = 0
  762. def next(self) -> ObjectID | None:
  763. """Get the next SHA from the graph walker.
  764. Returns: Next SHA or None if done
  765. """
  766. if not self._cached:
  767. if not self._impl and self.stateless_rpc:
  768. return None
  769. assert self._impl is not None
  770. return next(self._impl) # type: ignore[call-overload, no-any-return]
  771. self._cache_index += 1
  772. if self._cache_index > len(self._cache):
  773. return None
  774. return self._cache[self._cache_index]
  775. __next__ = next
  776. def read_proto_line(
  777. self, allowed: Iterable[bytes | None] | None
  778. ) -> tuple[bytes | None, bytes | int | None]:
  779. """Read a line from the wire.
  780. Args:
  781. allowed: An iterable of command names that should be allowed.
  782. Returns: A tuple of (command, value); see _split_proto_line.
  783. Raises:
  784. UnexpectedCommandError: If an error occurred reading the line.
  785. """
  786. return _split_proto_line(self.proto.read_pkt_line(), allowed)
  787. def _handle_shallow_request(self, wants: Sequence[ObjectID]) -> None:
  788. """Handle shallow clone requests from the client.
  789. Args:
  790. wants: List of wanted object SHAs
  791. """
  792. while True:
  793. command, val = self.read_proto_line((COMMAND_DEEPEN, COMMAND_SHALLOW))
  794. if command == COMMAND_DEEPEN:
  795. assert isinstance(val, int)
  796. depth = val
  797. break
  798. assert isinstance(val, bytes)
  799. self.client_shallow.add(ObjectID(val))
  800. self.read_proto_line((None,)) # consume client's flush-pkt
  801. shallow, not_shallow = find_shallow(self.store, wants, depth)
  802. # Update self.shallow instead of reassigning it since we passed a
  803. # reference to it before this method was called.
  804. self.shallow.update(shallow - not_shallow)
  805. new_shallow = self.shallow - self.client_shallow
  806. unshallow = self.unshallow = not_shallow & self.client_shallow
  807. self.update_shallow(new_shallow, unshallow)
  808. def update_shallow(
  809. self, new_shallow: AbstractSet[ObjectID], unshallow: AbstractSet[ObjectID]
  810. ) -> None:
  811. """Update shallow/unshallow information to the client.
  812. Args:
  813. new_shallow: Set of newly shallow commits
  814. unshallow: Set of commits to unshallow
  815. """
  816. for sha in sorted(new_shallow):
  817. self.proto.write_pkt_line(format_shallow_line(sha))
  818. for sha in sorted(unshallow):
  819. self.proto.write_pkt_line(format_unshallow_line(sha))
  820. self.proto.write_pkt_line(None)
  821. def notify_done(self) -> None:
  822. """Notify that the client sent 'done'."""
  823. # relay the message down to the handler.
  824. self.handler.notify_done()
  825. def send_ack(self, sha: ObjectID, ack_type: bytes = b"") -> None:
  826. """Send an ACK to the client.
  827. Args:
  828. sha: SHA to acknowledge
  829. ack_type: Type of ACK (e.g., b'continue', b'ready')
  830. """
  831. self.proto.write_pkt_line(format_ack_line(sha, ack_type))
  832. def send_nak(self) -> None:
  833. """Send a NAK to the client."""
  834. self.proto.write_pkt_line(NAK_LINE)
  835. def handle_done(self, done_required: bool, done_received: bool) -> bool:
  836. """Handle the 'done' command.
  837. Args:
  838. done_required: Whether done is required
  839. done_received: Whether done was received
  840. Returns: True if done handling succeeded
  841. """
  842. # Delegate this to the implementation.
  843. assert self._impl is not None
  844. return self._impl.handle_done(done_required, done_received)
  845. def set_wants(self, wants: list[ObjectID]) -> None:
  846. """Set the list of wanted objects.
  847. Args:
  848. wants: List of wanted object SHAs
  849. """
  850. self._wants = wants
  851. def all_wants_satisfied(self, haves: AbstractSet[ObjectID]) -> bool:
  852. """Check whether all the current wants are satisfied by a set of haves.
  853. Args:
  854. haves: A set of commits we know the client has.
  855. Note: Wants are specified with set_wants rather than passed in since
  856. in the current interface they are determined outside this class.
  857. """
  858. return _all_wants_satisfied(self.store, haves, set(self._wants))
  859. def set_ack_type(self, ack_type: int) -> None:
  860. """Set the acknowledgment type for the graph walker.
  861. Args:
  862. ack_type: One of SINGLE_ACK, MULTI_ACK, or MULTI_ACK_DETAILED
  863. """
  864. impl_classes: dict[int, type[AckGraphWalkerImpl]] = {
  865. MULTI_ACK: MultiAckGraphWalkerImpl,
  866. MULTI_ACK_DETAILED: MultiAckDetailedGraphWalkerImpl,
  867. SINGLE_ACK: SingleAckGraphWalkerImpl,
  868. }
  869. self._impl = impl_classes[ack_type](self)
  870. _GRAPH_WALKER_COMMANDS = (COMMAND_HAVE, COMMAND_DONE, None)
  871. class SingleAckGraphWalkerImpl(AckGraphWalkerImpl):
  872. """Graph walker implementation that speaks the single-ack protocol."""
  873. def __init__(self, walker: "_ProtocolGraphWalker") -> None:
  874. """Initialize a SingleAckGraphWalkerImpl.
  875. Args:
  876. walker: Parent ProtocolGraphWalker instance
  877. """
  878. self.walker = walker
  879. self._common: list[ObjectID] = []
  880. def ack(self, have_ref: ObjectID) -> None:
  881. """Acknowledge a have reference.
  882. Args:
  883. have_ref: Object ID to acknowledge
  884. """
  885. if not self._common:
  886. self.walker.send_ack(have_ref)
  887. self._common.append(have_ref)
  888. def next(self) -> ObjectID | None:
  889. """Get next SHA from graph walker.
  890. Returns:
  891. SHA bytes or None if done
  892. """
  893. command, sha = self.walker.read_proto_line(_GRAPH_WALKER_COMMANDS)
  894. if command in (None, COMMAND_DONE):
  895. # defer the handling of done
  896. self.walker.notify_done()
  897. return None
  898. elif command == COMMAND_HAVE:
  899. assert isinstance(sha, bytes)
  900. return ObjectID(sha)
  901. return None
  902. __next__ = next
  903. def handle_done(self, done_required: bool, done_received: bool) -> bool:
  904. """Handle done command.
  905. Args:
  906. done_required: Whether done is required
  907. done_received: Whether done was received
  908. Returns:
  909. True if handling completed successfully
  910. """
  911. if not self._common:
  912. self.walker.send_nak()
  913. if done_required and not done_received:
  914. # we are not done, especially when done is required; skip
  915. # the pack for this request and especially do not handle
  916. # the done.
  917. return False
  918. if not done_received and not self._common:
  919. # Okay we are not actually done then since the walker picked
  920. # up no haves. This is usually triggered when client attempts
  921. # to pull from a source that has no common base_commit.
  922. # See: test_server.MultiAckDetailedGraphWalkerImplTestCase.\
  923. # test_multi_ack_stateless_nodone
  924. return False
  925. return True
  926. class MultiAckGraphWalkerImpl(AckGraphWalkerImpl):
  927. """Graph walker implementation that speaks the multi-ack protocol."""
  928. def __init__(self, walker: "_ProtocolGraphWalker") -> None:
  929. """Initialize multi-ack graph walker.
  930. Args:
  931. walker: Parent ProtocolGraphWalker instance
  932. """
  933. self.walker = walker
  934. self._found_base = False
  935. self._common: list[ObjectID] = []
  936. def ack(self, have_ref: ObjectID) -> None:
  937. """Acknowledge a have reference.
  938. Args:
  939. have_ref: Object ID to acknowledge
  940. """
  941. self._common.append(have_ref)
  942. if not self._found_base:
  943. self.walker.send_ack(have_ref, b"continue")
  944. if self.walker.all_wants_satisfied(set(self._common)):
  945. self._found_base = True
  946. # else we blind ack within next
  947. def next(self) -> ObjectID | None:
  948. """Get next SHA from graph walker.
  949. Returns:
  950. SHA bytes or None if done
  951. """
  952. while True:
  953. command, sha = self.walker.read_proto_line(_GRAPH_WALKER_COMMANDS)
  954. if command is None:
  955. self.walker.send_nak()
  956. # in multi-ack mode, a flush-pkt indicates the client wants to
  957. # flush but more have lines are still coming
  958. continue
  959. elif command == COMMAND_DONE:
  960. self.walker.notify_done()
  961. return None
  962. elif command == COMMAND_HAVE:
  963. assert isinstance(sha, bytes)
  964. sha_id = ObjectID(sha)
  965. if self._found_base:
  966. # blind ack
  967. self.walker.send_ack(sha_id, b"continue")
  968. return sha_id
  969. __next__ = next
  970. def handle_done(self, done_required: bool, done_received: bool) -> bool:
  971. """Handle done command.
  972. Args:
  973. done_required: Whether done is required
  974. done_received: Whether done was received
  975. Returns:
  976. True if handling completed successfully
  977. """
  978. if done_required and not done_received:
  979. # we are not done, especially when done is required; skip
  980. # the pack for this request and especially do not handle
  981. # the done.
  982. return False
  983. if not done_received and not self._common:
  984. # Okay we are not actually done then since the walker picked
  985. # up no haves. This is usually triggered when client attempts
  986. # to pull from a source that has no common base_commit.
  987. # See: test_server.MultiAckDetailedGraphWalkerImplTestCase.\
  988. # test_multi_ack_stateless_nodone
  989. return False
  990. # don't nak unless no common commits were found, even if not
  991. # everything is satisfied
  992. if self._common:
  993. self.walker.send_ack(self._common[-1])
  994. else:
  995. self.walker.send_nak()
  996. return True
  997. class MultiAckDetailedGraphWalkerImpl(AckGraphWalkerImpl):
  998. """Graph walker implementation speaking the multi-ack-detailed protocol."""
  999. def __init__(self, walker: "_ProtocolGraphWalker") -> None:
  1000. """Initialize multi-ack-detailed graph walker.
  1001. Args:
  1002. walker: Parent ProtocolGraphWalker instance
  1003. """
  1004. self.walker = walker
  1005. self._common: list[ObjectID] = []
  1006. def ack(self, have_ref: ObjectID) -> None:
  1007. """Acknowledge a have reference.
  1008. Args:
  1009. have_ref: Object ID to acknowledge
  1010. """
  1011. # Should only be called iff have_ref is common
  1012. self._common.append(have_ref)
  1013. self.walker.send_ack(have_ref, b"common")
  1014. def next(self) -> ObjectID | None:
  1015. """Get next SHA from graph walker.
  1016. Returns:
  1017. SHA bytes or None if done
  1018. """
  1019. while True:
  1020. command, sha = self.walker.read_proto_line(_GRAPH_WALKER_COMMANDS)
  1021. if command is None:
  1022. if self.walker.all_wants_satisfied(set(self._common)):
  1023. self.walker.send_ack(self._common[-1], b"ready")
  1024. self.walker.send_nak()
  1025. if self.walker.stateless_rpc:
  1026. # The HTTP version of this request a flush-pkt always
  1027. # signifies an end of request, so we also return
  1028. # nothing here as if we are done (but not really, as
  1029. # it depends on whether no-done capability was
  1030. # specified and that's handled in handle_done which
  1031. # may or may not call post_nodone_check depending on
  1032. # that).
  1033. return None
  1034. elif command == COMMAND_DONE:
  1035. # Let the walker know that we got a done.
  1036. self.walker.notify_done()
  1037. break
  1038. elif command == COMMAND_HAVE:
  1039. # return the sha and let the caller ACK it with the
  1040. # above ack method.
  1041. assert isinstance(sha, bytes)
  1042. return ObjectID(sha)
  1043. # don't nak unless no common commits were found, even if not
  1044. # everything is satisfied
  1045. return None
  1046. __next__ = next
  1047. def handle_done(self, done_required: bool, done_received: bool) -> bool:
  1048. """Handle done command.
  1049. Args:
  1050. done_required: Whether done is required
  1051. done_received: Whether done was received
  1052. Returns:
  1053. True if handling completed successfully
  1054. """
  1055. if done_required and not done_received:
  1056. # we are not done, especially when done is required; skip
  1057. # the pack for this request and especially do not handle
  1058. # the done.
  1059. return False
  1060. if not done_received and not self._common:
  1061. # Okay we are not actually done then since the walker picked
  1062. # up no haves. This is usually triggered when client attempts
  1063. # to pull from a source that has no common base_commit.
  1064. # See: test_server.MultiAckDetailedGraphWalkerImplTestCase.\
  1065. # test_multi_ack_stateless_nodone
  1066. return False
  1067. # don't nak unless no common commits were found, even if not
  1068. # everything is satisfied
  1069. if self._common:
  1070. self.walker.send_ack(self._common[-1])
  1071. else:
  1072. self.walker.send_nak()
  1073. return True
  1074. class ReceivePackHandler(PackHandler):
  1075. """Protocol handler for downloading a pack from the client."""
  1076. def __init__(
  1077. self,
  1078. backend: Backend,
  1079. args: Sequence[str],
  1080. proto: Protocol,
  1081. stateless_rpc: bool = False,
  1082. advertise_refs: bool = False,
  1083. ) -> None:
  1084. """Initialize receive-pack handler.
  1085. Args:
  1086. backend: Backend instance
  1087. args: Command arguments
  1088. proto: Protocol instance
  1089. stateless_rpc: Whether to use stateless RPC
  1090. advertise_refs: Whether to advertise refs
  1091. """
  1092. super().__init__(backend, proto, stateless_rpc=stateless_rpc)
  1093. self.repo = backend.open_repository(args[0])
  1094. self.advertise_refs = advertise_refs
  1095. @classmethod
  1096. def capabilities(cls) -> Iterable[bytes]:
  1097. """Return supported capabilities.
  1098. Returns:
  1099. List of capability names
  1100. """
  1101. return [
  1102. CAPABILITY_REPORT_STATUS,
  1103. CAPABILITY_DELETE_REFS,
  1104. CAPABILITY_QUIET,
  1105. CAPABILITY_OFS_DELTA,
  1106. CAPABILITY_SIDE_BAND_64K,
  1107. CAPABILITY_NO_DONE,
  1108. ]
  1109. def _apply_pack(
  1110. self, refs: list[tuple[ObjectID, ObjectID, Ref]]
  1111. ) -> Iterator[tuple[bytes, bytes]]:
  1112. """Apply received pack to repository.
  1113. Args:
  1114. refs: List of (old_sha, new_sha, ref_name) tuples
  1115. Yields:
  1116. Tuples of (ref_name, error_message) for any errors
  1117. """
  1118. all_exceptions = (
  1119. IOError,
  1120. OSError,
  1121. ChecksumMismatch,
  1122. ApplyDeltaError,
  1123. AssertionError,
  1124. socket.error,
  1125. zlib.error,
  1126. ObjectFormatException,
  1127. )
  1128. will_send_pack = False
  1129. for command in refs:
  1130. if command[1] != ZERO_SHA:
  1131. will_send_pack = True
  1132. if will_send_pack:
  1133. # TODO: more informative error messages than just the exception
  1134. # string
  1135. try:
  1136. recv = getattr(self.proto, "recv", None)
  1137. self.repo.object_store.add_thin_pack(self.proto.read, recv) # type: ignore[attr-defined]
  1138. yield (b"unpack", b"ok")
  1139. except all_exceptions as e:
  1140. yield (b"unpack", str(e).replace("\n", "").encode("utf-8"))
  1141. # The pack may still have been moved in, but it may contain
  1142. # broken objects. We trust a later GC to clean it up.
  1143. else:
  1144. # The git protocol want to find a status entry related to unpack
  1145. # process even if no pack data has been sent.
  1146. yield (b"unpack", b"ok")
  1147. for oldsha, sha, ref in refs:
  1148. ref_status = b"ok"
  1149. try:
  1150. if sha == ZERO_SHA:
  1151. if CAPABILITY_DELETE_REFS not in self.capabilities():
  1152. raise GitProtocolError(
  1153. "Attempted to delete refs without delete-refs capability."
  1154. )
  1155. try:
  1156. self.repo.refs.remove_if_equals(ref, oldsha)
  1157. except all_exceptions:
  1158. ref_status = b"failed to delete"
  1159. else:
  1160. try:
  1161. self.repo.refs.set_if_equals(ref, oldsha, sha)
  1162. except all_exceptions:
  1163. ref_status = b"failed to write"
  1164. except KeyError:
  1165. ref_status = b"bad ref"
  1166. yield (ref, ref_status)
  1167. def _report_status(self, status: Sequence[tuple[bytes, bytes]]) -> None:
  1168. """Report status to client.
  1169. Args:
  1170. status: List of (ref_name, status_message) tuples
  1171. """
  1172. if self.has_capability(CAPABILITY_SIDE_BAND_64K):
  1173. writer = BufferedPktLineWriter(
  1174. lambda d: self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, d)
  1175. )
  1176. write = writer.write
  1177. def flush() -> None:
  1178. writer.flush()
  1179. self.proto.write_pkt_line(None)
  1180. else:
  1181. write = self.proto.write_pkt_line # type: ignore[assignment]
  1182. def flush() -> None:
  1183. pass
  1184. for name, msg in status:
  1185. if name == b"unpack":
  1186. write(b"unpack " + msg + b"\n")
  1187. elif msg == b"ok":
  1188. write(b"ok " + name + b"\n")
  1189. else:
  1190. write(b"ng " + name + b" " + msg + b"\n")
  1191. write(None) # type: ignore
  1192. flush()
  1193. def _on_post_receive(self, client_refs: dict[bytes, tuple[bytes, bytes]]) -> None:
  1194. """Run post-receive hook.
  1195. Args:
  1196. client_refs: Dictionary of ref changes from client
  1197. """
  1198. hook = self.repo.hooks.get("post-receive", None) # type: ignore[attr-defined]
  1199. if not hook:
  1200. return
  1201. try:
  1202. output = hook.execute(client_refs)
  1203. if output:
  1204. self.proto.write_sideband(SIDE_BAND_CHANNEL_PROGRESS, output)
  1205. except HookError as err:
  1206. self.proto.write_sideband(SIDE_BAND_CHANNEL_FATAL, str(err).encode("utf-8"))
  1207. def handle(self) -> None:
  1208. """Handle receive-pack request."""
  1209. if self.advertise_refs or not self.stateless_rpc:
  1210. refs = sorted(self.repo.get_refs().items())
  1211. symrefs = sorted(self.repo.refs.get_symrefs().items())
  1212. if not refs:
  1213. refs = [(CAPABILITIES_REF, ZERO_SHA)]
  1214. logger.info("Sending capabilities: %s", self.capabilities())
  1215. self.proto.write_pkt_line(
  1216. format_ref_line(
  1217. refs[0][0],
  1218. refs[0][1],
  1219. list(self.capabilities()) + symref_capabilities(symrefs),
  1220. )
  1221. )
  1222. for i in range(1, len(refs)):
  1223. ref = refs[i]
  1224. self.proto.write_pkt_line(format_ref_line(ref[0], ref[1]))
  1225. self.proto.write_pkt_line(None)
  1226. if self.advertise_refs:
  1227. return
  1228. client_refs = []
  1229. ref_line = self.proto.read_pkt_line()
  1230. # if ref is none then client doesn't want to send us anything..
  1231. if ref_line is None:
  1232. return
  1233. ref_line, caps = extract_capabilities(ref_line)
  1234. self.set_client_capabilities(caps)
  1235. # client will now send us a list of (oldsha, newsha, ref)
  1236. while ref_line:
  1237. (oldsha, newsha, ref_name) = ref_line.split()
  1238. client_refs.append((ObjectID(oldsha), ObjectID(newsha), Ref(ref_name)))
  1239. ref_line = self.proto.read_pkt_line()
  1240. # backend can now deal with this refs and read a pack using self.read
  1241. status = list(self._apply_pack(client_refs))
  1242. self._on_post_receive(client_refs) # type: ignore[arg-type]
  1243. # when we have read all the pack from the client, send a status report
  1244. # if the client asked for it
  1245. if self.has_capability(CAPABILITY_REPORT_STATUS):
  1246. self._report_status(status)
  1247. class UploadArchiveHandler(Handler):
  1248. """Handler for git-upload-archive requests."""
  1249. def __init__(
  1250. self,
  1251. backend: Backend,
  1252. args: Sequence[str],
  1253. proto: Protocol,
  1254. stateless_rpc: bool = False,
  1255. ) -> None:
  1256. """Initialize upload-archive handler.
  1257. Args:
  1258. backend: Backend instance
  1259. args: Command arguments
  1260. proto: Protocol instance
  1261. stateless_rpc: Whether to use stateless RPC
  1262. """
  1263. super().__init__(backend, proto, stateless_rpc)
  1264. self.repo = backend.open_repository(args[0])
  1265. def handle(self) -> None:
  1266. """Handle upload-archive request."""
  1267. def write(x: bytes) -> None:
  1268. self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, x)
  1269. arguments = []
  1270. for pkt in self.proto.read_pkt_seq():
  1271. (key, value) = pkt.split(b" ", 1)
  1272. if key != b"argument":
  1273. raise GitProtocolError(f"unknown command {key!r}")
  1274. arguments.append(value.rstrip(b"\n"))
  1275. prefix = b""
  1276. format = "tar"
  1277. i = 0
  1278. store: BaseObjectStore = self.repo.object_store
  1279. while i < len(arguments):
  1280. argument = arguments[i]
  1281. if argument == b"--prefix":
  1282. i += 1
  1283. prefix = arguments[i]
  1284. elif argument == b"--format":
  1285. i += 1
  1286. format = arguments[i].decode("ascii")
  1287. else:
  1288. commit_sha = self.repo.refs[Ref(argument)]
  1289. commit_obj = store[commit_sha]
  1290. assert isinstance(commit_obj, Commit)
  1291. tree_obj = store[commit_obj.tree]
  1292. assert isinstance(tree_obj, Tree)
  1293. tree = tree_obj
  1294. i += 1
  1295. self.proto.write_pkt_line(b"ACK")
  1296. self.proto.write_pkt_line(None)
  1297. for chunk in tar_stream(
  1298. store,
  1299. tree,
  1300. mtime=int(time.time()),
  1301. prefix=prefix,
  1302. format=format,
  1303. ):
  1304. write(chunk)
  1305. self.proto.write_pkt_line(None)
  1306. # Default handler classes for git services.
  1307. DEFAULT_HANDLERS = {
  1308. b"git-upload-pack": UploadPackHandler,
  1309. b"git-receive-pack": ReceivePackHandler,
  1310. b"git-upload-archive": UploadArchiveHandler,
  1311. }
  1312. class TCPGitRequestHandler(socketserver.StreamRequestHandler):
  1313. """TCP request handler for git protocol."""
  1314. def __init__(
  1315. self,
  1316. handlers: dict[bytes, type[Handler]],
  1317. request: socket.socket,
  1318. client_address: tuple[str, int],
  1319. server: socketserver.TCPServer,
  1320. ) -> None:
  1321. """Initialize TCP request handler.
  1322. Args:
  1323. handlers: Dictionary mapping commands to handler classes
  1324. request: Request socket
  1325. client_address: Client address tuple
  1326. server: Server instance
  1327. """
  1328. self.handlers = handlers
  1329. socketserver.StreamRequestHandler.__init__(
  1330. self, request, client_address, server
  1331. )
  1332. def handle(self) -> None:
  1333. """Handle TCP git request."""
  1334. proto = ReceivableProtocol(self.connection.recv, self.wfile.write)
  1335. command, args = proto.read_cmd()
  1336. logger.info("Handling %s request, args=%s", command, args)
  1337. cls = self.handlers.get(command, None)
  1338. if not callable(cls):
  1339. raise GitProtocolError(f"Invalid service {command!r}")
  1340. h = cls(self.server.backend, args, proto) # type: ignore
  1341. h.handle()
  1342. class TCPGitServer(socketserver.TCPServer):
  1343. """TCP server for git protocol."""
  1344. allow_reuse_address = True
  1345. serve = socketserver.TCPServer.serve_forever
  1346. def _make_handler(
  1347. self,
  1348. request: socket.socket,
  1349. client_address: tuple[str, int],
  1350. server: socketserver.TCPServer,
  1351. ) -> TCPGitRequestHandler:
  1352. """Create request handler instance.
  1353. Args:
  1354. request: Request socket
  1355. client_address: Client address tuple
  1356. server: Server instance
  1357. Returns:
  1358. TCPGitRequestHandler instance
  1359. """
  1360. return TCPGitRequestHandler(self.handlers, request, client_address, server)
  1361. def __init__(
  1362. self,
  1363. backend: Backend,
  1364. listen_addr: str,
  1365. port: int = TCP_GIT_PORT,
  1366. handlers: dict[bytes, type[Handler]] | None = None,
  1367. ) -> None:
  1368. """Initialize TCP git server.
  1369. Args:
  1370. backend: Backend instance
  1371. listen_addr: Address to listen on
  1372. port: Port to listen on (default: TCP_GIT_PORT)
  1373. handlers: Optional dictionary of custom handlers
  1374. """
  1375. self.handlers = dict(DEFAULT_HANDLERS)
  1376. if handlers is not None:
  1377. self.handlers.update(handlers)
  1378. self.backend = backend
  1379. logger.info("Listening for TCP connections on %s:%d", listen_addr, port)
  1380. socketserver.TCPServer.__init__(self, (listen_addr, port), self._make_handler)
  1381. def verify_request(
  1382. self,
  1383. request: socket.socket | tuple[bytes, socket.socket],
  1384. client_address: tuple[str, int] | socket.socket,
  1385. ) -> bool:
  1386. """Verify incoming request.
  1387. Args:
  1388. request: Request socket
  1389. client_address: Client address tuple
  1390. Returns:
  1391. True to accept request
  1392. """
  1393. logger.info("Handling request from %s", client_address)
  1394. return True
  1395. def handle_error(
  1396. self,
  1397. request: socket.socket | tuple[bytes, socket.socket],
  1398. client_address: tuple[str, int] | socket.socket,
  1399. ) -> None:
  1400. """Handle request processing errors.
  1401. Args:
  1402. request: Request socket
  1403. client_address: Client address tuple
  1404. """
  1405. logger.exception(
  1406. "Exception happened during processing of request from %s",
  1407. client_address,
  1408. )
  1409. def main(argv: list[str] = sys.argv) -> None:
  1410. """Entry point for starting a TCP git server."""
  1411. import optparse
  1412. parser = optparse.OptionParser()
  1413. parser.add_option(
  1414. "-l",
  1415. "--listen_address",
  1416. dest="listen_address",
  1417. default="localhost",
  1418. help="Binding IP address.",
  1419. )
  1420. parser.add_option(
  1421. "-p",
  1422. "--port",
  1423. dest="port",
  1424. type=int,
  1425. default=TCP_GIT_PORT,
  1426. help="Binding TCP port.",
  1427. )
  1428. options, args = parser.parse_args(argv)
  1429. log_utils.default_logging_config()
  1430. if len(args) > 1:
  1431. gitdir = args[1]
  1432. else:
  1433. gitdir = "."
  1434. # TODO(jelmer): Support git-daemon-export-ok and --export-all.
  1435. backend = FileSystemBackend(gitdir)
  1436. server = TCPGitServer(backend, options.listen_address, options.port)
  1437. server.serve_forever()
  1438. def serve_command(
  1439. handler_cls: type[Handler],
  1440. argv: list[str] = sys.argv,
  1441. backend: Backend | None = None,
  1442. inf: IO[bytes] | None = None,
  1443. outf: IO[bytes] | None = None,
  1444. ) -> int:
  1445. """Serve a single command.
  1446. This is mostly useful for the implementation of commands used by e.g.
  1447. git+ssh.
  1448. Args:
  1449. handler_cls: `Handler` class to use for the request
  1450. argv: execv-style command-line arguments. Defaults to sys.argv.
  1451. backend: `Backend` to use
  1452. inf: File-like object to read from, defaults to standard input.
  1453. outf: File-like object to write to, defaults to standard output.
  1454. Returns: Exit code for use with sys.exit. 0 on success, 1 on failure.
  1455. """
  1456. if backend is None:
  1457. backend = FileSystemBackend()
  1458. if inf is None:
  1459. inf = sys.stdin.buffer
  1460. if outf is None:
  1461. outf = sys.stdout.buffer
  1462. def send_fn(data: bytes) -> None:
  1463. outf.write(data)
  1464. outf.flush()
  1465. proto = Protocol(inf.read, send_fn)
  1466. handler = handler_cls(backend, argv[1:], proto) # type: ignore[arg-type]
  1467. # FIXME: Catch exceptions and write a single-line summary to outf.
  1468. handler.handle()
  1469. return 0
  1470. def generate_info_refs(repo: "BaseRepo") -> Iterator[bytes]:
  1471. """Generate an info refs file."""
  1472. refs = repo.get_refs()
  1473. return write_info_refs(refs, repo.object_store)
  1474. def generate_objects_info_packs(repo: "BaseRepo") -> Iterator[bytes]:
  1475. """Generate an index for for packs."""
  1476. for pack in repo.object_store.packs:
  1477. yield (b"P " + os.fsencode(pack.data.filename) + b"\n")
  1478. def update_server_info(repo: "BaseRepo") -> None:
  1479. """Generate server info for dumb file access.
  1480. This generates info/refs and objects/info/packs,
  1481. similar to "git update-server-info".
  1482. """
  1483. repo._put_named_file(
  1484. os.path.join("info", "refs"), b"".join(generate_info_refs(repo))
  1485. )
  1486. repo._put_named_file(
  1487. os.path.join("objects", "info", "packs"),
  1488. b"".join(generate_objects_info_packs(repo)),
  1489. )
  1490. if __name__ == "__main__":
  1491. main()