refs.py 42 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372
  1. # refs.py -- For dealing with git refs
  2. # Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
  3. #
  4. # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
  5. # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
  6. # General Public License as public by the Free Software Foundation; version 2.0
  7. # or (at your option) any later version. You can redistribute it and/or
  8. # modify it under the terms of either of these two licenses.
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. #
  16. # You should have received a copy of the licenses; if not, see
  17. # <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
  18. # and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
  19. # License, Version 2.0.
  20. #
  21. """Ref handling."""
  22. import os
  23. import warnings
  24. from collections.abc import Iterator
  25. from contextlib import suppress
  26. from typing import Any, Optional, Union
  27. from .errors import PackedRefsException, RefFormatError
  28. from .file import GitFile, ensure_dir_exists
  29. from .objects import ZERO_SHA, ObjectID, Tag, git_line, valid_hexsha
  30. from .pack import ObjectContainer
  31. Ref = bytes
  32. HEADREF = b"HEAD"
  33. SYMREF = b"ref: "
  34. LOCAL_BRANCH_PREFIX = b"refs/heads/"
  35. LOCAL_TAG_PREFIX = b"refs/tags/"
  36. LOCAL_REMOTE_PREFIX = b"refs/remotes/"
  37. LOCAL_NOTES_PREFIX = b"refs/notes/"
  38. BAD_REF_CHARS = set(b"\177 ~^:?*[")
  39. PEELED_TAG_SUFFIX = b"^{}"
  40. # For backwards compatibility
  41. ANNOTATED_TAG_SUFFIX = PEELED_TAG_SUFFIX
  42. class SymrefLoop(Exception):
  43. """There is a loop between one or more symrefs."""
  44. def __init__(self, ref, depth) -> None:
  45. self.ref = ref
  46. self.depth = depth
  47. def parse_symref_value(contents: bytes) -> bytes:
  48. """Parse a symref value.
  49. Args:
  50. contents: Contents to parse
  51. Returns: Destination
  52. """
  53. if contents.startswith(SYMREF):
  54. return contents[len(SYMREF) :].rstrip(b"\r\n")
  55. raise ValueError(contents)
  56. def check_ref_format(refname: Ref) -> bool:
  57. """Check if a refname is correctly formatted.
  58. Implements all the same rules as git-check-ref-format[1].
  59. [1]
  60. http://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html
  61. Args:
  62. refname: The refname to check
  63. Returns: True if refname is valid, False otherwise
  64. """
  65. # These could be combined into one big expression, but are listed
  66. # separately to parallel [1].
  67. if b"/." in refname or refname.startswith(b"."):
  68. return False
  69. if b"/" not in refname:
  70. return False
  71. if b".." in refname:
  72. return False
  73. for i, c in enumerate(refname):
  74. if ord(refname[i : i + 1]) < 0o40 or c in BAD_REF_CHARS:
  75. return False
  76. if refname[-1] in b"/.":
  77. return False
  78. if refname.endswith(b".lock"):
  79. return False
  80. if b"@{" in refname:
  81. return False
  82. if b"\\" in refname:
  83. return False
  84. return True
  85. def parse_remote_ref(ref: bytes) -> tuple[bytes, bytes]:
  86. """Parse a remote ref into remote name and branch name.
  87. Args:
  88. ref: Remote ref like b"refs/remotes/origin/main"
  89. Returns:
  90. Tuple of (remote_name, branch_name)
  91. Raises:
  92. ValueError: If ref is not a valid remote ref
  93. """
  94. if not ref.startswith(LOCAL_REMOTE_PREFIX):
  95. raise ValueError(f"Not a remote ref: {ref!r}")
  96. # Remove the prefix
  97. remainder = ref[len(LOCAL_REMOTE_PREFIX) :]
  98. # Split into remote name and branch name
  99. parts = remainder.split(b"/", 1)
  100. if len(parts) != 2:
  101. raise ValueError(f"Invalid remote ref format: {ref!r}")
  102. remote_name, branch_name = parts
  103. return (remote_name, branch_name)
  104. class RefsContainer:
  105. """A container for refs."""
  106. def __init__(self, logger=None) -> None:
  107. self._logger = logger
  108. def _log(
  109. self,
  110. ref,
  111. old_sha,
  112. new_sha,
  113. committer=None,
  114. timestamp=None,
  115. timezone=None,
  116. message=None,
  117. ) -> None:
  118. if self._logger is None:
  119. return
  120. if message is None:
  121. return
  122. self._logger(ref, old_sha, new_sha, committer, timestamp, timezone, message)
  123. def set_symbolic_ref(
  124. self,
  125. name,
  126. other,
  127. committer=None,
  128. timestamp=None,
  129. timezone=None,
  130. message=None,
  131. ) -> None:
  132. """Make a ref point at another ref.
  133. Args:
  134. name: Name of the ref to set
  135. other: Name of the ref to point at
  136. message: Optional message
  137. """
  138. raise NotImplementedError(self.set_symbolic_ref)
  139. def get_packed_refs(self) -> dict[Ref, ObjectID]:
  140. """Get contents of the packed-refs file.
  141. Returns: Dictionary mapping ref names to SHA1s
  142. Note: Will return an empty dictionary when no packed-refs file is
  143. present.
  144. """
  145. raise NotImplementedError(self.get_packed_refs)
  146. def add_packed_refs(self, new_refs: dict[Ref, Optional[ObjectID]]) -> None:
  147. """Add the given refs as packed refs.
  148. Args:
  149. new_refs: A mapping of ref names to targets; if a target is None that
  150. means remove the ref
  151. """
  152. raise NotImplementedError(self.add_packed_refs)
  153. def get_peeled(self, name) -> Optional[ObjectID]:
  154. """Return the cached peeled value of a ref, if available.
  155. Args:
  156. name: Name of the ref to peel
  157. Returns: The peeled value of the ref. If the ref is known not point to
  158. a tag, this will be the SHA the ref refers to. If the ref may point
  159. to a tag, but no cached information is available, None is returned.
  160. """
  161. return None
  162. def import_refs(
  163. self,
  164. base: Ref,
  165. other: dict[Ref, ObjectID],
  166. committer: Optional[bytes] = None,
  167. timestamp: Optional[bytes] = None,
  168. timezone: Optional[bytes] = None,
  169. message: Optional[bytes] = None,
  170. prune: bool = False,
  171. ) -> None:
  172. if prune:
  173. to_delete = set(self.subkeys(base))
  174. else:
  175. to_delete = set()
  176. for name, value in other.items():
  177. if value is None:
  178. to_delete.add(name)
  179. else:
  180. self.set_if_equals(
  181. b"/".join((base, name)), None, value, message=message
  182. )
  183. if to_delete:
  184. try:
  185. to_delete.remove(name)
  186. except KeyError:
  187. pass
  188. for ref in to_delete:
  189. self.remove_if_equals(b"/".join((base, ref)), None, message=message)
  190. def allkeys(self) -> Iterator[Ref]:
  191. """All refs present in this container."""
  192. raise NotImplementedError(self.allkeys)
  193. def __iter__(self):
  194. return iter(self.allkeys())
  195. def keys(self, base=None):
  196. """Refs present in this container.
  197. Args:
  198. base: An optional base to return refs under.
  199. Returns: An unsorted set of valid refs in this container, including
  200. packed refs.
  201. """
  202. if base is not None:
  203. return self.subkeys(base)
  204. else:
  205. return self.allkeys()
  206. def subkeys(self, base):
  207. """Refs present in this container under a base.
  208. Args:
  209. base: The base to return refs under.
  210. Returns: A set of valid refs in this container under the base; the base
  211. prefix is stripped from the ref names returned.
  212. """
  213. keys = set()
  214. base_len = len(base) + 1
  215. for refname in self.allkeys():
  216. if refname.startswith(base):
  217. keys.add(refname[base_len:])
  218. return keys
  219. def as_dict(self, base=None) -> dict[Ref, ObjectID]:
  220. """Return the contents of this container as a dictionary."""
  221. ret = {}
  222. keys = self.keys(base)
  223. if base is None:
  224. base = b""
  225. else:
  226. base = base.rstrip(b"/")
  227. for key in keys:
  228. try:
  229. ret[key] = self[(base + b"/" + key).strip(b"/")]
  230. except (SymrefLoop, KeyError):
  231. continue # Unable to resolve
  232. return ret
  233. def _check_refname(self, name) -> None:
  234. """Ensure a refname is valid and lives in refs or is HEAD.
  235. HEAD is not a valid refname according to git-check-ref-format, but this
  236. class needs to be able to touch HEAD. Also, check_ref_format expects
  237. refnames without the leading 'refs/', but this class requires that
  238. so it cannot touch anything outside the refs dir (or HEAD).
  239. Args:
  240. name: The name of the reference.
  241. Raises:
  242. KeyError: if a refname is not HEAD or is otherwise not valid.
  243. """
  244. if name in (HEADREF, b"refs/stash"):
  245. return
  246. if not name.startswith(b"refs/") or not check_ref_format(name[5:]):
  247. raise RefFormatError(name)
  248. def read_ref(self, refname):
  249. """Read a reference without following any references.
  250. Args:
  251. refname: The name of the reference
  252. Returns: The contents of the ref file, or None if it does
  253. not exist.
  254. """
  255. contents = self.read_loose_ref(refname)
  256. if not contents:
  257. contents = self.get_packed_refs().get(refname, None)
  258. return contents
  259. def read_loose_ref(self, name) -> bytes:
  260. """Read a loose reference and return its contents.
  261. Args:
  262. name: the refname to read
  263. Returns: The contents of the ref file, or None if it does
  264. not exist.
  265. """
  266. raise NotImplementedError(self.read_loose_ref)
  267. def follow(self, name) -> tuple[list[bytes], bytes]:
  268. """Follow a reference name.
  269. Returns: a tuple of (refnames, sha), wheres refnames are the names of
  270. references in the chain
  271. """
  272. contents = SYMREF + name
  273. depth = 0
  274. refnames = []
  275. while contents.startswith(SYMREF):
  276. refname = contents[len(SYMREF) :]
  277. refnames.append(refname)
  278. contents = self.read_ref(refname)
  279. if not contents:
  280. break
  281. depth += 1
  282. if depth > 5:
  283. raise SymrefLoop(name, depth)
  284. return refnames, contents
  285. def __contains__(self, refname) -> bool:
  286. if self.read_ref(refname):
  287. return True
  288. return False
  289. def __getitem__(self, name) -> ObjectID:
  290. """Get the SHA1 for a reference name.
  291. This method follows all symbolic references.
  292. """
  293. _, sha = self.follow(name)
  294. if sha is None:
  295. raise KeyError(name)
  296. return sha
  297. def set_if_equals(
  298. self,
  299. name,
  300. old_ref,
  301. new_ref,
  302. committer=None,
  303. timestamp=None,
  304. timezone=None,
  305. message=None,
  306. ) -> bool:
  307. """Set a refname to new_ref only if it currently equals old_ref.
  308. This method follows all symbolic references if applicable for the
  309. subclass, and can be used to perform an atomic compare-and-swap
  310. operation.
  311. Args:
  312. name: The refname to set.
  313. old_ref: The old sha the refname must refer to, or None to set
  314. unconditionally.
  315. new_ref: The new sha the refname will refer to.
  316. message: Message for reflog
  317. Returns: True if the set was successful, False otherwise.
  318. """
  319. raise NotImplementedError(self.set_if_equals)
  320. def add_if_new(
  321. self, name, ref, committer=None, timestamp=None, timezone=None, message=None
  322. ) -> bool:
  323. """Add a new reference only if it does not already exist.
  324. Args:
  325. name: Ref name
  326. ref: Ref value
  327. """
  328. raise NotImplementedError(self.add_if_new)
  329. def __setitem__(self, name, ref) -> None:
  330. """Set a reference name to point to the given SHA1.
  331. This method follows all symbolic references if applicable for the
  332. subclass.
  333. Note: This method unconditionally overwrites the contents of a
  334. reference. To update atomically only if the reference has not
  335. changed, use set_if_equals().
  336. Args:
  337. name: The refname to set.
  338. ref: The new sha the refname will refer to.
  339. """
  340. if not (valid_hexsha(ref) or ref.startswith(SYMREF)):
  341. raise ValueError(f"{ref!r} must be a valid sha (40 chars) or a symref")
  342. self.set_if_equals(name, None, ref)
  343. def remove_if_equals(
  344. self,
  345. name,
  346. old_ref,
  347. committer=None,
  348. timestamp=None,
  349. timezone=None,
  350. message=None,
  351. ) -> bool:
  352. """Remove a refname only if it currently equals old_ref.
  353. This method does not follow symbolic references, even if applicable for
  354. the subclass. It can be used to perform an atomic compare-and-delete
  355. operation.
  356. Args:
  357. name: The refname to delete.
  358. old_ref: The old sha the refname must refer to, or None to
  359. delete unconditionally.
  360. message: Message for reflog
  361. Returns: True if the delete was successful, False otherwise.
  362. """
  363. raise NotImplementedError(self.remove_if_equals)
  364. def __delitem__(self, name) -> None:
  365. """Remove a refname.
  366. This method does not follow symbolic references, even if applicable for
  367. the subclass.
  368. Note: This method unconditionally deletes the contents of a reference.
  369. To delete atomically only if the reference has not changed, use
  370. remove_if_equals().
  371. Args:
  372. name: The refname to delete.
  373. """
  374. self.remove_if_equals(name, None)
  375. def get_symrefs(self):
  376. """Get a dict with all symrefs in this container.
  377. Returns: Dictionary mapping source ref to target ref
  378. """
  379. ret = {}
  380. for src in self.allkeys():
  381. try:
  382. dst = parse_symref_value(self.read_ref(src))
  383. except ValueError:
  384. pass
  385. else:
  386. ret[src] = dst
  387. return ret
  388. def pack_refs(self, all: bool = False) -> None:
  389. """Pack loose refs into packed-refs file.
  390. Args:
  391. all: If True, pack all refs. If False, only pack tags.
  392. """
  393. raise NotImplementedError(self.pack_refs)
  394. class DictRefsContainer(RefsContainer):
  395. """RefsContainer backed by a simple dict.
  396. This container does not support symbolic or packed references and is not
  397. threadsafe.
  398. """
  399. def __init__(self, refs, logger=None) -> None:
  400. super().__init__(logger=logger)
  401. self._refs = refs
  402. self._peeled: dict[bytes, ObjectID] = {}
  403. self._watchers: set[Any] = set()
  404. def allkeys(self):
  405. return self._refs.keys()
  406. def read_loose_ref(self, name):
  407. return self._refs.get(name, None)
  408. def get_packed_refs(self):
  409. return {}
  410. def _notify(self, ref, newsha) -> None:
  411. for watcher in self._watchers:
  412. watcher._notify((ref, newsha))
  413. def set_symbolic_ref(
  414. self,
  415. name: Ref,
  416. other: Ref,
  417. committer=None,
  418. timestamp=None,
  419. timezone=None,
  420. message=None,
  421. ) -> None:
  422. old = self.follow(name)[-1]
  423. new = SYMREF + other
  424. self._refs[name] = new
  425. self._notify(name, new)
  426. self._log(
  427. name,
  428. old,
  429. new,
  430. committer=committer,
  431. timestamp=timestamp,
  432. timezone=timezone,
  433. message=message,
  434. )
  435. def set_if_equals(
  436. self,
  437. name,
  438. old_ref,
  439. new_ref,
  440. committer=None,
  441. timestamp=None,
  442. timezone=None,
  443. message=None,
  444. ) -> bool:
  445. if old_ref is not None and self._refs.get(name, ZERO_SHA) != old_ref:
  446. return False
  447. # Only update the specific ref requested, not the whole chain
  448. self._check_refname(name)
  449. old = self._refs.get(name)
  450. self._refs[name] = new_ref
  451. self._notify(name, new_ref)
  452. self._log(
  453. name,
  454. old,
  455. new_ref,
  456. committer=committer,
  457. timestamp=timestamp,
  458. timezone=timezone,
  459. message=message,
  460. )
  461. return True
  462. def add_if_new(
  463. self,
  464. name: Ref,
  465. ref: ObjectID,
  466. committer=None,
  467. timestamp=None,
  468. timezone=None,
  469. message: Optional[bytes] = None,
  470. ) -> bool:
  471. if name in self._refs:
  472. return False
  473. self._refs[name] = ref
  474. self._notify(name, ref)
  475. self._log(
  476. name,
  477. None,
  478. ref,
  479. committer=committer,
  480. timestamp=timestamp,
  481. timezone=timezone,
  482. message=message,
  483. )
  484. return True
  485. def remove_if_equals(
  486. self,
  487. name,
  488. old_ref,
  489. committer=None,
  490. timestamp=None,
  491. timezone=None,
  492. message=None,
  493. ) -> bool:
  494. if old_ref is not None and self._refs.get(name, ZERO_SHA) != old_ref:
  495. return False
  496. try:
  497. old = self._refs.pop(name)
  498. except KeyError:
  499. pass
  500. else:
  501. self._notify(name, None)
  502. self._log(
  503. name,
  504. old,
  505. None,
  506. committer=committer,
  507. timestamp=timestamp,
  508. timezone=timezone,
  509. message=message,
  510. )
  511. return True
  512. def get_peeled(self, name):
  513. return self._peeled.get(name)
  514. def _update(self, refs) -> None:
  515. """Update multiple refs; intended only for testing."""
  516. # TODO(dborowitz): replace this with a public function that uses
  517. # set_if_equal.
  518. for ref, sha in refs.items():
  519. self.set_if_equals(ref, None, sha)
  520. def _update_peeled(self, peeled) -> None:
  521. """Update cached peeled refs; intended only for testing."""
  522. self._peeled.update(peeled)
  523. class InfoRefsContainer(RefsContainer):
  524. """Refs container that reads refs from a info/refs file."""
  525. def __init__(self, f) -> None:
  526. self._refs = {}
  527. self._peeled = {}
  528. refs = read_info_refs(f)
  529. (self._refs, self._peeled) = split_peeled_refs(refs)
  530. def allkeys(self):
  531. return self._refs.keys()
  532. def read_loose_ref(self, name):
  533. return self._refs.get(name, None)
  534. def get_packed_refs(self):
  535. return {}
  536. def get_peeled(self, name):
  537. try:
  538. return self._peeled[name]
  539. except KeyError:
  540. return self._refs[name]
  541. class DiskRefsContainer(RefsContainer):
  542. """Refs container that reads refs from disk."""
  543. def __init__(
  544. self,
  545. path: Union[str, bytes, os.PathLike],
  546. worktree_path: Optional[Union[str, bytes, os.PathLike]] = None,
  547. logger=None,
  548. ) -> None:
  549. super().__init__(logger=logger)
  550. # Convert path-like objects to strings, then to bytes for Git compatibility
  551. self.path = os.fsencode(os.fspath(path))
  552. if worktree_path is None:
  553. self.worktree_path = self.path
  554. else:
  555. self.worktree_path = os.fsencode(os.fspath(worktree_path))
  556. self._packed_refs = None
  557. self._peeled_refs = None
  558. def __repr__(self) -> str:
  559. return f"{self.__class__.__name__}({self.path!r})"
  560. def subkeys(self, base):
  561. subkeys = set()
  562. path = self.refpath(base)
  563. for root, unused_dirs, files in os.walk(path):
  564. dir = root[len(path) :]
  565. if os.path.sep != "/":
  566. dir = dir.replace(os.fsencode(os.path.sep), b"/")
  567. dir = dir.strip(b"/")
  568. for filename in files:
  569. refname = b"/".join(([dir] if dir else []) + [filename])
  570. # check_ref_format requires at least one /, so we prepend the
  571. # base before calling it.
  572. if check_ref_format(base + b"/" + refname):
  573. subkeys.add(refname)
  574. for key in self.get_packed_refs():
  575. if key.startswith(base):
  576. subkeys.add(key[len(base) :].strip(b"/"))
  577. return subkeys
  578. def allkeys(self):
  579. allkeys = set()
  580. if os.path.exists(self.refpath(HEADREF)):
  581. allkeys.add(HEADREF)
  582. path = self.refpath(b"")
  583. refspath = self.refpath(b"refs")
  584. for root, unused_dirs, files in os.walk(refspath):
  585. dir = root[len(path) :]
  586. if os.path.sep != "/":
  587. dir = dir.replace(os.fsencode(os.path.sep), b"/")
  588. for filename in files:
  589. refname = b"/".join([dir, filename])
  590. if check_ref_format(refname):
  591. allkeys.add(refname)
  592. allkeys.update(self.get_packed_refs())
  593. return allkeys
  594. def refpath(self, name):
  595. """Return the disk path of a ref."""
  596. if os.path.sep != "/":
  597. name = name.replace(b"/", os.fsencode(os.path.sep))
  598. # TODO: as the 'HEAD' reference is working tree specific, it
  599. # should actually not be a part of RefsContainer
  600. if name == HEADREF:
  601. return os.path.join(self.worktree_path, name)
  602. else:
  603. return os.path.join(self.path, name)
  604. def get_packed_refs(self):
  605. """Get contents of the packed-refs file.
  606. Returns: Dictionary mapping ref names to SHA1s
  607. Note: Will return an empty dictionary when no packed-refs file is
  608. present.
  609. """
  610. # TODO: invalidate the cache on repacking
  611. if self._packed_refs is None:
  612. # set both to empty because we want _peeled_refs to be
  613. # None if and only if _packed_refs is also None.
  614. self._packed_refs = {}
  615. self._peeled_refs = {}
  616. path = os.path.join(self.path, b"packed-refs")
  617. try:
  618. f = GitFile(path, "rb")
  619. except FileNotFoundError:
  620. return {}
  621. with f:
  622. first_line = next(iter(f)).rstrip()
  623. if first_line.startswith(b"# pack-refs") and b" peeled" in first_line:
  624. for sha, name, peeled in read_packed_refs_with_peeled(f):
  625. self._packed_refs[name] = sha
  626. if peeled:
  627. self._peeled_refs[name] = peeled
  628. else:
  629. f.seek(0)
  630. for sha, name in read_packed_refs(f):
  631. self._packed_refs[name] = sha
  632. return self._packed_refs
  633. def add_packed_refs(self, new_refs: dict[Ref, Optional[ObjectID]]) -> None:
  634. """Add the given refs as packed refs.
  635. Args:
  636. new_refs: A mapping of ref names to targets; if a target is None that
  637. means remove the ref
  638. """
  639. if not new_refs:
  640. return
  641. path = os.path.join(self.path, b"packed-refs")
  642. with GitFile(path, "wb") as f:
  643. # reread cached refs from disk, while holding the lock
  644. packed_refs = self.get_packed_refs().copy()
  645. for ref, target in new_refs.items():
  646. # sanity check
  647. if ref == HEADREF:
  648. raise ValueError("cannot pack HEAD")
  649. # remove any loose refs pointing to this one -- please
  650. # note that this bypasses remove_if_equals as we don't
  651. # want to affect packed refs in here
  652. with suppress(OSError):
  653. os.remove(self.refpath(ref))
  654. if target is not None:
  655. packed_refs[ref] = target
  656. else:
  657. packed_refs.pop(ref, None)
  658. write_packed_refs(f, packed_refs, self._peeled_refs)
  659. self._packed_refs = packed_refs
  660. def get_peeled(self, name):
  661. """Return the cached peeled value of a ref, if available.
  662. Args:
  663. name: Name of the ref to peel
  664. Returns: The peeled value of the ref. If the ref is known not point to
  665. a tag, this will be the SHA the ref refers to. If the ref may point
  666. to a tag, but no cached information is available, None is returned.
  667. """
  668. self.get_packed_refs()
  669. if self._peeled_refs is None or name not in self._packed_refs:
  670. # No cache: no peeled refs were read, or this ref is loose
  671. return None
  672. if name in self._peeled_refs:
  673. return self._peeled_refs[name]
  674. else:
  675. # Known not peelable
  676. return self[name]
  677. def read_loose_ref(self, name):
  678. """Read a reference file and return its contents.
  679. If the reference file a symbolic reference, only read the first line of
  680. the file. Otherwise, only read the first 40 bytes.
  681. Args:
  682. name: the refname to read, relative to refpath
  683. Returns: The contents of the ref file, or None if the file does not
  684. exist.
  685. Raises:
  686. IOError: if any other error occurs
  687. """
  688. filename = self.refpath(name)
  689. try:
  690. with GitFile(filename, "rb") as f:
  691. header = f.read(len(SYMREF))
  692. if header == SYMREF:
  693. # Read only the first line
  694. return header + next(iter(f)).rstrip(b"\r\n")
  695. else:
  696. # Read only the first 40 bytes
  697. return header + f.read(40 - len(SYMREF))
  698. except (OSError, UnicodeError):
  699. # don't assume anything specific about the error; in
  700. # particular, invalid or forbidden paths can raise weird
  701. # errors depending on the specific operating system
  702. return None
  703. def _remove_packed_ref(self, name) -> None:
  704. if self._packed_refs is None:
  705. return
  706. filename = os.path.join(self.path, b"packed-refs")
  707. # reread cached refs from disk, while holding the lock
  708. f = GitFile(filename, "wb")
  709. try:
  710. self._packed_refs = None
  711. self.get_packed_refs()
  712. if name not in self._packed_refs:
  713. return
  714. del self._packed_refs[name]
  715. with suppress(KeyError):
  716. del self._peeled_refs[name]
  717. write_packed_refs(f, self._packed_refs, self._peeled_refs)
  718. f.close()
  719. finally:
  720. f.abort()
  721. def set_symbolic_ref(
  722. self,
  723. name,
  724. other,
  725. committer=None,
  726. timestamp=None,
  727. timezone=None,
  728. message=None,
  729. ) -> None:
  730. """Make a ref point at another ref.
  731. Args:
  732. name: Name of the ref to set
  733. other: Name of the ref to point at
  734. message: Optional message to describe the change
  735. """
  736. self._check_refname(name)
  737. self._check_refname(other)
  738. filename = self.refpath(name)
  739. f = GitFile(filename, "wb")
  740. try:
  741. f.write(SYMREF + other + b"\n")
  742. sha = self.follow(name)[-1]
  743. self._log(
  744. name,
  745. sha,
  746. sha,
  747. committer=committer,
  748. timestamp=timestamp,
  749. timezone=timezone,
  750. message=message,
  751. )
  752. except BaseException:
  753. f.abort()
  754. raise
  755. else:
  756. f.close()
  757. def set_if_equals(
  758. self,
  759. name,
  760. old_ref,
  761. new_ref,
  762. committer=None,
  763. timestamp=None,
  764. timezone=None,
  765. message=None,
  766. ) -> bool:
  767. """Set a refname to new_ref only if it currently equals old_ref.
  768. This method follows all symbolic references, and can be used to perform
  769. an atomic compare-and-swap operation.
  770. Args:
  771. name: The refname to set.
  772. old_ref: The old sha the refname must refer to, or None to set
  773. unconditionally.
  774. new_ref: The new sha the refname will refer to.
  775. message: Set message for reflog
  776. Returns: True if the set was successful, False otherwise.
  777. """
  778. self._check_refname(name)
  779. try:
  780. realnames, _ = self.follow(name)
  781. realname = realnames[-1]
  782. except (KeyError, IndexError, SymrefLoop):
  783. realname = name
  784. filename = self.refpath(realname)
  785. # make sure none of the ancestor folders is in packed refs
  786. probe_ref = os.path.dirname(realname)
  787. packed_refs = self.get_packed_refs()
  788. while probe_ref:
  789. if packed_refs.get(probe_ref, None) is not None:
  790. raise NotADirectoryError(filename)
  791. probe_ref = os.path.dirname(probe_ref)
  792. ensure_dir_exists(os.path.dirname(filename))
  793. with GitFile(filename, "wb") as f:
  794. if old_ref is not None:
  795. try:
  796. # read again while holding the lock to handle race conditions
  797. orig_ref = self.read_loose_ref(realname)
  798. if orig_ref is None:
  799. orig_ref = self.get_packed_refs().get(realname, ZERO_SHA)
  800. if orig_ref != old_ref:
  801. f.abort()
  802. return False
  803. except OSError:
  804. f.abort()
  805. raise
  806. # Check if ref already has the desired value while holding the lock
  807. # This avoids fsync when ref is unchanged but still detects lock conflicts
  808. current_ref = self.read_loose_ref(realname)
  809. if current_ref is None:
  810. current_ref = packed_refs.get(realname, None)
  811. if current_ref is not None and current_ref == new_ref:
  812. # Ref already has desired value, abort write to avoid fsync
  813. f.abort()
  814. return True
  815. try:
  816. f.write(new_ref + b"\n")
  817. except OSError:
  818. f.abort()
  819. raise
  820. self._log(
  821. realname,
  822. old_ref,
  823. new_ref,
  824. committer=committer,
  825. timestamp=timestamp,
  826. timezone=timezone,
  827. message=message,
  828. )
  829. return True
  830. def add_if_new(
  831. self,
  832. name: bytes,
  833. ref: bytes,
  834. committer=None,
  835. timestamp=None,
  836. timezone=None,
  837. message: Optional[bytes] = None,
  838. ) -> bool:
  839. """Add a new reference only if it does not already exist.
  840. This method follows symrefs, and only ensures that the last ref in the
  841. chain does not exist.
  842. Args:
  843. name: The refname to set.
  844. ref: The new sha the refname will refer to.
  845. message: Optional message for reflog
  846. Returns: True if the add was successful, False otherwise.
  847. """
  848. try:
  849. realnames, contents = self.follow(name)
  850. if contents is not None:
  851. return False
  852. realname = realnames[-1]
  853. except (KeyError, IndexError):
  854. realname = name
  855. self._check_refname(realname)
  856. filename = self.refpath(realname)
  857. ensure_dir_exists(os.path.dirname(filename))
  858. with GitFile(filename, "wb") as f:
  859. if os.path.exists(filename) or name in self.get_packed_refs():
  860. f.abort()
  861. return False
  862. try:
  863. f.write(ref + b"\n")
  864. except OSError:
  865. f.abort()
  866. raise
  867. else:
  868. self._log(
  869. name,
  870. None,
  871. ref,
  872. committer=committer,
  873. timestamp=timestamp,
  874. timezone=timezone,
  875. message=message,
  876. )
  877. return True
  878. def remove_if_equals(
  879. self,
  880. name,
  881. old_ref,
  882. committer=None,
  883. timestamp=None,
  884. timezone=None,
  885. message=None,
  886. ) -> bool:
  887. """Remove a refname only if it currently equals old_ref.
  888. This method does not follow symbolic references. It can be used to
  889. perform an atomic compare-and-delete operation.
  890. Args:
  891. name: The refname to delete.
  892. old_ref: The old sha the refname must refer to, or None to
  893. delete unconditionally.
  894. message: Optional message
  895. Returns: True if the delete was successful, False otherwise.
  896. """
  897. self._check_refname(name)
  898. filename = self.refpath(name)
  899. ensure_dir_exists(os.path.dirname(filename))
  900. f = GitFile(filename, "wb")
  901. try:
  902. if old_ref is not None:
  903. orig_ref = self.read_loose_ref(name)
  904. if orig_ref is None:
  905. orig_ref = self.get_packed_refs().get(name, ZERO_SHA)
  906. if orig_ref != old_ref:
  907. return False
  908. # remove the reference file itself
  909. try:
  910. found = os.path.lexists(filename)
  911. except OSError:
  912. # may only be packed, or otherwise unstorable
  913. found = False
  914. if found:
  915. os.remove(filename)
  916. self._remove_packed_ref(name)
  917. self._log(
  918. name,
  919. old_ref,
  920. None,
  921. committer=committer,
  922. timestamp=timestamp,
  923. timezone=timezone,
  924. message=message,
  925. )
  926. finally:
  927. # never write, we just wanted the lock
  928. f.abort()
  929. # outside of the lock, clean-up any parent directory that might now
  930. # be empty. this ensures that re-creating a reference of the same
  931. # name of what was previously a directory works as expected
  932. parent = name
  933. while True:
  934. try:
  935. parent, _ = parent.rsplit(b"/", 1)
  936. except ValueError:
  937. break
  938. if parent == b"refs":
  939. break
  940. parent_filename = self.refpath(parent)
  941. try:
  942. os.rmdir(parent_filename)
  943. except OSError:
  944. # this can be caused by the parent directory being
  945. # removed by another process, being not empty, etc.
  946. # in any case, this is non fatal because we already
  947. # removed the reference, just ignore it
  948. break
  949. return True
  950. def pack_refs(self, all: bool = False) -> None:
  951. """Pack loose refs into packed-refs file.
  952. Args:
  953. all: If True, pack all refs. If False, only pack tags.
  954. """
  955. refs_to_pack: dict[Ref, Optional[ObjectID]] = {}
  956. for ref in self.allkeys():
  957. if ref == HEADREF:
  958. # Never pack HEAD
  959. continue
  960. if all or ref.startswith(LOCAL_TAG_PREFIX):
  961. try:
  962. sha = self[ref]
  963. if sha:
  964. refs_to_pack[ref] = sha
  965. except KeyError:
  966. # Broken ref, skip it
  967. pass
  968. if refs_to_pack:
  969. self.add_packed_refs(refs_to_pack)
  970. def _split_ref_line(line):
  971. """Split a single ref line into a tuple of SHA1 and name."""
  972. fields = line.rstrip(b"\n\r").split(b" ")
  973. if len(fields) != 2:
  974. raise PackedRefsException(f"invalid ref line {line!r}")
  975. sha, name = fields
  976. if not valid_hexsha(sha):
  977. raise PackedRefsException(f"Invalid hex sha {sha!r}")
  978. if not check_ref_format(name):
  979. raise PackedRefsException(f"invalid ref name {name!r}")
  980. return (sha, name)
  981. def read_packed_refs(f):
  982. """Read a packed refs file.
  983. Args:
  984. f: file-like object to read from
  985. Returns: Iterator over tuples with SHA1s and ref names.
  986. """
  987. for line in f:
  988. if line.startswith(b"#"):
  989. # Comment
  990. continue
  991. if line.startswith(b"^"):
  992. raise PackedRefsException("found peeled ref in packed-refs without peeled")
  993. yield _split_ref_line(line)
  994. def read_packed_refs_with_peeled(f):
  995. """Read a packed refs file including peeled refs.
  996. Assumes the "# pack-refs with: peeled" line was already read. Yields tuples
  997. with ref names, SHA1s, and peeled SHA1s (or None).
  998. Args:
  999. f: file-like object to read from, seek'ed to the second line
  1000. """
  1001. last = None
  1002. for line in f:
  1003. if line[0] == b"#":
  1004. continue
  1005. line = line.rstrip(b"\r\n")
  1006. if line.startswith(b"^"):
  1007. if not last:
  1008. raise PackedRefsException("unexpected peeled ref line")
  1009. if not valid_hexsha(line[1:]):
  1010. raise PackedRefsException(f"Invalid hex sha {line[1:]!r}")
  1011. sha, name = _split_ref_line(last)
  1012. last = None
  1013. yield (sha, name, line[1:])
  1014. else:
  1015. if last:
  1016. sha, name = _split_ref_line(last)
  1017. yield (sha, name, None)
  1018. last = line
  1019. if last:
  1020. sha, name = _split_ref_line(last)
  1021. yield (sha, name, None)
  1022. def write_packed_refs(f, packed_refs, peeled_refs=None) -> None:
  1023. """Write a packed refs file.
  1024. Args:
  1025. f: empty file-like object to write to
  1026. packed_refs: dict of refname to sha of packed refs to write
  1027. peeled_refs: dict of refname to peeled value of sha
  1028. """
  1029. if peeled_refs is None:
  1030. peeled_refs = {}
  1031. else:
  1032. f.write(b"# pack-refs with: peeled\n")
  1033. for refname in sorted(packed_refs.keys()):
  1034. f.write(git_line(packed_refs[refname], refname))
  1035. if refname in peeled_refs:
  1036. f.write(b"^" + peeled_refs[refname] + b"\n")
  1037. def read_info_refs(f):
  1038. ret = {}
  1039. for line in f.readlines():
  1040. (sha, name) = line.rstrip(b"\r\n").split(b"\t", 1)
  1041. ret[name] = sha
  1042. return ret
  1043. def write_info_refs(refs, store: ObjectContainer):
  1044. """Generate info refs."""
  1045. # TODO: Avoid recursive import :(
  1046. from .object_store import peel_sha
  1047. for name, sha in sorted(refs.items()):
  1048. # get_refs() includes HEAD as a special case, but we don't want to
  1049. # advertise it
  1050. if name == HEADREF:
  1051. continue
  1052. try:
  1053. o = store[sha]
  1054. except KeyError:
  1055. continue
  1056. unpeeled, peeled = peel_sha(store, sha)
  1057. yield o.id + b"\t" + name + b"\n"
  1058. if o.id != peeled.id:
  1059. yield peeled.id + b"\t" + name + PEELED_TAG_SUFFIX + b"\n"
  1060. def is_local_branch(x):
  1061. return x.startswith(LOCAL_BRANCH_PREFIX)
  1062. def strip_peeled_refs(refs):
  1063. """Remove all peeled refs."""
  1064. return {
  1065. ref: sha for (ref, sha) in refs.items() if not ref.endswith(PEELED_TAG_SUFFIX)
  1066. }
  1067. def split_peeled_refs(refs):
  1068. """Split peeled refs from regular refs."""
  1069. peeled = {}
  1070. regular = {}
  1071. for ref, sha in refs.items():
  1072. if ref.endswith(PEELED_TAG_SUFFIX):
  1073. peeled[ref[: -len(PEELED_TAG_SUFFIX)]] = sha
  1074. else:
  1075. regular[ref] = sha
  1076. return regular, peeled
  1077. def _set_origin_head(refs, origin, origin_head) -> None:
  1078. # set refs/remotes/origin/HEAD
  1079. origin_base = b"refs/remotes/" + origin + b"/"
  1080. if origin_head and origin_head.startswith(LOCAL_BRANCH_PREFIX):
  1081. origin_ref = origin_base + HEADREF
  1082. target_ref = origin_base + origin_head[len(LOCAL_BRANCH_PREFIX) :]
  1083. if target_ref in refs:
  1084. refs.set_symbolic_ref(origin_ref, target_ref)
  1085. def _set_default_branch(
  1086. refs: RefsContainer,
  1087. origin: bytes,
  1088. origin_head: Optional[bytes],
  1089. branch: bytes,
  1090. ref_message: Optional[bytes],
  1091. ) -> bytes:
  1092. """Set the default branch."""
  1093. origin_base = b"refs/remotes/" + origin + b"/"
  1094. if branch:
  1095. origin_ref = origin_base + branch
  1096. if origin_ref in refs:
  1097. local_ref = LOCAL_BRANCH_PREFIX + branch
  1098. refs.add_if_new(local_ref, refs[origin_ref], ref_message)
  1099. head_ref = local_ref
  1100. elif LOCAL_TAG_PREFIX + branch in refs:
  1101. head_ref = LOCAL_TAG_PREFIX + branch
  1102. else:
  1103. raise ValueError(f"{os.fsencode(branch)!r} is not a valid branch or tag")
  1104. elif origin_head:
  1105. head_ref = origin_head
  1106. if origin_head.startswith(LOCAL_BRANCH_PREFIX):
  1107. origin_ref = origin_base + origin_head[len(LOCAL_BRANCH_PREFIX) :]
  1108. else:
  1109. origin_ref = origin_head
  1110. try:
  1111. refs.add_if_new(head_ref, refs[origin_ref], ref_message)
  1112. except KeyError:
  1113. pass
  1114. else:
  1115. raise ValueError("neither origin_head nor branch are provided")
  1116. return head_ref
  1117. def _set_head(refs, head_ref, ref_message):
  1118. if head_ref.startswith(LOCAL_TAG_PREFIX):
  1119. # detach HEAD at specified tag
  1120. head = refs[head_ref]
  1121. if isinstance(head, Tag):
  1122. _cls, obj = head.object
  1123. head = obj.get_object(obj).id
  1124. del refs[HEADREF]
  1125. refs.set_if_equals(HEADREF, None, head, message=ref_message)
  1126. else:
  1127. # set HEAD to specific branch
  1128. try:
  1129. head = refs[head_ref]
  1130. refs.set_symbolic_ref(HEADREF, head_ref)
  1131. refs.set_if_equals(HEADREF, None, head, message=ref_message)
  1132. except KeyError:
  1133. head = None
  1134. return head
  1135. def _import_remote_refs(
  1136. refs_container: RefsContainer,
  1137. remote_name: str,
  1138. refs: dict[str, str],
  1139. message: Optional[bytes] = None,
  1140. prune: bool = False,
  1141. prune_tags: bool = False,
  1142. ) -> None:
  1143. stripped_refs = strip_peeled_refs(refs)
  1144. branches = {
  1145. n[len(LOCAL_BRANCH_PREFIX) :]: v
  1146. for (n, v) in stripped_refs.items()
  1147. if n.startswith(LOCAL_BRANCH_PREFIX)
  1148. }
  1149. refs_container.import_refs(
  1150. b"refs/remotes/" + remote_name.encode(),
  1151. branches,
  1152. message=message,
  1153. prune=prune,
  1154. )
  1155. tags = {
  1156. n[len(LOCAL_TAG_PREFIX) :]: v
  1157. for (n, v) in stripped_refs.items()
  1158. if n.startswith(LOCAL_TAG_PREFIX) and not n.endswith(PEELED_TAG_SUFFIX)
  1159. }
  1160. refs_container.import_refs(
  1161. LOCAL_TAG_PREFIX, tags, message=message, prune=prune_tags
  1162. )
  1163. def serialize_refs(store, refs):
  1164. # TODO: Avoid recursive import :(
  1165. from .object_store import peel_sha
  1166. ret = {}
  1167. for ref, sha in refs.items():
  1168. try:
  1169. unpeeled, peeled = peel_sha(store, sha)
  1170. except KeyError:
  1171. warnings.warn(
  1172. "ref {} points at non-present sha {}".format(
  1173. ref.decode("utf-8", "replace"), sha.decode("ascii")
  1174. ),
  1175. UserWarning,
  1176. )
  1177. continue
  1178. else:
  1179. if isinstance(unpeeled, Tag):
  1180. ret[ref + PEELED_TAG_SUFFIX] = peeled.id
  1181. ret[ref] = unpeeled.id
  1182. return ret