refs.py 40 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298
  1. # refs.py -- For dealing with git refs
  2. # Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
  3. #
  4. # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
  5. # General Public License as public by the Free Software Foundation; version 2.0
  6. # or (at your option) any later version. You can redistribute it and/or
  7. # modify it under the terms of either of these two licenses.
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. #
  15. # You should have received a copy of the licenses; if not, see
  16. # <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
  17. # and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
  18. # License, Version 2.0.
  19. #
  20. """Ref handling."""
  21. import os
  22. import warnings
  23. from collections.abc import Iterator
  24. from contextlib import suppress
  25. from typing import Any, Optional
  26. from .errors import PackedRefsException, RefFormatError
  27. from .file import GitFile, ensure_dir_exists
  28. from .objects import ZERO_SHA, ObjectID, Tag, git_line, valid_hexsha
  29. from .pack import ObjectContainer
  30. Ref = bytes
  31. HEADREF = b"HEAD"
  32. SYMREF = b"ref: "
  33. LOCAL_BRANCH_PREFIX = b"refs/heads/"
  34. LOCAL_TAG_PREFIX = b"refs/tags/"
  35. LOCAL_REMOTE_PREFIX = b"refs/remotes/"
  36. BAD_REF_CHARS = set(b"\177 ~^:?*[")
  37. PEELED_TAG_SUFFIX = b"^{}"
  38. # For backwards compatibility
  39. ANNOTATED_TAG_SUFFIX = PEELED_TAG_SUFFIX
  40. class SymrefLoop(Exception):
  41. """There is a loop between one or more symrefs."""
  42. def __init__(self, ref, depth) -> None:
  43. self.ref = ref
  44. self.depth = depth
  45. def parse_symref_value(contents: bytes) -> bytes:
  46. """Parse a symref value.
  47. Args:
  48. contents: Contents to parse
  49. Returns: Destination
  50. """
  51. if contents.startswith(SYMREF):
  52. return contents[len(SYMREF) :].rstrip(b"\r\n")
  53. raise ValueError(contents)
  54. def check_ref_format(refname: Ref) -> bool:
  55. """Check if a refname is correctly formatted.
  56. Implements all the same rules as git-check-ref-format[1].
  57. [1]
  58. http://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html
  59. Args:
  60. refname: The refname to check
  61. Returns: True if refname is valid, False otherwise
  62. """
  63. # These could be combined into one big expression, but are listed
  64. # separately to parallel [1].
  65. if b"/." in refname or refname.startswith(b"."):
  66. return False
  67. if b"/" not in refname:
  68. return False
  69. if b".." in refname:
  70. return False
  71. for i, c in enumerate(refname):
  72. if ord(refname[i : i + 1]) < 0o40 or c in BAD_REF_CHARS:
  73. return False
  74. if refname[-1] in b"/.":
  75. return False
  76. if refname.endswith(b".lock"):
  77. return False
  78. if b"@{" in refname:
  79. return False
  80. if b"\\" in refname:
  81. return False
  82. return True
  83. class RefsContainer:
  84. """A container for refs."""
  85. def __init__(self, logger=None) -> None:
  86. self._logger = logger
  87. def _log(
  88. self,
  89. ref,
  90. old_sha,
  91. new_sha,
  92. committer=None,
  93. timestamp=None,
  94. timezone=None,
  95. message=None,
  96. ) -> None:
  97. if self._logger is None:
  98. return
  99. if message is None:
  100. return
  101. self._logger(ref, old_sha, new_sha, committer, timestamp, timezone, message)
  102. def set_symbolic_ref(
  103. self,
  104. name,
  105. other,
  106. committer=None,
  107. timestamp=None,
  108. timezone=None,
  109. message=None,
  110. ) -> None:
  111. """Make a ref point at another ref.
  112. Args:
  113. name: Name of the ref to set
  114. other: Name of the ref to point at
  115. message: Optional message
  116. """
  117. raise NotImplementedError(self.set_symbolic_ref)
  118. def get_packed_refs(self) -> dict[Ref, ObjectID]:
  119. """Get contents of the packed-refs file.
  120. Returns: Dictionary mapping ref names to SHA1s
  121. Note: Will return an empty dictionary when no packed-refs file is
  122. present.
  123. """
  124. raise NotImplementedError(self.get_packed_refs)
  125. def add_packed_refs(self, new_refs: dict[Ref, Optional[ObjectID]]) -> None:
  126. """Add the given refs as packed refs.
  127. Args:
  128. new_refs: A mapping of ref names to targets; if a target is None that
  129. means remove the ref
  130. """
  131. raise NotImplementedError(self.add_packed_refs)
  132. def get_peeled(self, name) -> Optional[ObjectID]:
  133. """Return the cached peeled value of a ref, if available.
  134. Args:
  135. name: Name of the ref to peel
  136. Returns: The peeled value of the ref. If the ref is known not point to
  137. a tag, this will be the SHA the ref refers to. If the ref may point
  138. to a tag, but no cached information is available, None is returned.
  139. """
  140. return None
  141. def import_refs(
  142. self,
  143. base: Ref,
  144. other: dict[Ref, ObjectID],
  145. committer: Optional[bytes] = None,
  146. timestamp: Optional[bytes] = None,
  147. timezone: Optional[bytes] = None,
  148. message: Optional[bytes] = None,
  149. prune: bool = False,
  150. ) -> None:
  151. if prune:
  152. to_delete = set(self.subkeys(base))
  153. else:
  154. to_delete = set()
  155. for name, value in other.items():
  156. if value is None:
  157. to_delete.add(name)
  158. else:
  159. self.set_if_equals(
  160. b"/".join((base, name)), None, value, message=message
  161. )
  162. if to_delete:
  163. try:
  164. to_delete.remove(name)
  165. except KeyError:
  166. pass
  167. for ref in to_delete:
  168. self.remove_if_equals(b"/".join((base, ref)), None, message=message)
  169. def allkeys(self) -> Iterator[Ref]:
  170. """All refs present in this container."""
  171. raise NotImplementedError(self.allkeys)
  172. def __iter__(self):
  173. return iter(self.allkeys())
  174. def keys(self, base=None):
  175. """Refs present in this container.
  176. Args:
  177. base: An optional base to return refs under.
  178. Returns: An unsorted set of valid refs in this container, including
  179. packed refs.
  180. """
  181. if base is not None:
  182. return self.subkeys(base)
  183. else:
  184. return self.allkeys()
  185. def subkeys(self, base):
  186. """Refs present in this container under a base.
  187. Args:
  188. base: The base to return refs under.
  189. Returns: A set of valid refs in this container under the base; the base
  190. prefix is stripped from the ref names returned.
  191. """
  192. keys = set()
  193. base_len = len(base) + 1
  194. for refname in self.allkeys():
  195. if refname.startswith(base):
  196. keys.add(refname[base_len:])
  197. return keys
  198. def as_dict(self, base=None) -> dict[Ref, ObjectID]:
  199. """Return the contents of this container as a dictionary."""
  200. ret = {}
  201. keys = self.keys(base)
  202. if base is None:
  203. base = b""
  204. else:
  205. base = base.rstrip(b"/")
  206. for key in keys:
  207. try:
  208. ret[key] = self[(base + b"/" + key).strip(b"/")]
  209. except (SymrefLoop, KeyError):
  210. continue # Unable to resolve
  211. return ret
  212. def _check_refname(self, name) -> None:
  213. """Ensure a refname is valid and lives in refs or is HEAD.
  214. HEAD is not a valid refname according to git-check-ref-format, but this
  215. class needs to be able to touch HEAD. Also, check_ref_format expects
  216. refnames without the leading 'refs/', but this class requires that
  217. so it cannot touch anything outside the refs dir (or HEAD).
  218. Args:
  219. name: The name of the reference.
  220. Raises:
  221. KeyError: if a refname is not HEAD or is otherwise not valid.
  222. """
  223. if name in (HEADREF, b"refs/stash"):
  224. return
  225. if not name.startswith(b"refs/") or not check_ref_format(name[5:]):
  226. raise RefFormatError(name)
  227. def read_ref(self, refname):
  228. """Read a reference without following any references.
  229. Args:
  230. refname: The name of the reference
  231. Returns: The contents of the ref file, or None if it does
  232. not exist.
  233. """
  234. contents = self.read_loose_ref(refname)
  235. if not contents:
  236. contents = self.get_packed_refs().get(refname, None)
  237. return contents
  238. def read_loose_ref(self, name) -> bytes:
  239. """Read a loose reference and return its contents.
  240. Args:
  241. name: the refname to read
  242. Returns: The contents of the ref file, or None if it does
  243. not exist.
  244. """
  245. raise NotImplementedError(self.read_loose_ref)
  246. def follow(self, name) -> tuple[list[bytes], bytes]:
  247. """Follow a reference name.
  248. Returns: a tuple of (refnames, sha), wheres refnames are the names of
  249. references in the chain
  250. """
  251. contents = SYMREF + name
  252. depth = 0
  253. refnames = []
  254. while contents.startswith(SYMREF):
  255. refname = contents[len(SYMREF) :]
  256. refnames.append(refname)
  257. contents = self.read_ref(refname)
  258. if not contents:
  259. break
  260. depth += 1
  261. if depth > 5:
  262. raise SymrefLoop(name, depth)
  263. return refnames, contents
  264. def __contains__(self, refname) -> bool:
  265. if self.read_ref(refname):
  266. return True
  267. return False
  268. def __getitem__(self, name) -> ObjectID:
  269. """Get the SHA1 for a reference name.
  270. This method follows all symbolic references.
  271. """
  272. _, sha = self.follow(name)
  273. if sha is None:
  274. raise KeyError(name)
  275. return sha
  276. def set_if_equals(
  277. self,
  278. name,
  279. old_ref,
  280. new_ref,
  281. committer=None,
  282. timestamp=None,
  283. timezone=None,
  284. message=None,
  285. ) -> bool:
  286. """Set a refname to new_ref only if it currently equals old_ref.
  287. This method follows all symbolic references if applicable for the
  288. subclass, and can be used to perform an atomic compare-and-swap
  289. operation.
  290. Args:
  291. name: The refname to set.
  292. old_ref: The old sha the refname must refer to, or None to set
  293. unconditionally.
  294. new_ref: The new sha the refname will refer to.
  295. message: Message for reflog
  296. Returns: True if the set was successful, False otherwise.
  297. """
  298. raise NotImplementedError(self.set_if_equals)
  299. def add_if_new(
  300. self, name, ref, committer=None, timestamp=None, timezone=None, message=None
  301. ) -> bool:
  302. """Add a new reference only if it does not already exist.
  303. Args:
  304. name: Ref name
  305. ref: Ref value
  306. """
  307. raise NotImplementedError(self.add_if_new)
  308. def __setitem__(self, name, ref) -> None:
  309. """Set a reference name to point to the given SHA1.
  310. This method follows all symbolic references if applicable for the
  311. subclass.
  312. Note: This method unconditionally overwrites the contents of a
  313. reference. To update atomically only if the reference has not
  314. changed, use set_if_equals().
  315. Args:
  316. name: The refname to set.
  317. ref: The new sha the refname will refer to.
  318. """
  319. if not (valid_hexsha(ref) or ref.startswith(SYMREF)):
  320. raise ValueError(f"{ref!r} must be a valid sha (40 chars) or a symref")
  321. self.set_if_equals(name, None, ref)
  322. def remove_if_equals(
  323. self,
  324. name,
  325. old_ref,
  326. committer=None,
  327. timestamp=None,
  328. timezone=None,
  329. message=None,
  330. ) -> bool:
  331. """Remove a refname only if it currently equals old_ref.
  332. This method does not follow symbolic references, even if applicable for
  333. the subclass. It can be used to perform an atomic compare-and-delete
  334. operation.
  335. Args:
  336. name: The refname to delete.
  337. old_ref: The old sha the refname must refer to, or None to
  338. delete unconditionally.
  339. message: Message for reflog
  340. Returns: True if the delete was successful, False otherwise.
  341. """
  342. raise NotImplementedError(self.remove_if_equals)
  343. def __delitem__(self, name) -> None:
  344. """Remove a refname.
  345. This method does not follow symbolic references, even if applicable for
  346. the subclass.
  347. Note: This method unconditionally deletes the contents of a reference.
  348. To delete atomically only if the reference has not changed, use
  349. remove_if_equals().
  350. Args:
  351. name: The refname to delete.
  352. """
  353. self.remove_if_equals(name, None)
  354. def get_symrefs(self):
  355. """Get a dict with all symrefs in this container.
  356. Returns: Dictionary mapping source ref to target ref
  357. """
  358. ret = {}
  359. for src in self.allkeys():
  360. try:
  361. dst = parse_symref_value(self.read_ref(src))
  362. except ValueError:
  363. pass
  364. else:
  365. ret[src] = dst
  366. return ret
  367. class DictRefsContainer(RefsContainer):
  368. """RefsContainer backed by a simple dict.
  369. This container does not support symbolic or packed references and is not
  370. threadsafe.
  371. """
  372. def __init__(self, refs, logger=None) -> None:
  373. super().__init__(logger=logger)
  374. self._refs = refs
  375. self._peeled: dict[bytes, ObjectID] = {}
  376. self._watchers: set[Any] = set()
  377. def allkeys(self):
  378. return self._refs.keys()
  379. def read_loose_ref(self, name):
  380. return self._refs.get(name, None)
  381. def get_packed_refs(self):
  382. return {}
  383. def _notify(self, ref, newsha) -> None:
  384. for watcher in self._watchers:
  385. watcher._notify((ref, newsha))
  386. def set_symbolic_ref(
  387. self,
  388. name: Ref,
  389. other: Ref,
  390. committer=None,
  391. timestamp=None,
  392. timezone=None,
  393. message=None,
  394. ) -> None:
  395. old = self.follow(name)[-1]
  396. new = SYMREF + other
  397. self._refs[name] = new
  398. self._notify(name, new)
  399. self._log(
  400. name,
  401. old,
  402. new,
  403. committer=committer,
  404. timestamp=timestamp,
  405. timezone=timezone,
  406. message=message,
  407. )
  408. def set_if_equals(
  409. self,
  410. name,
  411. old_ref,
  412. new_ref,
  413. committer=None,
  414. timestamp=None,
  415. timezone=None,
  416. message=None,
  417. ) -> bool:
  418. if old_ref is not None and self._refs.get(name, ZERO_SHA) != old_ref:
  419. return False
  420. realnames, _ = self.follow(name)
  421. for realname in realnames:
  422. self._check_refname(realname)
  423. old = self._refs.get(realname)
  424. self._refs[realname] = new_ref
  425. self._notify(realname, new_ref)
  426. self._log(
  427. realname,
  428. old,
  429. new_ref,
  430. committer=committer,
  431. timestamp=timestamp,
  432. timezone=timezone,
  433. message=message,
  434. )
  435. return True
  436. def add_if_new(
  437. self,
  438. name: Ref,
  439. ref: ObjectID,
  440. committer=None,
  441. timestamp=None,
  442. timezone=None,
  443. message: Optional[bytes] = None,
  444. ) -> bool:
  445. if name in self._refs:
  446. return False
  447. self._refs[name] = ref
  448. self._notify(name, ref)
  449. self._log(
  450. name,
  451. None,
  452. ref,
  453. committer=committer,
  454. timestamp=timestamp,
  455. timezone=timezone,
  456. message=message,
  457. )
  458. return True
  459. def remove_if_equals(
  460. self,
  461. name,
  462. old_ref,
  463. committer=None,
  464. timestamp=None,
  465. timezone=None,
  466. message=None,
  467. ) -> bool:
  468. if old_ref is not None and self._refs.get(name, ZERO_SHA) != old_ref:
  469. return False
  470. try:
  471. old = self._refs.pop(name)
  472. except KeyError:
  473. pass
  474. else:
  475. self._notify(name, None)
  476. self._log(
  477. name,
  478. old,
  479. None,
  480. committer=committer,
  481. timestamp=timestamp,
  482. timezone=timezone,
  483. message=message,
  484. )
  485. return True
  486. def get_peeled(self, name):
  487. return self._peeled.get(name)
  488. def _update(self, refs) -> None:
  489. """Update multiple refs; intended only for testing."""
  490. # TODO(dborowitz): replace this with a public function that uses
  491. # set_if_equal.
  492. for ref, sha in refs.items():
  493. self.set_if_equals(ref, None, sha)
  494. def _update_peeled(self, peeled) -> None:
  495. """Update cached peeled refs; intended only for testing."""
  496. self._peeled.update(peeled)
  497. class InfoRefsContainer(RefsContainer):
  498. """Refs container that reads refs from a info/refs file."""
  499. def __init__(self, f) -> None:
  500. self._refs = {}
  501. self._peeled = {}
  502. refs = read_info_refs(f)
  503. (self._refs, self._peeled) = split_peeled_refs(refs)
  504. def allkeys(self):
  505. return self._refs.keys()
  506. def read_loose_ref(self, name):
  507. return self._refs.get(name, None)
  508. def get_packed_refs(self):
  509. return {}
  510. def get_peeled(self, name):
  511. try:
  512. return self._peeled[name]
  513. except KeyError:
  514. return self._refs[name]
  515. class DiskRefsContainer(RefsContainer):
  516. """Refs container that reads refs from disk."""
  517. def __init__(self, path, worktree_path=None, logger=None) -> None:
  518. super().__init__(logger=logger)
  519. if getattr(path, "encode", None) is not None:
  520. path = os.fsencode(path)
  521. self.path = path
  522. if worktree_path is None:
  523. worktree_path = path
  524. if getattr(worktree_path, "encode", None) is not None:
  525. worktree_path = os.fsencode(worktree_path)
  526. self.worktree_path = worktree_path
  527. self._packed_refs = None
  528. self._peeled_refs = None
  529. def __repr__(self) -> str:
  530. return f"{self.__class__.__name__}({self.path!r})"
  531. def subkeys(self, base):
  532. subkeys = set()
  533. path = self.refpath(base)
  534. for root, unused_dirs, files in os.walk(path):
  535. dir = root[len(path) :]
  536. if os.path.sep != "/":
  537. dir = dir.replace(os.fsencode(os.path.sep), b"/")
  538. dir = dir.strip(b"/")
  539. for filename in files:
  540. refname = b"/".join(([dir] if dir else []) + [filename])
  541. # check_ref_format requires at least one /, so we prepend the
  542. # base before calling it.
  543. if check_ref_format(base + b"/" + refname):
  544. subkeys.add(refname)
  545. for key in self.get_packed_refs():
  546. if key.startswith(base):
  547. subkeys.add(key[len(base) :].strip(b"/"))
  548. return subkeys
  549. def allkeys(self):
  550. allkeys = set()
  551. if os.path.exists(self.refpath(HEADREF)):
  552. allkeys.add(HEADREF)
  553. path = self.refpath(b"")
  554. refspath = self.refpath(b"refs")
  555. for root, unused_dirs, files in os.walk(refspath):
  556. dir = root[len(path) :]
  557. if os.path.sep != "/":
  558. dir = dir.replace(os.fsencode(os.path.sep), b"/")
  559. for filename in files:
  560. refname = b"/".join([dir, filename])
  561. if check_ref_format(refname):
  562. allkeys.add(refname)
  563. allkeys.update(self.get_packed_refs())
  564. return allkeys
  565. def refpath(self, name):
  566. """Return the disk path of a ref."""
  567. if os.path.sep != "/":
  568. name = name.replace(b"/", os.fsencode(os.path.sep))
  569. # TODO: as the 'HEAD' reference is working tree specific, it
  570. # should actually not be a part of RefsContainer
  571. if name == HEADREF:
  572. return os.path.join(self.worktree_path, name)
  573. else:
  574. return os.path.join(self.path, name)
  575. def get_packed_refs(self):
  576. """Get contents of the packed-refs file.
  577. Returns: Dictionary mapping ref names to SHA1s
  578. Note: Will return an empty dictionary when no packed-refs file is
  579. present.
  580. """
  581. # TODO: invalidate the cache on repacking
  582. if self._packed_refs is None:
  583. # set both to empty because we want _peeled_refs to be
  584. # None if and only if _packed_refs is also None.
  585. self._packed_refs = {}
  586. self._peeled_refs = {}
  587. path = os.path.join(self.path, b"packed-refs")
  588. try:
  589. f = GitFile(path, "rb")
  590. except FileNotFoundError:
  591. return {}
  592. with f:
  593. first_line = next(iter(f)).rstrip()
  594. if first_line.startswith(b"# pack-refs") and b" peeled" in first_line:
  595. for sha, name, peeled in read_packed_refs_with_peeled(f):
  596. self._packed_refs[name] = sha
  597. if peeled:
  598. self._peeled_refs[name] = peeled
  599. else:
  600. f.seek(0)
  601. for sha, name in read_packed_refs(f):
  602. self._packed_refs[name] = sha
  603. return self._packed_refs
  604. def add_packed_refs(self, new_refs: dict[Ref, Optional[ObjectID]]) -> None:
  605. """Add the given refs as packed refs.
  606. Args:
  607. new_refs: A mapping of ref names to targets; if a target is None that
  608. means remove the ref
  609. """
  610. if not new_refs:
  611. return
  612. path = os.path.join(self.path, b"packed-refs")
  613. with GitFile(path, "wb") as f:
  614. # reread cached refs from disk, while holding the lock
  615. packed_refs = self.get_packed_refs().copy()
  616. for ref, target in new_refs.items():
  617. # sanity check
  618. if ref == HEADREF:
  619. raise ValueError("cannot pack HEAD")
  620. # remove any loose refs pointing to this one -- please
  621. # note that this bypasses remove_if_equals as we don't
  622. # want to affect packed refs in here
  623. with suppress(OSError):
  624. os.remove(self.refpath(ref))
  625. if target is not None:
  626. packed_refs[ref] = target
  627. else:
  628. packed_refs.pop(ref, None)
  629. write_packed_refs(f, packed_refs, self._peeled_refs)
  630. self._packed_refs = packed_refs
  631. def get_peeled(self, name):
  632. """Return the cached peeled value of a ref, if available.
  633. Args:
  634. name: Name of the ref to peel
  635. Returns: The peeled value of the ref. If the ref is known not point to
  636. a tag, this will be the SHA the ref refers to. If the ref may point
  637. to a tag, but no cached information is available, None is returned.
  638. """
  639. self.get_packed_refs()
  640. if self._peeled_refs is None or name not in self._packed_refs:
  641. # No cache: no peeled refs were read, or this ref is loose
  642. return None
  643. if name in self._peeled_refs:
  644. return self._peeled_refs[name]
  645. else:
  646. # Known not peelable
  647. return self[name]
  648. def read_loose_ref(self, name):
  649. """Read a reference file and return its contents.
  650. If the reference file a symbolic reference, only read the first line of
  651. the file. Otherwise, only read the first 40 bytes.
  652. Args:
  653. name: the refname to read, relative to refpath
  654. Returns: The contents of the ref file, or None if the file does not
  655. exist.
  656. Raises:
  657. IOError: if any other error occurs
  658. """
  659. filename = self.refpath(name)
  660. try:
  661. with GitFile(filename, "rb") as f:
  662. header = f.read(len(SYMREF))
  663. if header == SYMREF:
  664. # Read only the first line
  665. return header + next(iter(f)).rstrip(b"\r\n")
  666. else:
  667. # Read only the first 40 bytes
  668. return header + f.read(40 - len(SYMREF))
  669. except (OSError, UnicodeError):
  670. # don't assume anything specific about the error; in
  671. # particular, invalid or forbidden paths can raise weird
  672. # errors depending on the specific operating system
  673. return None
  674. def _remove_packed_ref(self, name) -> None:
  675. if self._packed_refs is None:
  676. return
  677. filename = os.path.join(self.path, b"packed-refs")
  678. # reread cached refs from disk, while holding the lock
  679. f = GitFile(filename, "wb")
  680. try:
  681. self._packed_refs = None
  682. self.get_packed_refs()
  683. if name not in self._packed_refs:
  684. return
  685. del self._packed_refs[name]
  686. with suppress(KeyError):
  687. del self._peeled_refs[name]
  688. write_packed_refs(f, self._packed_refs, self._peeled_refs)
  689. f.close()
  690. finally:
  691. f.abort()
  692. def set_symbolic_ref(
  693. self,
  694. name,
  695. other,
  696. committer=None,
  697. timestamp=None,
  698. timezone=None,
  699. message=None,
  700. ) -> None:
  701. """Make a ref point at another ref.
  702. Args:
  703. name: Name of the ref to set
  704. other: Name of the ref to point at
  705. message: Optional message to describe the change
  706. """
  707. self._check_refname(name)
  708. self._check_refname(other)
  709. filename = self.refpath(name)
  710. f = GitFile(filename, "wb")
  711. try:
  712. f.write(SYMREF + other + b"\n")
  713. sha = self.follow(name)[-1]
  714. self._log(
  715. name,
  716. sha,
  717. sha,
  718. committer=committer,
  719. timestamp=timestamp,
  720. timezone=timezone,
  721. message=message,
  722. )
  723. except BaseException:
  724. f.abort()
  725. raise
  726. else:
  727. f.close()
  728. def set_if_equals(
  729. self,
  730. name,
  731. old_ref,
  732. new_ref,
  733. committer=None,
  734. timestamp=None,
  735. timezone=None,
  736. message=None,
  737. ) -> bool:
  738. """Set a refname to new_ref only if it currently equals old_ref.
  739. This method follows all symbolic references, and can be used to perform
  740. an atomic compare-and-swap operation.
  741. Args:
  742. name: The refname to set.
  743. old_ref: The old sha the refname must refer to, or None to set
  744. unconditionally.
  745. new_ref: The new sha the refname will refer to.
  746. message: Set message for reflog
  747. Returns: True if the set was successful, False otherwise.
  748. """
  749. self._check_refname(name)
  750. try:
  751. realnames, _ = self.follow(name)
  752. realname = realnames[-1]
  753. except (KeyError, IndexError, SymrefLoop):
  754. realname = name
  755. filename = self.refpath(realname)
  756. # make sure none of the ancestor folders is in packed refs
  757. probe_ref = os.path.dirname(realname)
  758. packed_refs = self.get_packed_refs()
  759. while probe_ref:
  760. if packed_refs.get(probe_ref, None) is not None:
  761. raise NotADirectoryError(filename)
  762. probe_ref = os.path.dirname(probe_ref)
  763. ensure_dir_exists(os.path.dirname(filename))
  764. with GitFile(filename, "wb") as f:
  765. if old_ref is not None:
  766. try:
  767. # read again while holding the lock
  768. orig_ref = self.read_loose_ref(realname)
  769. if orig_ref is None:
  770. orig_ref = self.get_packed_refs().get(realname, ZERO_SHA)
  771. if orig_ref != old_ref:
  772. f.abort()
  773. return False
  774. except OSError:
  775. f.abort()
  776. raise
  777. try:
  778. f.write(new_ref + b"\n")
  779. except OSError:
  780. f.abort()
  781. raise
  782. self._log(
  783. realname,
  784. old_ref,
  785. new_ref,
  786. committer=committer,
  787. timestamp=timestamp,
  788. timezone=timezone,
  789. message=message,
  790. )
  791. return True
  792. def add_if_new(
  793. self,
  794. name: bytes,
  795. ref: bytes,
  796. committer=None,
  797. timestamp=None,
  798. timezone=None,
  799. message: Optional[bytes] = None,
  800. ) -> bool:
  801. """Add a new reference only if it does not already exist.
  802. This method follows symrefs, and only ensures that the last ref in the
  803. chain does not exist.
  804. Args:
  805. name: The refname to set.
  806. ref: The new sha the refname will refer to.
  807. message: Optional message for reflog
  808. Returns: True if the add was successful, False otherwise.
  809. """
  810. try:
  811. realnames, contents = self.follow(name)
  812. if contents is not None:
  813. return False
  814. realname = realnames[-1]
  815. except (KeyError, IndexError):
  816. realname = name
  817. self._check_refname(realname)
  818. filename = self.refpath(realname)
  819. ensure_dir_exists(os.path.dirname(filename))
  820. with GitFile(filename, "wb") as f:
  821. if os.path.exists(filename) or name in self.get_packed_refs():
  822. f.abort()
  823. return False
  824. try:
  825. f.write(ref + b"\n")
  826. except OSError:
  827. f.abort()
  828. raise
  829. else:
  830. self._log(
  831. name,
  832. None,
  833. ref,
  834. committer=committer,
  835. timestamp=timestamp,
  836. timezone=timezone,
  837. message=message,
  838. )
  839. return True
  840. def remove_if_equals(
  841. self,
  842. name,
  843. old_ref,
  844. committer=None,
  845. timestamp=None,
  846. timezone=None,
  847. message=None,
  848. ) -> bool:
  849. """Remove a refname only if it currently equals old_ref.
  850. This method does not follow symbolic references. It can be used to
  851. perform an atomic compare-and-delete operation.
  852. Args:
  853. name: The refname to delete.
  854. old_ref: The old sha the refname must refer to, or None to
  855. delete unconditionally.
  856. message: Optional message
  857. Returns: True if the delete was successful, False otherwise.
  858. """
  859. self._check_refname(name)
  860. filename = self.refpath(name)
  861. ensure_dir_exists(os.path.dirname(filename))
  862. f = GitFile(filename, "wb")
  863. try:
  864. if old_ref is not None:
  865. orig_ref = self.read_loose_ref(name)
  866. if orig_ref is None:
  867. orig_ref = self.get_packed_refs().get(name, ZERO_SHA)
  868. if orig_ref != old_ref:
  869. return False
  870. # remove the reference file itself
  871. try:
  872. found = os.path.lexists(filename)
  873. except OSError:
  874. # may only be packed, or otherwise unstorable
  875. found = False
  876. if found:
  877. os.remove(filename)
  878. self._remove_packed_ref(name)
  879. self._log(
  880. name,
  881. old_ref,
  882. None,
  883. committer=committer,
  884. timestamp=timestamp,
  885. timezone=timezone,
  886. message=message,
  887. )
  888. finally:
  889. # never write, we just wanted the lock
  890. f.abort()
  891. # outside of the lock, clean-up any parent directory that might now
  892. # be empty. this ensures that re-creating a reference of the same
  893. # name of what was previously a directory works as expected
  894. parent = name
  895. while True:
  896. try:
  897. parent, _ = parent.rsplit(b"/", 1)
  898. except ValueError:
  899. break
  900. if parent == b"refs":
  901. break
  902. parent_filename = self.refpath(parent)
  903. try:
  904. os.rmdir(parent_filename)
  905. except OSError:
  906. # this can be caused by the parent directory being
  907. # removed by another process, being not empty, etc.
  908. # in any case, this is non fatal because we already
  909. # removed the reference, just ignore it
  910. break
  911. return True
  912. def _split_ref_line(line):
  913. """Split a single ref line into a tuple of SHA1 and name."""
  914. fields = line.rstrip(b"\n\r").split(b" ")
  915. if len(fields) != 2:
  916. raise PackedRefsException(f"invalid ref line {line!r}")
  917. sha, name = fields
  918. if not valid_hexsha(sha):
  919. raise PackedRefsException(f"Invalid hex sha {sha!r}")
  920. if not check_ref_format(name):
  921. raise PackedRefsException(f"invalid ref name {name!r}")
  922. return (sha, name)
  923. def read_packed_refs(f):
  924. """Read a packed refs file.
  925. Args:
  926. f: file-like object to read from
  927. Returns: Iterator over tuples with SHA1s and ref names.
  928. """
  929. for line in f:
  930. if line.startswith(b"#"):
  931. # Comment
  932. continue
  933. if line.startswith(b"^"):
  934. raise PackedRefsException("found peeled ref in packed-refs without peeled")
  935. yield _split_ref_line(line)
  936. def read_packed_refs_with_peeled(f):
  937. """Read a packed refs file including peeled refs.
  938. Assumes the "# pack-refs with: peeled" line was already read. Yields tuples
  939. with ref names, SHA1s, and peeled SHA1s (or None).
  940. Args:
  941. f: file-like object to read from, seek'ed to the second line
  942. """
  943. last = None
  944. for line in f:
  945. if line[0] == b"#":
  946. continue
  947. line = line.rstrip(b"\r\n")
  948. if line.startswith(b"^"):
  949. if not last:
  950. raise PackedRefsException("unexpected peeled ref line")
  951. if not valid_hexsha(line[1:]):
  952. raise PackedRefsException(f"Invalid hex sha {line[1:]!r}")
  953. sha, name = _split_ref_line(last)
  954. last = None
  955. yield (sha, name, line[1:])
  956. else:
  957. if last:
  958. sha, name = _split_ref_line(last)
  959. yield (sha, name, None)
  960. last = line
  961. if last:
  962. sha, name = _split_ref_line(last)
  963. yield (sha, name, None)
  964. def write_packed_refs(f, packed_refs, peeled_refs=None) -> None:
  965. """Write a packed refs file.
  966. Args:
  967. f: empty file-like object to write to
  968. packed_refs: dict of refname to sha of packed refs to write
  969. peeled_refs: dict of refname to peeled value of sha
  970. """
  971. if peeled_refs is None:
  972. peeled_refs = {}
  973. else:
  974. f.write(b"# pack-refs with: peeled\n")
  975. for refname in sorted(packed_refs.keys()):
  976. f.write(git_line(packed_refs[refname], refname))
  977. if refname in peeled_refs:
  978. f.write(b"^" + peeled_refs[refname] + b"\n")
  979. def read_info_refs(f):
  980. ret = {}
  981. for line in f.readlines():
  982. (sha, name) = line.rstrip(b"\r\n").split(b"\t", 1)
  983. ret[name] = sha
  984. return ret
  985. def write_info_refs(refs, store: ObjectContainer):
  986. """Generate info refs."""
  987. # TODO: Avoid recursive import :(
  988. from .object_store import peel_sha
  989. for name, sha in sorted(refs.items()):
  990. # get_refs() includes HEAD as a special case, but we don't want to
  991. # advertise it
  992. if name == HEADREF:
  993. continue
  994. try:
  995. o = store[sha]
  996. except KeyError:
  997. continue
  998. unpeeled, peeled = peel_sha(store, sha)
  999. yield o.id + b"\t" + name + b"\n"
  1000. if o.id != peeled.id:
  1001. yield peeled.id + b"\t" + name + PEELED_TAG_SUFFIX + b"\n"
  1002. def is_local_branch(x):
  1003. return x.startswith(LOCAL_BRANCH_PREFIX)
  1004. def strip_peeled_refs(refs):
  1005. """Remove all peeled refs."""
  1006. return {
  1007. ref: sha for (ref, sha) in refs.items() if not ref.endswith(PEELED_TAG_SUFFIX)
  1008. }
  1009. def split_peeled_refs(refs):
  1010. """Split peeled refs from regular refs."""
  1011. peeled = {}
  1012. regular = {}
  1013. for ref, sha in refs.items():
  1014. if ref.endswith(PEELED_TAG_SUFFIX):
  1015. peeled[ref[: -len(PEELED_TAG_SUFFIX)]] = sha
  1016. else:
  1017. regular[ref] = sha
  1018. return regular, peeled
  1019. def _set_origin_head(refs, origin, origin_head) -> None:
  1020. # set refs/remotes/origin/HEAD
  1021. origin_base = b"refs/remotes/" + origin + b"/"
  1022. if origin_head and origin_head.startswith(LOCAL_BRANCH_PREFIX):
  1023. origin_ref = origin_base + HEADREF
  1024. target_ref = origin_base + origin_head[len(LOCAL_BRANCH_PREFIX) :]
  1025. if target_ref in refs:
  1026. refs.set_symbolic_ref(origin_ref, target_ref)
  1027. def _set_default_branch(
  1028. refs: RefsContainer,
  1029. origin: bytes,
  1030. origin_head: Optional[bytes],
  1031. branch: bytes,
  1032. ref_message: Optional[bytes],
  1033. ) -> bytes:
  1034. """Set the default branch."""
  1035. origin_base = b"refs/remotes/" + origin + b"/"
  1036. if branch:
  1037. origin_ref = origin_base + branch
  1038. if origin_ref in refs:
  1039. local_ref = LOCAL_BRANCH_PREFIX + branch
  1040. refs.add_if_new(local_ref, refs[origin_ref], ref_message)
  1041. head_ref = local_ref
  1042. elif LOCAL_TAG_PREFIX + branch in refs:
  1043. head_ref = LOCAL_TAG_PREFIX + branch
  1044. else:
  1045. raise ValueError(f"{os.fsencode(branch)!r} is not a valid branch or tag")
  1046. elif origin_head:
  1047. head_ref = origin_head
  1048. if origin_head.startswith(LOCAL_BRANCH_PREFIX):
  1049. origin_ref = origin_base + origin_head[len(LOCAL_BRANCH_PREFIX) :]
  1050. else:
  1051. origin_ref = origin_head
  1052. try:
  1053. refs.add_if_new(head_ref, refs[origin_ref], ref_message)
  1054. except KeyError:
  1055. pass
  1056. else:
  1057. raise ValueError("neither origin_head nor branch are provided")
  1058. return head_ref
  1059. def _set_head(refs, head_ref, ref_message):
  1060. if head_ref.startswith(LOCAL_TAG_PREFIX):
  1061. # detach HEAD at specified tag
  1062. head = refs[head_ref]
  1063. if isinstance(head, Tag):
  1064. _cls, obj = head.object
  1065. head = obj.get_object(obj).id
  1066. del refs[HEADREF]
  1067. refs.set_if_equals(HEADREF, None, head, message=ref_message)
  1068. else:
  1069. # set HEAD to specific branch
  1070. try:
  1071. head = refs[head_ref]
  1072. refs.set_symbolic_ref(HEADREF, head_ref)
  1073. refs.set_if_equals(HEADREF, None, head, message=ref_message)
  1074. except KeyError:
  1075. head = None
  1076. return head
  1077. def _import_remote_refs(
  1078. refs_container: RefsContainer,
  1079. remote_name: str,
  1080. refs: dict[str, str],
  1081. message: Optional[bytes] = None,
  1082. prune: bool = False,
  1083. prune_tags: bool = False,
  1084. ) -> None:
  1085. stripped_refs = strip_peeled_refs(refs)
  1086. branches = {
  1087. n[len(LOCAL_BRANCH_PREFIX) :]: v
  1088. for (n, v) in stripped_refs.items()
  1089. if n.startswith(LOCAL_BRANCH_PREFIX)
  1090. }
  1091. refs_container.import_refs(
  1092. b"refs/remotes/" + remote_name.encode(),
  1093. branches,
  1094. message=message,
  1095. prune=prune,
  1096. )
  1097. tags = {
  1098. n[len(LOCAL_TAG_PREFIX) :]: v
  1099. for (n, v) in stripped_refs.items()
  1100. if n.startswith(LOCAL_TAG_PREFIX) and not n.endswith(PEELED_TAG_SUFFIX)
  1101. }
  1102. refs_container.import_refs(
  1103. LOCAL_TAG_PREFIX, tags, message=message, prune=prune_tags
  1104. )
  1105. def serialize_refs(store, refs):
  1106. # TODO: Avoid recursive import :(
  1107. from .object_store import peel_sha
  1108. ret = {}
  1109. for ref, sha in refs.items():
  1110. try:
  1111. unpeeled, peeled = peel_sha(store, sha)
  1112. except KeyError:
  1113. warnings.warn(
  1114. "ref {} points at non-present sha {}".format(
  1115. ref.decode("utf-8", "replace"), sha.decode("ascii")
  1116. ),
  1117. UserWarning,
  1118. )
  1119. continue
  1120. else:
  1121. if isinstance(unpeeled, Tag):
  1122. ret[ref + PEELED_TAG_SUFFIX] = peeled.id
  1123. ret[ref] = unpeeled.id
  1124. return ret