objects.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644
  1. # objects.py -- Access to base git objects
  2. # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
  3. # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
  4. #
  5. # This program is free software; you can redistribute it and/or
  6. # modify it under the terms of the GNU General Public License
  7. # as published by the Free Software Foundation; version 2
  8. # of the License or (at your option) a later version of the License.
  9. #
  10. # This program is distributed in the hope that it will be useful,
  11. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. # GNU General Public License for more details.
  14. #
  15. # You should have received a copy of the GNU General Public License
  16. # along with this program; if not, write to the Free Software
  17. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  18. # MA 02110-1301, USA.
  19. """Access to base git objects."""
  20. import binascii
  21. from cStringIO import (
  22. StringIO,
  23. )
  24. import mmap
  25. import os
  26. import stat
  27. import time
  28. import zlib
  29. from dulwich.errors import (
  30. NotBlobError,
  31. NotCommitError,
  32. NotTreeError,
  33. )
  34. from dulwich.file import GitFile
  35. from dulwich.misc import (
  36. make_sha,
  37. )
  38. BLOB_ID = "blob"
  39. TAG_ID = "tag"
  40. TREE_ID = "tree"
  41. COMMIT_ID = "commit"
  42. PARENT_ID = "parent"
  43. AUTHOR_ID = "author"
  44. COMMITTER_ID = "committer"
  45. OBJECT_ID = "object"
  46. TYPE_ID = "type"
  47. TAGGER_ID = "tagger"
  48. ENCODING_ID = "encoding"
  49. S_IFGITLINK = 0160000
  50. def S_ISGITLINK(m):
  51. return (stat.S_IFMT(m) == S_IFGITLINK)
  52. def _decompress(string):
  53. dcomp = zlib.decompressobj()
  54. dcomped = dcomp.decompress(string)
  55. dcomped += dcomp.flush()
  56. return dcomped
  57. def sha_to_hex(sha):
  58. """Takes a string and returns the hex of the sha within"""
  59. hexsha = binascii.hexlify(sha)
  60. assert len(hexsha) == 40, "Incorrect length of sha1 string: %d" % hexsha
  61. return hexsha
  62. def hex_to_sha(hex):
  63. """Takes a hex sha and returns a binary sha"""
  64. assert len(hex) == 40, "Incorrent length of hexsha: %s" % hex
  65. return binascii.unhexlify(hex)
  66. def serializable_property(name, docstring=None):
  67. def set(obj, value):
  68. obj._ensure_parsed()
  69. setattr(obj, "_"+name, value)
  70. obj._needs_serialization = True
  71. def get(obj):
  72. obj._ensure_parsed()
  73. return getattr(obj, "_"+name)
  74. return property(get, set, doc=docstring)
  75. class ShaFile(object):
  76. """A git SHA file."""
  77. @classmethod
  78. def _parse_legacy_object(cls, map):
  79. """Parse a legacy object, creating it and setting object._text"""
  80. text = _decompress(map)
  81. object = None
  82. for posstype in type_map.keys():
  83. if text.startswith(posstype):
  84. object = type_map[posstype]()
  85. text = text[len(posstype):]
  86. break
  87. assert object is not None, "%s is not a known object type" % text[:9]
  88. assert text[0] == ' ', "%s is not a space" % text[0]
  89. text = text[1:]
  90. size = 0
  91. i = 0
  92. while text[0] >= '0' and text[0] <= '9':
  93. if i > 0 and size == 0:
  94. raise AssertionError("Size is not in canonical format")
  95. size = (size * 10) + int(text[0])
  96. text = text[1:]
  97. i += 1
  98. object._size = size
  99. assert text[0] == "\0", "Size not followed by null"
  100. text = text[1:]
  101. object.set_raw_string(text)
  102. return object
  103. def as_legacy_object(self):
  104. text = self.as_raw_string()
  105. return zlib.compress("%s %d\0%s" % (self._type, len(text), text))
  106. def as_raw_string(self):
  107. if self._needs_serialization:
  108. self.serialize()
  109. return self._text
  110. def __str__(self):
  111. return self.as_raw_string()
  112. def __hash__(self):
  113. return hash(self.id)
  114. def as_pretty_string(self):
  115. return self.as_raw_string()
  116. def _ensure_parsed(self):
  117. if self._needs_parsing:
  118. self._parse_text()
  119. def set_raw_string(self, text):
  120. if type(text) != str:
  121. raise TypeError(text)
  122. self._text = text
  123. self._sha = None
  124. self._needs_parsing = True
  125. self._needs_serialization = False
  126. @classmethod
  127. def _parse_object(cls, map):
  128. """Parse a new style object , creating it and setting object._text"""
  129. used = 0
  130. byte = ord(map[used])
  131. used += 1
  132. num_type = (byte >> 4) & 7
  133. try:
  134. object = num_type_map[num_type]()
  135. except KeyError:
  136. raise AssertionError("Not a known type: %d" % num_type)
  137. while (byte & 0x80) != 0:
  138. byte = ord(map[used])
  139. used += 1
  140. raw = map[used:]
  141. object.set_raw_string(_decompress(raw))
  142. return object
  143. @classmethod
  144. def _parse_file(cls, map):
  145. word = (ord(map[0]) << 8) + ord(map[1])
  146. if ord(map[0]) == 0x78 and (word % 31) == 0:
  147. return cls._parse_legacy_object(map)
  148. else:
  149. return cls._parse_object(map)
  150. def __init__(self):
  151. """Don't call this directly"""
  152. self._sha = None
  153. def _parse_text(self):
  154. """For subclasses to do initialisation time parsing"""
  155. @classmethod
  156. def from_file(cls, filename):
  157. """Get the contents of a SHA file on disk"""
  158. size = os.path.getsize(filename)
  159. f = GitFile(filename, 'rb')
  160. try:
  161. map = mmap.mmap(f.fileno(), size, access=mmap.ACCESS_READ)
  162. shafile = cls._parse_file(map)
  163. return shafile
  164. finally:
  165. f.close()
  166. @classmethod
  167. def from_raw_string(cls, type, string):
  168. """Creates an object of the indicated type from the raw string given.
  169. Type is the numeric type of an object. String is the raw uncompressed
  170. contents.
  171. """
  172. real_class = num_type_map[type]
  173. obj = real_class()
  174. obj.type = type
  175. obj.set_raw_string(string)
  176. return obj
  177. @classmethod
  178. def from_string(cls, string):
  179. """Create a blob from a string."""
  180. shafile = cls()
  181. shafile.set_raw_string(string)
  182. return shafile
  183. def _raw_length(self):
  184. """Returns the length of the raw string of this object."""
  185. return len(self.as_raw_string())
  186. def _header(self):
  187. return "%s %lu\0" % (self._type, self._raw_length())
  188. def sha(self):
  189. """The SHA1 object that is the name of this object."""
  190. if self._needs_serialization or self._sha is None:
  191. self._sha = make_sha()
  192. self._sha.update(self._header())
  193. self._sha.update(self.as_raw_string())
  194. return self._sha
  195. @property
  196. def id(self):
  197. return self.sha().hexdigest()
  198. def get_type(self):
  199. return self._num_type
  200. def set_type(self, type):
  201. self._num_type = type
  202. type = property(get_type, set_type)
  203. def __repr__(self):
  204. return "<%s %s>" % (self.__class__.__name__, self.id)
  205. def __ne__(self, other):
  206. return self.id != other.id
  207. def __eq__(self, other):
  208. """Return true id the sha of the two objects match.
  209. The __le__ etc methods aren't overriden as they make no sense,
  210. certainly at this level.
  211. """
  212. return self.id == other.id
  213. class Blob(ShaFile):
  214. """A Git Blob object."""
  215. _type = BLOB_ID
  216. _num_type = 3
  217. _needs_serialization = False
  218. _needs_parsing = False
  219. def get_data(self):
  220. return self._text
  221. def set_data(self, data):
  222. self._text = data
  223. data = property(get_data, set_data,
  224. "The text contained within the blob object.")
  225. @classmethod
  226. def from_file(cls, filename):
  227. blob = ShaFile.from_file(filename)
  228. if blob._type != cls._type:
  229. raise NotBlobError(filename)
  230. return blob
  231. class Tag(ShaFile):
  232. """A Git Tag object."""
  233. _type = TAG_ID
  234. _num_type = 4
  235. def __init__(self):
  236. super(Tag, self).__init__()
  237. self._needs_parsing = False
  238. self._needs_serialization = True
  239. @classmethod
  240. def from_file(cls, filename):
  241. blob = ShaFile.from_file(filename)
  242. if blob._type != cls._type:
  243. raise NotBlobError(filename)
  244. return blob
  245. @classmethod
  246. def from_string(cls, string):
  247. """Create a blob from a string."""
  248. shafile = cls()
  249. shafile.set_raw_string(string)
  250. return shafile
  251. def serialize(self):
  252. f = StringIO()
  253. f.write("%s %s\n" % (OBJECT_ID, self._object_sha))
  254. f.write("%s %s\n" % (TYPE_ID, num_type_map[self._object_type]._type))
  255. f.write("%s %s\n" % (TAG_ID, self._name))
  256. if self._tagger:
  257. if self._tag_time is None:
  258. f.write("%s %s\n" % (TAGGER_ID, self._tagger))
  259. else:
  260. f.write("%s %s %d %s\n" % (TAGGER_ID, self._tagger, self._tag_time, format_timezone(self._tag_timezone)))
  261. f.write("\n") # To close headers
  262. f.write(self._message)
  263. self._text = f.getvalue()
  264. self._needs_serialization = False
  265. def _parse_text(self):
  266. """Grab the metadata attached to the tag"""
  267. self._tagger = None
  268. f = StringIO(self._text)
  269. for l in f:
  270. l = l.rstrip("\n")
  271. if l == "":
  272. break # empty line indicates end of headers
  273. (field, value) = l.split(" ", 1)
  274. if field == OBJECT_ID:
  275. self._object_sha = value
  276. elif field == TYPE_ID:
  277. self._object_type = type_map[value]
  278. elif field == TAG_ID:
  279. self._name = value
  280. elif field == TAGGER_ID:
  281. try:
  282. sep = value.index("> ")
  283. except ValueError:
  284. self._tagger = value
  285. self._tag_time = None
  286. self._tag_timezone = None
  287. else:
  288. self._tagger = value[0:sep+1]
  289. (timetext, timezonetext) = value[sep+2:].rsplit(" ", 1)
  290. try:
  291. self._tag_time = int(timetext)
  292. except ValueError: #Not a unix timestamp
  293. self._tag_time = time.strptime(timetext)
  294. self._tag_timezone = parse_timezone(timezonetext)
  295. else:
  296. raise AssertionError("Unknown field %s" % field)
  297. self._message = f.read()
  298. self._needs_parsing = False
  299. def get_object(self):
  300. """Returns the object pointed by this tag, represented as a tuple(type, sha)"""
  301. self._ensure_parsed()
  302. return (self._object_type, self._object_sha)
  303. def set_object(self, value):
  304. self._ensure_parsed()
  305. (self._object_type, self._object_sha) = value
  306. self._needs_serialization = True
  307. object = property(get_object, set_object)
  308. name = serializable_property("name", "The name of this tag")
  309. tagger = serializable_property("tagger",
  310. "Returns the name of the person who created this tag")
  311. tag_time = serializable_property("tag_time",
  312. "The creation timestamp of the tag. As the number of seconds since the epoch")
  313. tag_timezone = serializable_property("tag_timezone",
  314. "The timezone that tag_time is in.")
  315. message = serializable_property("message", "The message attached to this tag")
  316. def parse_tree(text):
  317. ret = {}
  318. count = 0
  319. l = len(text)
  320. while count < l:
  321. mode_end = text.index(' ', count)
  322. mode = int(text[count:mode_end], 8)
  323. name_end = text.index('\0', mode_end)
  324. name = text[mode_end+1:name_end]
  325. count = name_end+21
  326. sha = text[name_end+1:count]
  327. ret[name] = (mode, sha_to_hex(sha))
  328. return ret
  329. class Tree(ShaFile):
  330. """A Git tree object"""
  331. _type = TREE_ID
  332. _num_type = 2
  333. def __init__(self):
  334. super(Tree, self).__init__()
  335. self._entries = {}
  336. self._needs_parsing = False
  337. self._needs_serialization = True
  338. @classmethod
  339. def from_file(cls, filename):
  340. tree = ShaFile.from_file(filename)
  341. if tree._type != cls._type:
  342. raise NotTreeError(filename)
  343. return tree
  344. def __contains__(self, name):
  345. self._ensure_parsed()
  346. return name in self._entries
  347. def __getitem__(self, name):
  348. self._ensure_parsed()
  349. return self._entries[name]
  350. def __setitem__(self, name, value):
  351. assert isinstance(value, tuple)
  352. assert len(value) == 2
  353. self._ensure_parsed()
  354. self._entries[name] = value
  355. self._needs_serialization = True
  356. def __delitem__(self, name):
  357. self._ensure_parsed()
  358. del self._entries[name]
  359. self._needs_serialization = True
  360. def __len__(self):
  361. self._ensure_parsed()
  362. return len(self._entries)
  363. def add(self, mode, name, hexsha):
  364. assert type(mode) == int
  365. assert type(name) == str
  366. assert type(hexsha) == str
  367. self._ensure_parsed()
  368. self._entries[name] = mode, hexsha
  369. self._needs_serialization = True
  370. def entries(self):
  371. """Return a list of tuples describing the tree entries"""
  372. self._ensure_parsed()
  373. # The order of this is different from iteritems() for historical reasons
  374. return [(mode, name, hexsha) for (name, mode, hexsha) in self.iteritems()]
  375. def iteritems(self):
  376. def cmp_entry((name1, value1), (name2, value2)):
  377. if stat.S_ISDIR(value1[0]):
  378. name1 += "/"
  379. if stat.S_ISDIR(value2[0]):
  380. name2 += "/"
  381. return cmp(name1, name2)
  382. self._ensure_parsed()
  383. for name, entry in sorted(self._entries.iteritems(), cmp=cmp_entry):
  384. yield name, entry[0], entry[1]
  385. def _parse_text(self):
  386. """Grab the entries in the tree"""
  387. self._entries = parse_tree(self._text)
  388. self._needs_parsing = False
  389. def serialize(self):
  390. f = StringIO()
  391. for name, mode, hexsha in self.iteritems():
  392. f.write("%04o %s\0%s" % (mode, name, hex_to_sha(hexsha)))
  393. self._text = f.getvalue()
  394. self._needs_serialization = False
  395. def as_pretty_string(self):
  396. text = ""
  397. for name, mode, hexsha in self.iteritems():
  398. if mode & stat.S_IFDIR:
  399. kind = "tree"
  400. else:
  401. kind = "blob"
  402. text += "%04o %s %s\t%s\n" % (mode, kind, hexsha, name)
  403. return text
  404. def parse_timezone(text):
  405. offset = int(text)
  406. signum = (offset < 0) and -1 or 1
  407. offset = abs(offset)
  408. hours = int(offset / 100)
  409. minutes = (offset % 100)
  410. return signum * (hours * 3600 + minutes * 60)
  411. def format_timezone(offset):
  412. if offset % 60 != 0:
  413. raise ValueError("Unable to handle non-minute offset.")
  414. sign = (offset < 0) and '-' or '+'
  415. offset = abs(offset)
  416. return '%c%02d%02d' % (sign, offset / 3600, (offset / 60) % 60)
  417. class Commit(ShaFile):
  418. """A git commit object"""
  419. _type = COMMIT_ID
  420. _num_type = 1
  421. def __init__(self):
  422. super(Commit, self).__init__()
  423. self._parents = []
  424. self._encoding = None
  425. self._needs_parsing = False
  426. self._needs_serialization = True
  427. self._extra = {}
  428. @classmethod
  429. def from_file(cls, filename):
  430. commit = ShaFile.from_file(filename)
  431. if commit._type != cls._type:
  432. raise NotCommitError(filename)
  433. return commit
  434. def _parse_text(self):
  435. self._parents = []
  436. self._extra = []
  437. self._author = None
  438. f = StringIO(self._text)
  439. for l in f:
  440. l = l.rstrip("\n")
  441. if l == "":
  442. # Empty line indicates end of headers
  443. break
  444. (field, value) = l.split(" ", 1)
  445. if field == TREE_ID:
  446. self._tree = value
  447. elif field == PARENT_ID:
  448. self._parents.append(value)
  449. elif field == AUTHOR_ID:
  450. self._author, timetext, timezonetext = value.rsplit(" ", 2)
  451. self._author_time = int(timetext)
  452. self._author_timezone = parse_timezone(timezonetext)
  453. elif field == COMMITTER_ID:
  454. self._committer, timetext, timezonetext = value.rsplit(" ", 2)
  455. self._commit_time = int(timetext)
  456. self._commit_timezone = parse_timezone(timezonetext)
  457. elif field == ENCODING_ID:
  458. self._encoding = value
  459. else:
  460. self._extra.append((field, value))
  461. self._message = f.read()
  462. self._needs_parsing = False
  463. def serialize(self):
  464. f = StringIO()
  465. f.write("%s %s\n" % (TREE_ID, self._tree))
  466. for p in self._parents:
  467. f.write("%s %s\n" % (PARENT_ID, p))
  468. f.write("%s %s %s %s\n" % (AUTHOR_ID, self._author, str(self._author_time), format_timezone(self._author_timezone)))
  469. f.write("%s %s %s %s\n" % (COMMITTER_ID, self._committer, str(self._commit_time), format_timezone(self._commit_timezone)))
  470. if self.encoding:
  471. f.write("%s %s\n" % (ENCODING_ID, self.encoding))
  472. for k, v in self.extra:
  473. if "\n" in k or "\n" in v:
  474. raise AssertionError("newline in extra data: %r -> %r" % (k, v))
  475. f.write("%s %s\n" % (k, v))
  476. f.write("\n") # There must be a new line after the headers
  477. f.write(self._message)
  478. self._text = f.getvalue()
  479. self._needs_serialization = False
  480. tree = serializable_property("tree", "Tree that is the state of this commit")
  481. def get_parents(self):
  482. """Return a list of parents of this commit."""
  483. self._ensure_parsed()
  484. return self._parents
  485. def set_parents(self, value):
  486. """Return a list of parents of this commit."""
  487. self._ensure_parsed()
  488. self._needs_serialization = True
  489. self._parents = value
  490. parents = property(get_parents, set_parents)
  491. def get_extra(self):
  492. """Return extra settings of this commit."""
  493. self._ensure_parsed()
  494. return self._extra
  495. extra = property(get_extra)
  496. author = serializable_property("author",
  497. "The name of the author of the commit")
  498. committer = serializable_property("committer",
  499. "The name of the committer of the commit")
  500. message = serializable_property("message",
  501. "The commit message")
  502. commit_time = serializable_property("commit_time",
  503. "The timestamp of the commit. As the number of seconds since the epoch.")
  504. commit_timezone = serializable_property("commit_timezone",
  505. "The zone the commit time is in")
  506. author_time = serializable_property("author_time",
  507. "The timestamp the commit was written. as the number of seconds since the epoch.")
  508. author_timezone = serializable_property("author_timezone",
  509. "Returns the zone the author time is in.")
  510. encoding = serializable_property("encoding",
  511. "Encoding of the commit message.")
  512. type_map = {
  513. BLOB_ID : Blob,
  514. TREE_ID : Tree,
  515. COMMIT_ID : Commit,
  516. TAG_ID: Tag,
  517. }
  518. num_type_map = {
  519. 0: None,
  520. 1: Commit,
  521. 2: Tree,
  522. 3: Blob,
  523. 4: Tag,
  524. # 5 Is reserved for further expansion
  525. }
  526. try:
  527. # Try to import C versions
  528. from dulwich._objects import parse_tree
  529. except ImportError:
  530. pass