client.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. # client.py -- Implementation of the server side git protocols
  2. # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
  3. # Copyright (C) 2008 John Carr
  4. #
  5. # This program is free software; you can redistribute it and/or
  6. # modify it under the terms of the GNU General Public License
  7. # as published by the Free Software Foundation; either version 2
  8. # or (at your option) a later version of the License.
  9. #
  10. # This program is distributed in the hope that it will be useful,
  11. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. # GNU General Public License for more details.
  14. #
  15. # You should have received a copy of the GNU General Public License
  16. # along with this program; if not, write to the Free Software
  17. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  18. # MA 02110-1301, USA.
  19. """Client side support for the Git protocol."""
  20. __docformat__ = 'restructuredText'
  21. import os
  22. import select
  23. import socket
  24. import subprocess
  25. from dulwich.errors import (
  26. ChecksumMismatch,
  27. SendPackError,
  28. UpdateRefsError,
  29. )
  30. from dulwich.protocol import (
  31. Protocol,
  32. TCP_GIT_PORT,
  33. ZERO_SHA,
  34. extract_capabilities,
  35. )
  36. from dulwich.pack import (
  37. write_pack_data,
  38. )
  39. def _fileno_can_read(fileno):
  40. """Check if a file descriptor is readable."""
  41. return len(select.select([fileno], [], [], 0)[0]) > 0
  42. COMMON_CAPABILITIES = ["ofs-delta"]
  43. FETCH_CAPABILITIES = ["multi_ack", "side-band-64k"] + COMMON_CAPABILITIES
  44. SEND_CAPABILITIES = ['report-status'] + COMMON_CAPABILITIES
  45. # TODO(durin42): this doesn't correctly degrade if the server doesn't
  46. # support some capabilities. This should work properly with servers
  47. # that don't support side-band-64k and multi_ack.
  48. class GitClient(object):
  49. """Git smart server client.
  50. """
  51. def __init__(self, can_read, read, write, thin_packs=True,
  52. report_activity=None):
  53. """Create a new GitClient instance.
  54. :param can_read: Function that returns True if there is data available
  55. to be read.
  56. :param read: Callback for reading data, takes number of bytes to read
  57. :param write: Callback for writing data
  58. :param thin_packs: Whether or not thin packs should be retrieved
  59. :param report_activity: Optional callback for reporting transport
  60. activity.
  61. """
  62. self.proto = Protocol(read, write, report_activity)
  63. self._can_read = can_read
  64. self._fetch_capabilities = list(FETCH_CAPABILITIES)
  65. self._send_capabilities = list(SEND_CAPABILITIES)
  66. if thin_packs:
  67. self._fetch_capabilities.append("thin-pack")
  68. def read_refs(self):
  69. server_capabilities = None
  70. refs = {}
  71. # Receive refs from server
  72. for pkt in self.proto.read_pkt_seq():
  73. (sha, ref) = pkt.rstrip("\n").split(" ", 1)
  74. if server_capabilities is None:
  75. (ref, server_capabilities) = extract_capabilities(ref)
  76. refs[ref] = sha
  77. return refs, server_capabilities
  78. def _parse_status_report(self):
  79. unpack = self.proto.read_pkt_line().strip()
  80. if unpack != 'unpack ok':
  81. st = True
  82. # flush remaining error data
  83. while st is not None:
  84. st = self.proto.read_pkt_line()
  85. raise SendPackError(unpack)
  86. statuses = []
  87. errs = False
  88. ref_status = self.proto.read_pkt_line()
  89. while ref_status:
  90. ref_status = ref_status.strip()
  91. statuses.append(ref_status)
  92. if not ref_status.startswith('ok '):
  93. errs = True
  94. ref_status = self.proto.read_pkt_line()
  95. if errs:
  96. ref_status = {}
  97. ok = set()
  98. for status in statuses:
  99. if ' ' not in status:
  100. # malformed response, move on to the next one
  101. continue
  102. status, ref = status.split(' ', 1)
  103. if status == 'ng':
  104. if ' ' in ref:
  105. ref, status = ref.split(' ', 1)
  106. else:
  107. ok.add(ref)
  108. ref_status[ref] = status
  109. raise UpdateRefsError('%s failed to update' %
  110. ', '.join([ref for ref in ref_status
  111. if ref not in ok]),
  112. ref_status=ref_status)
  113. # TODO(durin42): add side-band-64k capability support here and advertise it
  114. def send_pack(self, path, determine_wants, generate_pack_contents):
  115. """Upload a pack to a remote repository.
  116. :param path: Repository path
  117. :param generate_pack_contents: Function that can return the shas of the
  118. objects to upload.
  119. :raises SendPackError: if server rejects the pack data
  120. :raises UpdateRefsError: if the server supports report-status
  121. and rejects ref updates
  122. """
  123. old_refs, server_capabilities = self.read_refs()
  124. if 'report-status' not in server_capabilities:
  125. self._send_capabilities.remove('report-status')
  126. new_refs = determine_wants(old_refs)
  127. if not new_refs:
  128. self.proto.write_pkt_line(None)
  129. return {}
  130. want = []
  131. have = [x for x in old_refs.values() if not x == ZERO_SHA]
  132. sent_capabilities = False
  133. for refname in set(new_refs.keys() + old_refs.keys()):
  134. old_sha1 = old_refs.get(refname, ZERO_SHA)
  135. new_sha1 = new_refs.get(refname, ZERO_SHA)
  136. if old_sha1 != new_sha1:
  137. if sent_capabilities:
  138. self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1,
  139. refname))
  140. else:
  141. self.proto.write_pkt_line(
  142. "%s %s %s\0%s" % (old_sha1, new_sha1, refname,
  143. ' '.join(self._send_capabilities)))
  144. sent_capabilities = True
  145. if not new_sha1 in (have, ZERO_SHA):
  146. want.append(new_sha1)
  147. self.proto.write_pkt_line(None)
  148. if not want:
  149. return new_refs
  150. objects = generate_pack_contents(have, want)
  151. (entries, sha) = write_pack_data(self.proto.write_file(), objects,
  152. len(objects))
  153. if 'report-status' in self._send_capabilities:
  154. self._parse_status_report()
  155. # wait for EOF before returning
  156. data = self.proto.read()
  157. if data:
  158. raise SendPackError('Unexpected response %r' % data)
  159. return new_refs
  160. def fetch(self, path, target, determine_wants=None, progress=None):
  161. """Fetch into a target repository.
  162. :param path: Path to fetch from
  163. :param target: Target repository to fetch into
  164. :param determine_wants: Optional function to determine what refs
  165. to fetch
  166. :param progress: Optional progress function
  167. :return: remote refs
  168. """
  169. if determine_wants is None:
  170. determine_wants = target.object_store.determine_wants_all
  171. f, commit = target.object_store.add_pack()
  172. try:
  173. return self.fetch_pack(path, determine_wants,
  174. target.get_graph_walker(), f.write, progress)
  175. finally:
  176. commit()
  177. def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
  178. progress):
  179. """Retrieve a pack from a git smart server.
  180. :param determine_wants: Callback that returns list of commits to fetch
  181. :param graph_walker: Object with next() and ack().
  182. :param pack_data: Callback called for each bit of data in the pack
  183. :param progress: Callback for progress reports (strings)
  184. """
  185. (refs, server_capabilities) = self.read_refs()
  186. wants = determine_wants(refs)
  187. if not wants:
  188. self.proto.write_pkt_line(None)
  189. return refs
  190. assert isinstance(wants, list) and type(wants[0]) == str
  191. self.proto.write_pkt_line("want %s %s\n" % (
  192. wants[0], ' '.join(self._fetch_capabilities)))
  193. for want in wants[1:]:
  194. self.proto.write_pkt_line("want %s\n" % want)
  195. self.proto.write_pkt_line(None)
  196. have = graph_walker.next()
  197. while have:
  198. self.proto.write_pkt_line("have %s\n" % have)
  199. if self._can_read():
  200. pkt = self.proto.read_pkt_line()
  201. parts = pkt.rstrip("\n").split(" ")
  202. if parts[0] == "ACK":
  203. graph_walker.ack(parts[1])
  204. assert parts[2] == "continue"
  205. have = graph_walker.next()
  206. self.proto.write_pkt_line("done\n")
  207. pkt = self.proto.read_pkt_line()
  208. while pkt:
  209. parts = pkt.rstrip("\n").split(" ")
  210. if parts[0] == "ACK":
  211. graph_walker.ack(pkt.split(" ")[1])
  212. if len(parts) < 3 or parts[2] != "continue":
  213. break
  214. pkt = self.proto.read_pkt_line()
  215. # TODO(durin42): this is broken if the server didn't support the
  216. # side-band-64k capability.
  217. for pkt in self.proto.read_pkt_seq():
  218. channel = ord(pkt[0])
  219. pkt = pkt[1:]
  220. if channel == 1:
  221. pack_data(pkt)
  222. elif channel == 2:
  223. if progress is not None:
  224. progress(pkt)
  225. else:
  226. raise AssertionError("Invalid sideband channel %d" % channel)
  227. return refs
  228. class TCPGitClient(GitClient):
  229. """A Git Client that works over TCP directly (i.e. git://)."""
  230. def __init__(self, host, port=None, *args, **kwargs):
  231. self._socket = socket.socket(type=socket.SOCK_STREAM)
  232. if port is None:
  233. port = TCP_GIT_PORT
  234. self._socket.connect((host, port))
  235. self.rfile = self._socket.makefile('rb', -1)
  236. self.wfile = self._socket.makefile('wb', 0)
  237. self.host = host
  238. super(TCPGitClient, self).__init__(lambda: _fileno_can_read(self._socket.fileno()), self.rfile.read, self.wfile.write, *args, **kwargs)
  239. def send_pack(self, path, changed_refs, generate_pack_contents):
  240. """Send a pack to a remote host.
  241. :param path: Path of the repository on the remote host
  242. """
  243. self.proto.send_cmd("git-receive-pack", path, "host=%s" % self.host)
  244. return super(TCPGitClient, self).send_pack(path, changed_refs, generate_pack_contents)
  245. def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
  246. """Fetch a pack from the remote host.
  247. :param path: Path of the reposiutory on the remote host
  248. :param determine_wants: Callback that receives available refs dict and
  249. should return list of sha's to fetch.
  250. :param graph_walker: GraphWalker instance used to find missing shas
  251. :param pack_data: Callback for writing pack data
  252. :param progress: Callback for writing progress
  253. """
  254. self.proto.send_cmd("git-upload-pack", path, "host=%s" % self.host)
  255. return super(TCPGitClient, self).fetch_pack(path, determine_wants,
  256. graph_walker, pack_data, progress)
  257. class SubprocessGitClient(GitClient):
  258. """Git client that talks to a server using a subprocess."""
  259. def __init__(self, *args, **kwargs):
  260. self.proc = None
  261. self._args = args
  262. self._kwargs = kwargs
  263. def _connect(self, service, *args, **kwargs):
  264. argv = [service] + list(args)
  265. self.proc = subprocess.Popen(argv, bufsize=0,
  266. stdin=subprocess.PIPE,
  267. stdout=subprocess.PIPE)
  268. def read_fn(size):
  269. return self.proc.stdout.read(size)
  270. def write_fn(data):
  271. self.proc.stdin.write(data)
  272. self.proc.stdin.flush()
  273. return GitClient(lambda: _fileno_can_read(self.proc.stdout.fileno()), read_fn, write_fn, *args, **kwargs)
  274. def send_pack(self, path, changed_refs, generate_pack_contents):
  275. """Upload a pack to the server.
  276. :param path: Path to the git repository on the server
  277. :param changed_refs: Dictionary with new values for the refs
  278. :param generate_pack_contents: Function that returns an iterator over
  279. objects to send
  280. """
  281. client = self._connect("git-receive-pack", path)
  282. return client.send_pack(path, changed_refs, generate_pack_contents)
  283. def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
  284. progress):
  285. """Retrieve a pack from the server
  286. :param path: Path to the git repository on the server
  287. :param determine_wants: Function that receives existing refs
  288. on the server and returns a list of desired shas
  289. :param graph_walker: GraphWalker instance
  290. :param pack_data: Function that can write pack data
  291. :param progress: Function that can write progress texts
  292. """
  293. client = self._connect("git-upload-pack", path)
  294. return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
  295. progress)
  296. class SSHSubprocess(object):
  297. """A socket-like object that talks to an ssh subprocess via pipes."""
  298. def __init__(self, proc):
  299. self.proc = proc
  300. self.read = self.recv = proc.stdout.read
  301. self.write = self.send = proc.stdin.write
  302. def close(self):
  303. self.proc.stdin.close()
  304. self.proc.stdout.close()
  305. self.proc.wait()
  306. class SSHVendor(object):
  307. def connect_ssh(self, host, command, username=None, port=None):
  308. #FIXME: This has no way to deal with passwords..
  309. args = ['ssh', '-x']
  310. if port is not None:
  311. args.extend(['-p', str(port)])
  312. if username is not None:
  313. host = "%s@%s" % (username, host)
  314. args.append(host)
  315. proc = subprocess.Popen(args + command,
  316. stdin=subprocess.PIPE,
  317. stdout=subprocess.PIPE)
  318. return SSHSubprocess(proc)
  319. # Can be overridden by users
  320. get_ssh_vendor = SSHVendor
  321. class SSHGitClient(GitClient):
  322. def __init__(self, host, port=None, username=None, *args, **kwargs):
  323. self.host = host
  324. self.port = port
  325. self.username = username
  326. self._args = args
  327. self._kwargs = kwargs
  328. def send_pack(self, path, determine_wants, generate_pack_contents):
  329. remote = get_ssh_vendor().connect_ssh(
  330. self.host, ["git-receive-pack '%s'" % path],
  331. port=self.port, username=self.username)
  332. client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
  333. return client.send_pack(path, determine_wants, generate_pack_contents)
  334. def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
  335. progress):
  336. remote = get_ssh_vendor().connect_ssh(self.host, ["git-upload-pack '%s'" % path], port=self.port, username=self.username)
  337. client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
  338. return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
  339. progress)
  340. def get_transport_and_path(uri):
  341. """Obtain a git client from a URI or path.
  342. :param uri: URI or path
  343. :return: Tuple with client instance and relative path.
  344. """
  345. from dulwich.client import TCPGitClient, SSHGitClient, SubprocessGitClient
  346. for handler, transport in (("git://", TCPGitClient), ("git+ssh://", SSHGitClient)):
  347. if uri.startswith(handler):
  348. host, path = uri[len(handler):].split("/", 1)
  349. return transport(host), "/"+path
  350. # if its not git or git+ssh, try a local url..
  351. return SubprocessGitClient(), uri