|
@@ -133,6 +133,15 @@ class HTTPUnauthorized(Exception):
|
|
|
self.url = url
|
|
|
|
|
|
|
|
|
+class HTTPProxyUnauthorized(Exception):
|
|
|
+ """Raised when proxy authentication fails."""
|
|
|
+
|
|
|
+ def __init__(self, proxy_authenticate, url):
|
|
|
+ Exception.__init__(self, "No valid proxy credentials provided")
|
|
|
+ self.proxy_authenticate = proxy_authenticate
|
|
|
+ self.url = url
|
|
|
+
|
|
|
+
|
|
|
def _fileno_can_read(fileno):
|
|
|
"""Check if a file descriptor is readable."""
|
|
|
return len(select.select([fileno], [], [], 0)[0]) > 0
|
|
@@ -549,8 +558,8 @@ class GitClient(object):
|
|
|
Args:
|
|
|
path: Remote path to fetch from
|
|
|
determine_wants: Function determine what refs
|
|
|
- to fetch. Receives dictionary of name->sha, should return
|
|
|
- list of shas to fetch.
|
|
|
+ to fetch. Receives dictionary of name->sha, should return
|
|
|
+ list of shas to fetch.
|
|
|
graph_walker: Object with next() and ack().
|
|
|
pack_data: Callback called for each bit of data in the pack
|
|
|
progress: Callback for progress reports (strings)
|
|
@@ -901,10 +910,10 @@ class TraditionalGitClient(GitClient):
|
|
|
Args:
|
|
|
path: Repository path (as bytestring)
|
|
|
update_refs: Function to determine changes to remote refs.
|
|
|
- Receive dict with existing remote refs, returns dict with
|
|
|
- changed refs (name -> sha, where sha=ZERO_SHA for deletions)
|
|
|
+ Receive dict with existing remote refs, returns dict with
|
|
|
+ changed refs (name -> sha, where sha=ZERO_SHA for deletions)
|
|
|
generate_pack_data: Function that can return a tuple with
|
|
|
- number of objects and pack data to upload.
|
|
|
+ number of objects and pack data to upload.
|
|
|
progress: Optional callback called with progress updates
|
|
|
|
|
|
Returns:
|
|
@@ -995,8 +1004,8 @@ class TraditionalGitClient(GitClient):
|
|
|
Args:
|
|
|
path: Remote path to fetch from
|
|
|
determine_wants: Function determine what refs
|
|
|
- to fetch. Receives dictionary of name->sha, should return
|
|
|
- list of shas to fetch.
|
|
|
+ to fetch. Receives dictionary of name->sha, should return
|
|
|
+ list of shas to fetch.
|
|
|
graph_walker: Object with next() and ack().
|
|
|
pack_data: Callback called for each bit of data in the pack
|
|
|
progress: Callback for progress reports (strings)
|
|
@@ -1292,9 +1301,9 @@ class LocalGitClient(GitClient):
|
|
|
Args:
|
|
|
path: Repository path (as bytestring)
|
|
|
update_refs: Function to determine changes to remote refs.
|
|
|
- Receive dict with existing remote refs, returns dict with
|
|
|
- changed refs (name -> sha, where sha=ZERO_SHA for deletions)
|
|
|
- with number of items and pack data to upload.
|
|
|
+ Receive dict with existing remote refs, returns dict with
|
|
|
+ changed refs (name -> sha, where sha=ZERO_SHA for deletions)
|
|
|
+ with number of items and pack data to upload.
|
|
|
progress: Optional progress function
|
|
|
|
|
|
Returns:
|
|
@@ -1353,8 +1362,8 @@ class LocalGitClient(GitClient):
|
|
|
path: Path to fetch from (as bytestring)
|
|
|
target: Target repository to fetch into
|
|
|
determine_wants: Optional function determine what refs
|
|
|
- to fetch. Receives dictionary of name->sha, should return
|
|
|
- list of shas to fetch. Defaults to all shas.
|
|
|
+ to fetch. Receives dictionary of name->sha, should return
|
|
|
+ list of shas to fetch. Defaults to all shas.
|
|
|
progress: Optional progress function
|
|
|
depth: Shallow fetch depth
|
|
|
|
|
@@ -1385,8 +1394,8 @@ class LocalGitClient(GitClient):
|
|
|
Args:
|
|
|
path: Remote path to fetch from
|
|
|
determine_wants: Function determine what refs
|
|
|
- to fetch. Receives dictionary of name->sha, should return
|
|
|
- list of shas to fetch.
|
|
|
+ to fetch. Receives dictionary of name->sha, should return
|
|
|
+ list of shas to fetch.
|
|
|
graph_walker: Object with next() and ack().
|
|
|
pack_data: Callback called for each bit of data in the pack
|
|
|
progress: Callback for progress reports (strings)
|
|
@@ -1759,8 +1768,6 @@ def default_urllib3_manager( # noqa: C901
|
|
|
if proxy_server is not None:
|
|
|
if proxy_manager_cls is None:
|
|
|
proxy_manager_cls = urllib3.ProxyManager
|
|
|
- # `urllib3` requires a `str` object in both Python 2 and 3, while
|
|
|
- # `ConfigDict` coerces entries to `bytes` on Python 3. Compensate.
|
|
|
if not isinstance(proxy_server, str):
|
|
|
proxy_server = proxy_server.decode()
|
|
|
manager = proxy_manager_cls(proxy_server, headers=headers, **kwargs)
|
|
@@ -1772,71 +1779,20 @@ def default_urllib3_manager( # noqa: C901
|
|
|
return manager
|
|
|
|
|
|
|
|
|
-class HttpGitClient(GitClient):
|
|
|
- def __init__(
|
|
|
- self,
|
|
|
- base_url,
|
|
|
- dumb=None,
|
|
|
- pool_manager=None,
|
|
|
- config=None,
|
|
|
- username=None,
|
|
|
- password=None,
|
|
|
- **kwargs
|
|
|
- ):
|
|
|
- self._base_url = base_url.rstrip("/") + "/"
|
|
|
- self._username = username
|
|
|
- self._password = password
|
|
|
- self.dumb = dumb
|
|
|
-
|
|
|
- if pool_manager is None:
|
|
|
- self.pool_manager = default_urllib3_manager(config)
|
|
|
- else:
|
|
|
- self.pool_manager = pool_manager
|
|
|
+class AbstractHttpGitClient(GitClient):
|
|
|
+ """Abstract base class for HTTP Git Clients.
|
|
|
|
|
|
- if username is not None:
|
|
|
- # No escaping needed: ":" is not allowed in username:
|
|
|
- # https://tools.ietf.org/html/rfc2617#section-2
|
|
|
- credentials = "%s:%s" % (username, password)
|
|
|
- import urllib3.util
|
|
|
+ This is agonistic of the actual HTTP implementation.
|
|
|
|
|
|
- basic_auth = urllib3.util.make_headers(basic_auth=credentials)
|
|
|
- self.pool_manager.headers.update(basic_auth)
|
|
|
+ Subclasses should provide an implementation of the
|
|
|
+ _http_request method.
|
|
|
+ """
|
|
|
|
|
|
+ def __init__(self, base_url, dumb=False, **kwargs):
|
|
|
+ self._base_url = base_url.rstrip("/") + "/"
|
|
|
+ self.dumb = dumb
|
|
|
GitClient.__init__(self, **kwargs)
|
|
|
|
|
|
- def get_url(self, path):
|
|
|
- return self._get_url(path).rstrip("/")
|
|
|
-
|
|
|
- @classmethod
|
|
|
- def from_parsedurl(cls, parsedurl, **kwargs):
|
|
|
- password = parsedurl.password
|
|
|
- if password is not None:
|
|
|
- kwargs["password"] = urlunquote(password)
|
|
|
- username = parsedurl.username
|
|
|
- if username is not None:
|
|
|
- kwargs["username"] = urlunquote(username)
|
|
|
- netloc = parsedurl.hostname
|
|
|
- if parsedurl.port:
|
|
|
- netloc = "%s:%s" % (netloc, parsedurl.port)
|
|
|
- if parsedurl.username:
|
|
|
- netloc = "%s@%s" % (parsedurl.username, netloc)
|
|
|
- parsedurl = parsedurl._replace(netloc=netloc)
|
|
|
- return cls(urlunparse(parsedurl), **kwargs)
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- return "%s(%r, dumb=%r)" % (
|
|
|
- type(self).__name__,
|
|
|
- self._base_url,
|
|
|
- self.dumb,
|
|
|
- )
|
|
|
-
|
|
|
- def _get_url(self, path):
|
|
|
- if not isinstance(path, str):
|
|
|
- # urllib3.util.url._encode_invalid_chars() converts the path back
|
|
|
- # to bytes using the utf-8 codec.
|
|
|
- path = path.decode("utf-8")
|
|
|
- return urljoin(self._base_url, path).rstrip("/") + "/"
|
|
|
-
|
|
|
def _http_request(self, url, headers=None, data=None, allow_compression=False):
|
|
|
"""Perform HTTP request.
|
|
|
|
|
@@ -1853,48 +1809,8 @@ class HttpGitClient(GitClient):
|
|
|
method for the response data.
|
|
|
|
|
|
"""
|
|
|
- req_headers = self.pool_manager.headers.copy()
|
|
|
- if headers is not None:
|
|
|
- req_headers.update(headers)
|
|
|
- req_headers["Pragma"] = "no-cache"
|
|
|
- if allow_compression:
|
|
|
- req_headers["Accept-Encoding"] = "gzip"
|
|
|
- else:
|
|
|
- req_headers["Accept-Encoding"] = "identity"
|
|
|
-
|
|
|
- if data is None:
|
|
|
- resp = self.pool_manager.request("GET", url, headers=req_headers)
|
|
|
- else:
|
|
|
- resp = self.pool_manager.request(
|
|
|
- "POST", url, headers=req_headers, body=data
|
|
|
- )
|
|
|
|
|
|
- if resp.status == 404:
|
|
|
- raise NotGitRepository()
|
|
|
- if resp.status == 401:
|
|
|
- raise HTTPUnauthorized(resp.getheader("WWW-Authenticate"), url)
|
|
|
- if resp.status != 200:
|
|
|
- raise GitProtocolError(
|
|
|
- "unexpected http resp %d for %s" % (resp.status, url)
|
|
|
- )
|
|
|
-
|
|
|
- # TODO: Optimization available by adding `preload_content=False` to the
|
|
|
- # request and just passing the `read` method on instead of going via
|
|
|
- # `BytesIO`, if we can guarantee that the entire response is consumed
|
|
|
- # before issuing the next to still allow for connection reuse from the
|
|
|
- # pool.
|
|
|
- read = BytesIO(resp.data).read
|
|
|
-
|
|
|
- resp.content_type = resp.getheader("Content-Type")
|
|
|
- # Check if geturl() is available (urllib3 version >= 1.23)
|
|
|
- try:
|
|
|
- resp_url = resp.geturl()
|
|
|
- except AttributeError:
|
|
|
- # get_redirect_location() is available for urllib3 >= 1.1
|
|
|
- resp.redirect_location = resp.get_redirect_location()
|
|
|
- else:
|
|
|
- resp.redirect_location = resp_url if resp_url != url else ""
|
|
|
- return resp, read
|
|
|
+ raise NotImplementedError(self._http_request)
|
|
|
|
|
|
def _discover_references(self, service, base_url):
|
|
|
assert base_url[-1] == "/"
|
|
@@ -1934,6 +1850,11 @@ class HttpGitClient(GitClient):
|
|
|
resp.close()
|
|
|
|
|
|
def _smart_request(self, service, url, data):
|
|
|
+ """Send a 'smart' HTTP request.
|
|
|
+
|
|
|
+ This is a simple wrapper around _http_request that sets
|
|
|
+ a couple of extra headers.
|
|
|
+ """
|
|
|
assert url[-1] == "/"
|
|
|
url = urljoin(url, service)
|
|
|
result_content_type = "application/x-%s-result" % service
|
|
@@ -1955,10 +1876,10 @@ class HttpGitClient(GitClient):
|
|
|
Args:
|
|
|
path: Repository path (as bytestring)
|
|
|
update_refs: Function to determine changes to remote refs.
|
|
|
- Receives dict with existing remote refs, returns dict with
|
|
|
- changed refs (name -> sha, where sha=ZERO_SHA for deletions)
|
|
|
+ Receives dict with existing remote refs, returns dict with
|
|
|
+ changed refs (name -> sha, where sha=ZERO_SHA for deletions)
|
|
|
generate_pack_data: Function that can return a tuple
|
|
|
- with number of elements and pack data to upload.
|
|
|
+ with number of elements and pack data to upload.
|
|
|
progress: Optional progress function
|
|
|
|
|
|
Returns:
|
|
@@ -2089,6 +2010,123 @@ class HttpGitClient(GitClient):
|
|
|
refs, _, _ = self._discover_references(b"git-upload-pack", url)
|
|
|
return refs
|
|
|
|
|
|
+ def get_url(self, path):
|
|
|
+ return self._get_url(path).rstrip("/")
|
|
|
+
|
|
|
+ def _get_url(self, path):
|
|
|
+ return urljoin(self._base_url, path).rstrip("/") + "/"
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def from_parsedurl(cls, parsedurl, **kwargs):
|
|
|
+ password = parsedurl.password
|
|
|
+ if password is not None:
|
|
|
+ kwargs["password"] = urlunquote(password)
|
|
|
+ username = parsedurl.username
|
|
|
+ if username is not None:
|
|
|
+ kwargs["username"] = urlunquote(username)
|
|
|
+ netloc = parsedurl.hostname
|
|
|
+ if parsedurl.port:
|
|
|
+ netloc = "%s:%s" % (netloc, parsedurl.port)
|
|
|
+ if parsedurl.username:
|
|
|
+ netloc = "%s@%s" % (parsedurl.username, netloc)
|
|
|
+ parsedurl = parsedurl._replace(netloc=netloc)
|
|
|
+ return cls(urlunparse(parsedurl), **kwargs)
|
|
|
+
|
|
|
+ def __repr__(self):
|
|
|
+ return "%s(%r, dumb=%r)" % (
|
|
|
+ type(self).__name__,
|
|
|
+ self._base_url,
|
|
|
+ self.dumb,
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+class Urllib3HttpGitClient(AbstractHttpGitClient):
|
|
|
+ def __init__(
|
|
|
+ self,
|
|
|
+ base_url,
|
|
|
+ dumb=None,
|
|
|
+ pool_manager=None,
|
|
|
+ config=None,
|
|
|
+ username=None,
|
|
|
+ password=None,
|
|
|
+ **kwargs
|
|
|
+ ):
|
|
|
+ self._username = username
|
|
|
+ self._password = password
|
|
|
+
|
|
|
+ if pool_manager is None:
|
|
|
+ self.pool_manager = default_urllib3_manager(config)
|
|
|
+ else:
|
|
|
+ self.pool_manager = pool_manager
|
|
|
+
|
|
|
+ if username is not None:
|
|
|
+ # No escaping needed: ":" is not allowed in username:
|
|
|
+ # https://tools.ietf.org/html/rfc2617#section-2
|
|
|
+ credentials = "%s:%s" % (username, password)
|
|
|
+ import urllib3.util
|
|
|
+
|
|
|
+ basic_auth = urllib3.util.make_headers(basic_auth=credentials)
|
|
|
+ self.pool_manager.headers.update(basic_auth)
|
|
|
+
|
|
|
+ super(Urllib3HttpGitClient, self).__init__(
|
|
|
+ base_url=base_url, dumb=dumb, **kwargs)
|
|
|
+
|
|
|
+ def _get_url(self, path):
|
|
|
+ if not isinstance(path, str):
|
|
|
+ # urllib3.util.url._encode_invalid_chars() converts the path back
|
|
|
+ # to bytes using the utf-8 codec.
|
|
|
+ path = path.decode("utf-8")
|
|
|
+ return urljoin(self._base_url, path).rstrip("/") + "/"
|
|
|
+
|
|
|
+ def _http_request(self, url, headers=None, data=None, allow_compression=False):
|
|
|
+ req_headers = self.pool_manager.headers.copy()
|
|
|
+ if headers is not None:
|
|
|
+ req_headers.update(headers)
|
|
|
+ req_headers["Pragma"] = "no-cache"
|
|
|
+ if allow_compression:
|
|
|
+ req_headers["Accept-Encoding"] = "gzip"
|
|
|
+ else:
|
|
|
+ req_headers["Accept-Encoding"] = "identity"
|
|
|
+
|
|
|
+ if data is None:
|
|
|
+ resp = self.pool_manager.request("GET", url, headers=req_headers)
|
|
|
+ else:
|
|
|
+ resp = self.pool_manager.request(
|
|
|
+ "POST", url, headers=req_headers, body=data
|
|
|
+ )
|
|
|
+
|
|
|
+ if resp.status == 404:
|
|
|
+ raise NotGitRepository()
|
|
|
+ if resp.status == 401:
|
|
|
+ raise HTTPUnauthorized(resp.getheader("WWW-Authenticate"), url)
|
|
|
+ if resp.status == 407:
|
|
|
+ raise HTTPProxyUnauthorized(resp.getheader("Proxy-Authenticate"), url)
|
|
|
+ if resp.status != 200:
|
|
|
+ raise GitProtocolError(
|
|
|
+ "unexpected http resp %d for %s" % (resp.status, url)
|
|
|
+ )
|
|
|
+
|
|
|
+ # TODO: Optimization available by adding `preload_content=False` to the
|
|
|
+ # request and just passing the `read` method on instead of going via
|
|
|
+ # `BytesIO`, if we can guarantee that the entire response is consumed
|
|
|
+ # before issuing the next to still allow for connection reuse from the
|
|
|
+ # pool.
|
|
|
+ read = BytesIO(resp.data).read
|
|
|
+
|
|
|
+ resp.content_type = resp.getheader("Content-Type")
|
|
|
+ # Check if geturl() is available (urllib3 version >= 1.23)
|
|
|
+ try:
|
|
|
+ resp_url = resp.geturl()
|
|
|
+ except AttributeError:
|
|
|
+ # get_redirect_location() is available for urllib3 >= 1.1
|
|
|
+ resp.redirect_location = resp.get_redirect_location()
|
|
|
+ else:
|
|
|
+ resp.redirect_location = resp_url if resp_url != url else ""
|
|
|
+ return resp, read
|
|
|
+
|
|
|
+
|
|
|
+HttpGitClient = Urllib3HttpGitClient
|
|
|
+
|
|
|
|
|
|
def get_transport_and_path_from_url(url, config=None, **kwargs):
|
|
|
"""Obtain a git client from a URL.
|