Import upstream version 0.20.2, md5 8bc7339b505768a84c74981b04a5157f

.github/workflows/pythonpackage.yml

@@ -0,0 +1,54 @@
+name: Python package
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: [3.5, 3.6, 3.7, 3.8, pypy3]
+        exclude:
+          # sqlite3 exit handling seems to get in the way
+          - os: macos-latest
+            python-version: pypy3
+          # doesn't support passing in bytestrings to os.scandir
+          - os: windows-latest
+            python-version: pypy3
+          # path encoding
+          - os: windows-latest
+            python-version: 3.5
+      fail-fast: false
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U pip coverage codecov flake8 fastimport
+    - name: Install mypy
+      run: |
+        pip install -U mypy
+      if: "matrix.python-version != 'pypy3'"
+    - name: Style checks
+      run: |
+        python -m flake8
+    - name: Typing checks
+      run: |
+        python -m mypy dulwich
+      if: "matrix.python-version != 'pypy3'"
+    - name: Build
+      run: |
+        python setup.py build_ext -i
+    - name: Coverage test suite run
+      run: |
+        python -m coverage run -p -m unittest dulwich.tests.test_suite
+    - name: Upload coverage details
+      run: |
+        codecov

.github/workflows/pythonpublish.yml

@@ -0,0 +1,60 @@
+name: Upload Python Package
+
+on:
+  push:
+    tags:
+      - dulwich-*
+
+jobs:
+  deploy:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest, windows-latest]
+        python-version: ['3.5', '3.6', '3.7', '3.8']
+        include:
+          - os: ubuntu-latest
+            python-version: '3.x'
+          # path encoding
+        exclude:
+          - os: windows-latest
+            python-version: 3.5
+      fail-fast: false
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine fastimport
+    - name: Run test suite
+      run: |
+        python -m unittest dulwich.tests.test_suite
+    - name: Build
+      run: |
+        python setup.py sdist bdist_wheel
+        mkdir wheelhouse
+        mv dist/*.whl wheelhouse
+      if: "matrix.os != 'ubuntu-latest'"
+    - name: Build and publish (Linux)
+      uses: RalfG/python-wheels-manylinux-build@v0.2.2
+      if: "matrix.os == 'ubuntu-latest'"
+    - name: Publish (Linux)
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        twine upload wheelhouse/*manylinux*
+      if: "matrix.os == 'ubuntu-latest'"
+    - name: Publish
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        twine upload wheelhouse/*
+      if: "matrix.os != 'ubuntu-latest'"

.gitignore

@@ -22,3 +22,4 @@ dulwich.egg-info/
 .coverage
 htmlcov/
 docs/api/*.txt
+.mypy_cache/

.travis.yml

@@ -1,69 +0,0 @@
-language: python
-sudo: false
-cache: pip
-
-
-python:
-  - 2.7
-  - 3.4
-  - 3.5
-  - 3.6
-  - pypy3.5
-
-env:
-  - PYTHONHASHSEED=random
-    TEST_REQUIRE="gevent greenlet geventhttpclient fastimport"
-    PURE=false
-
-matrix:
-  include:
-    - python: pypy
-      env: TEST_REQUIRE=fastimport
-    - python: 3.7
-      env: TEST_REQUIRE=fastimport
-      dist: xenial
-      sudo: true
-    - python: 3.8
-      env: TEST_REQUIRE=fastimport
-      dist: xenial
-      sudo: true
-    - python: 3.8
-      env: TEST_REQUIRE=fastimport
-      dist: xenial
-      sudo: true
-    - python: 3.6
-      env: PURE=true
-    - python: 2.7
-      env: PURE=true
-
-install:
-  - travis_retry pip install -U pip coverage codecov flake8 $TEST_REQUIRE
-
-script:
-  - if [ $PURE = false ]; then python setup.py build_ext -i; fi
-  - python -m coverage run -p -m unittest dulwich.tests.test_suite
-
-  # Style
-  - make style
-
-  - if [ $PURE = true ]; then SETUP_ARGS=--pure; fi
-  - python setup.py $SETUP_ARGS bdist_wheel
-
-after_success:
-  - python -m coverage combine
-  - codecov
-
-deploy:
-  provider: pypi
-  user: dulwich-bot
-  password:
-    secure: Q8DDDojBugQWzXvmmEQiU90UkVPk+OYoFZwv1H9LYpQ4u5CfwQNWpf8qXYhlGMdr/gzWaSWsqLvgWLpzfkvqS4Vyk2bO9mr+dSskfD8uwc82LiiL9CNd/NY03CjH9RaFgVMD/+exMjY/yCtlyH1jL4kjgOyNnC+x4B37CliZHcE=
-  skip_cleanup: true
-  skip_existing: true
-  file_glob: true
-  file:
-    - dist/dulwich*.whl
-    - dist/dulwich*.tar.gz
-  on:
-    tags: true
-    repo: dulwich/dulwich

AUTHORS

@@ -148,5 +148,7 @@ KS Chan <mrkschan@gmail.com>
 egor <egor@sourced.tech>
 Antoine Lambert <anlambert@softwareheritage.org>
 Lane Barlow <lane.barlow@gmail.com>
+Manuel Jacob <me@manueljacob.de>
+Brecht Machiels <brecht@mos6581.org>
 
 If you contributed but are missing from this list, please send me an e-mail.

CONTRIBUTING.rst

@@ -15,7 +15,7 @@ Furthermore, triple-quotes should always be """, single quotes are ' unless
 using " would result in less escaping within the string.
 
 Public methods, functions and classes should all have doc strings. Please use
-epydoc style docstrings to document parameters and return values.
+Google style docstrings to document parameters and return values.
 You can generate the documentation by running "make doc".
 
 Running the tests
@@ -34,11 +34,12 @@ Like Linux, Git treats filenames as arbitrary bytestrings. There is no prescribe
 encoding for these strings, and although it is fairly common to use UTF-8, any
 raw byte strings are supported.
 
-For this reason, Dulwich internally treats git-based filenames as bytestrings.
-It is up to the Dulwich API user to encode and decode them if necessary. In the
-future, the porcelain may accept unicode strings and convert them to bytestrings
-as necessary on the fly (using sys.getfilesystemencoding()).
+For this reason, the lower levels in Dulwich treat git-based filenames as
+bytestrings. It is up to the Dulwich API user to encode and decode them if
+necessary. The porcelain may accept unicode strings and convert them to
+bytestrings as necessary on the fly (using 'utf-8').
 
+* on-disk filenames: regular strings, or ideally, pathlib.Path instances
 * git-repository related filenames: bytes
 * object sha1 digests (20 bytes long): bytes
 * object sha1 hexdigests (40 bytes long): str (bytestrings on python2, strings

Makefile

@@ -1,7 +1,7 @@
-PYTHON = python -Werror
-PYFLAKES = pyflakes
+PYTHON = python3
+PYFLAKES = $(PYTHON) -m pyflakes
 PEP8 = pep8
-FLAKE8 ?= flake8
+FLAKE8 ?= $(PYTHON) -m flake8
 SETUP = $(PYTHON) setup.py
 TESTRUNNER ?= unittest
 RUNTEST = PYTHONHASHSEED=random PYTHONPATH=$(shell pwd)$(if $(PYTHONPATH),:$(PYTHONPATH),) $(PYTHON) -m $(TESTRUNNER) $(TEST_OPTIONS)
@@ -43,6 +43,9 @@ check-noextensions:: clean
 
 check-all: check check-pypy check-noextensions
 
+typing:
+	mypy dulwich
+
 clean::
 	$(SETUP) clean --all
 	rm -f dulwich/*.so

NEWS

@@ -1,3 +1,47 @@
+0.20.2	2020-06-01
+
+ * Brown bag release to fix uploads of Windows wheels.
+
+0.20.1	2020-06-01
+
+ * Publish binary wheels for: Windows, Linux, Mac OS X.
+   (Jelmer Vernooij, #711, #710, #629)
+
+0.20.0	2020-06-01
+
+ * Drop support for Python 2. (Jelmer Vernooij)
+
+ * Only return files from the loose store that look like git objects.
+   (Nicolas Dandrimont)
+
+ * Ignore agent= capability if sent by client.
+   (Jelmer Vernooij)
+
+ * Don't break when encountering block devices.
+   (Jelmer Vernooij)
+
+ * Decode URL paths in HttpGitClient using utf-8 rather than file system
+   encoding. (Manuel Jacob)
+
+ * Fix pushing from a shallow clone.
+   (Brecht Machiels, #705)
+
+0.19.16	2020-04-17
+
+ * Don't send "deepen None" to server if graph walker
+   supports shallow. (Jelmer Vernooij, #747)
+
+ * Support tweaking the compression level for
+   loose objects through the "core.looseCompression" and
+   "core.compression" settings. (Jelmer Vernooij)
+
+ * Support tweaking the compression level for
+   pack objects through the "core.packCompression" and
+   "core.compression" settings. (Jelmer Vernooij)
+
+ * Add a "dulwich.contrib.diffstat" module.
+   (Kevin Hendricks)
+
 0.19.15	2020-01-26
 
  * Properly handle files that are just executable for the

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.19.15
+Version: 0.20.2
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij
@@ -104,17 +104,20 @@ Description: .. image:: https://travis-ci.org/dulwich/dulwich.png?branch=master
         Supported versions of Python
         ----------------------------
         
-        At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6,
-        3.7 and Pypy.
+        At the moment, Dulwich supports (and is tested on) CPython 3.5, 3.6,
+        3.7, 3.8 and Pypy.
+        
+        The latest release series to support Python 2.x was the 0.19 series. See
+        the 0.19 branch in the Dulwich git repository.
         
 Keywords: git vcs
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.4
 Classifier: Programming Language :: Python :: 3.5
 Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Operating System :: POSIX

README.rst

@@ -93,5 +93,8 @@ file and `list of open issues <https://github.com/dulwich/dulwich/issues>`_.
 Supported versions of Python
 ----------------------------
 
-At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6,
-3.7 and Pypy.
+At the moment, Dulwich supports (and is tested on) CPython 3.5, 3.6,
+3.7, 3.8 and Pypy.
+
+The latest release series to support Python 2.x was the 0.19 series. See
+the 0.19 branch in the Dulwich git repository.

appveyor.yml

@@ -7,14 +7,6 @@ environment:
 
   matrix:
 
-    - PYTHON: "C:\\Python27"
-      PYTHON_VERSION: "2.7.x"
-      PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python27-x64"
-      PYTHON_VERSION: "2.7.x"
-      PYTHON_ARCH: "64"
-
     - PYTHON: "C:\\Python35"
       PYTHON_VERSION: "3.5.x"
       PYTHON_ARCH: "32"
@@ -31,6 +23,14 @@ environment:
       PYTHON_VERSION: "3.6.x"
       PYTHON_ARCH: "64"
 
+    - PYTHON: "C:\\Python37"
+      PYTHON_VERSION: "3.7.x"
+      PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python37-x64"
+      PYTHON_VERSION: "3.7.x"
+      PYTHON_ARCH: "64"
+
 install:
   # If there is a newer build queued for the same PR, cancel this one.
   # The AppVeyor 'rollout builds' option is supposed to serve the same

bin/dul-receive-pack

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3 -u
 # dul-receive-pack - git-receive-pack in python
 # Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
 #

bin/dul-upload-pack

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3 -u
 # dul-upload-pack - git-upload-pack in python
 # Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
 #

bin/dulwich

@@ -1,4 +1,4 @@
-#!/usr/bin/python -u
+#!/usr/bin/python3 -u
 #
 # dulwich - Simple command-line interface to Dulwich
 # Copyright (C) 2008-2011 Jelmer Vernooij <jelmer@jelmer.uk>

dulwich.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.19.15
+Version: 0.20.2
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij
@@ -104,17 +104,20 @@ Description: .. image:: https://travis-ci.org/dulwich/dulwich.png?branch=master
         Supported versions of Python
         ----------------------------
         
-        At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6,
-        3.7 and Pypy.
+        At the moment, Dulwich supports (and is tested on) CPython 3.5, 3.6,
+        3.7, 3.8 and Pypy.
+        
+        The latest release series to support Python 2.x was the 0.19 series. See
+        the 0.19 branch in the Dulwich git repository.
         
 Keywords: git vcs
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.4
 Classifier: Programming Language :: Python :: 3.5
 Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Operating System :: POSIX

dulwich.egg-info/SOURCES.txt

@@ -2,7 +2,6 @@
 .gitignore
 .mailmap
 .testr.conf
-.travis.yml
 AUTHORS
 CONTRIBUTING.rst
 COPYING
@@ -19,6 +18,8 @@ requirements.txt
 setup.cfg
 setup.py
 tox.ini
+.github/workflows/pythonpackage.yml
+.github/workflows/pythonpublish.yml
 bin/dul-receive-pack
 bin/dul-upload-pack
 bin/dulwich
@@ -59,6 +60,7 @@ dulwich/greenthreads.py
 dulwich/hooks.py
 dulwich/ignore.py
 dulwich/index.py
+dulwich/lfs.py
 dulwich/line_ending.py
 dulwich/log_utils.py
 dulwich/lru_cache.py
@@ -85,6 +87,7 @@ dulwich.egg-info/requires.txt
 dulwich.egg-info/top_level.txt
 dulwich/contrib/README.md
 dulwich/contrib/__init__.py
+dulwich/contrib/diffstat.py
 dulwich/contrib/paramiko_vendor.py
 dulwich/contrib/release_robot.py
 dulwich/contrib/swift.py
@@ -104,6 +107,7 @@ dulwich/tests/test_greenthreads.py
 dulwich/tests/test_hooks.py
 dulwich/tests/test_ignore.py
 dulwich/tests/test_index.py
+dulwich/tests/test_lfs.py
 dulwich/tests/test_line_ending.py
 dulwich/tests/test_lru_cache.py
 dulwich/tests/test_mailmap.py

dulwich/__init__.py

@@ -22,4 +22,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 19, 15)
+__version__ = (0, 20, 2)

dulwich/_diff_tree.c

@@ -270,6 +270,11 @@ done:
 	return result;
 }
 
+/* Not all environments define S_ISDIR */
+#if !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR)
+#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
+#endif
+
 static PyObject *py_is_tree(PyObject *self, PyObject *args)
 {
 	PyObject *entry, *mode, *result;

dulwich/_objects.c

@@ -141,6 +141,11 @@ struct tree_item {
 	PyObject *tuple;
 };
 
+/* Not all environments define S_ISDIR */
+#if !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR)
+#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
+#endif
+
 int cmp_tree_item(const void *_a, const void *_b)
 {
 	const struct tree_item *a = _a, *b = _b;

dulwich/client.py

@@ -30,34 +30,33 @@ The Dulwich client supports the following capabilities:
  * quiet
  * report-status
  * delete-refs
+ * shallow
 
 Known capabilities that are not supported:
 
- * shallow
  * no-progress
  * include-tag
 """
 
 from contextlib import closing
 from io import BytesIO, BufferedReader
+import os
 import select
 import socket
 import subprocess
 import sys
 
-try:
-    from urllib import quote as urlquote
-    from urllib import unquote as urlunquote
-except ImportError:
-    from urllib.parse import quote as urlquote
-    from urllib.parse import unquote as urlunquote
-
-try:
-    import urlparse
-except ImportError:
-    import urllib.parse as urlparse
+from urllib.parse import (
+    quote as urlquote,
+    unquote as urlunquote,
+    urlparse,
+    urljoin,
+    urlunsplit,
+    urlunparse,
+    )
 
 import dulwich
+from dulwich.config import get_xdg_config_home_path
 from dulwich.errors import (
     GitProtocolError,
     NotGitRepository,
@@ -72,6 +71,7 @@ from dulwich.protocol import (
     extract_capability_names,
     CAPABILITY_AGENT,
     CAPABILITY_DELETE_REFS,
+    CAPABILITY_INCLUDE_TAG,
     CAPABILITY_MULTI_ACK,
     CAPABILITY_MULTI_ACK_DETAILED,
     CAPABILITY_OFS_DELTA,
@@ -144,7 +144,9 @@ COMMON_CAPABILITIES = [CAPABILITY_OFS_DELTA, CAPABILITY_SIDE_BAND_64K]
 UPLOAD_CAPABILITIES = ([CAPABILITY_THIN_PACK, CAPABILITY_MULTI_ACK,
                         CAPABILITY_MULTI_ACK_DETAILED, CAPABILITY_SHALLOW]
                        + COMMON_CAPABILITIES)
-RECEIVE_CAPABILITIES = [CAPABILITY_REPORT_STATUS] + COMMON_CAPABILITIES
+RECEIVE_CAPABILITIES = (
+    [CAPABILITY_REPORT_STATUS, CAPABILITY_DELETE_REFS]
+    + COMMON_CAPABILITIES)
 
 
 class ReportStatusParser(object):
@@ -310,13 +312,16 @@ def _read_shallow_updates(proto):
 class GitClient(object):
     """Git smart server client."""
 
-    def __init__(self, thin_packs=True, report_activity=None, quiet=False):
+    def __init__(self, thin_packs=True, report_activity=None, quiet=False,
+                 include_tags=False):
         """Create a new GitClient instance.
 
         Args:
           thin_packs: Whether or not thin packs should be retrieved
           report_activity: Optional callback for reporting transport
             activity.
+          include_tags: send annotated tags when sending the objects they point
+            to
         """
         self._report_activity = report_activity
         self._report_status_parser = None
@@ -328,6 +333,8 @@ class GitClient(object):
             self._send_capabilities.add(CAPABILITY_QUIET)
         if not thin_packs:
             self._fetch_capabilities.remove(CAPABILITY_THIN_PACK)
+        if include_tags:
+            self._fetch_capabilities.add(CAPABILITY_INCLUDE_TAG)
 
     def get_url(self, path):
         """Retrieves full url to given path.
@@ -346,7 +353,7 @@ class GitClient(object):
         """Create an instance of this client from a urlparse.parsed object.
 
         Args:
-          parsedurl: Result of urlparse.urlparse()
+          parsedurl: Result of urlparse()
 
         Returns:
           A `GitClient` object
@@ -549,7 +556,7 @@ class GitClient(object):
                 else:
                     proto.write_pkt_line(
                         old_sha1 + b' ' + new_sha1 + b' ' + refname + b'\0' +
-                        b' '.join(capabilities))
+                        b' '.join(sorted(capabilities)))
                     sent_capabilities = True
             if new_sha1 not in have and new_sha1 != ZERO_SHA:
                 want.append(new_sha1)
@@ -629,7 +636,7 @@ class GitClient(object):
         """
         assert isinstance(wants, list) and isinstance(wants[0], bytes)
         proto.write_pkt_line(COMMAND_WANT + b' ' + wants[0] + b' ' +
-                             b' '.join(capabilities) + b'\n')
+                             b' '.join(sorted(capabilities)) + b'\n')
         for want in wants[1:]:
             proto.write_pkt_line(COMMAND_WANT + b' ' + want + b'\n')
         if depth not in (0, None) or getattr(graph_walker, 'shallow', None):
@@ -639,8 +646,9 @@ class GitClient(object):
                     "depth")
             for sha in graph_walker.shallow:
                 proto.write_pkt_line(COMMAND_SHALLOW + b' ' + sha + b'\n')
-            proto.write_pkt_line(COMMAND_DEEPEN + b' ' +
-                                 str(depth).encode('ascii') + b'\n')
+            if depth is not None:
+                proto.write_pkt_line(COMMAND_DEEPEN + b' ' +
+                                     str(depth).encode('ascii') + b'\n')
             proto.write_pkt_line(None)
             if can_read is not None:
                 (new_shallow, new_unshallow) = _read_shallow_updates(proto)
@@ -731,11 +739,11 @@ def check_wants(wants, refs):
 def remote_error_from_stderr(stderr):
     if stderr is None:
         return HangupException()
-    for l in stderr.readlines():
-        if l.startswith(b'ERROR: '):
+    for line in stderr.readlines():
+        if line.startswith(b'ERROR: '):
             return GitProtocolError(
-                l[len(b'ERROR: '):].decode('utf-8', 'replace'))
-        return GitProtocolError(l.decode('utf-8', 'replace'))
+                line[len(b'ERROR: '):].decode('utf-8', 'replace'))
+        return GitProtocolError(line.decode('utf-8', 'replace'))
     return HangupException()
 
 
@@ -964,7 +972,7 @@ class TCPGitClient(TraditionalGitClient):
         netloc = self._host
         if self._port is not None and self._port != TCP_GIT_PORT:
             netloc += ":%d" % self._port
-        return urlparse.urlunsplit(("git", netloc, path, '', ''))
+        return urlunsplit(("git", netloc, path, '', ''))
 
     def _connect(self, cmd, path):
         if not isinstance(cmd, bytes):
@@ -1013,10 +1021,7 @@ class SubprocessWrapper(object):
 
     def __init__(self, proc):
         self.proc = proc
-        if sys.version_info[0] == 2:
-            self.read = proc.stdout.read
-        else:
-            self.read = BufferedReader(proc.stdout).read
+        self.read = BufferedReader(proc.stdout).read
         self.write = proc.stdin.write
 
     @property
@@ -1094,7 +1099,7 @@ class LocalGitClient(GitClient):
         # Ignore the thin_packs argument
 
     def get_url(self, path):
-        return urlparse.urlunsplit(('file', '', path, '', ''))
+        return urlunsplit(('file', '', path, '', ''))
 
     @classmethod
     def from_parsedurl(cls, parsedurl, **kwargs):
@@ -1104,7 +1109,7 @@ class LocalGitClient(GitClient):
     def _open_repo(cls, path):
         from dulwich.repo import Repo
         if not isinstance(path, str):
-            path = path.decode(sys.getfilesystemencoding())
+            path = os.fsdecode(path)
         return closing(Repo(path))
 
     def send_pack(self, path, update_refs, generate_pack_data,
@@ -1116,7 +1121,6 @@ class LocalGitClient(GitClient):
           update_refs: Function to determine changes to remote refs.
         Receive dict with existing remote refs, returns dict with
         changed refs (name -> sha, where sha=ZERO_SHA for deletions)
-          generate_pack_data: Function that can return a tuple
         with number of items and pack data to upload.
           progress: Optional progress function
 
@@ -1379,7 +1383,7 @@ class SSHGitClient(TraditionalGitClient):
         if self.username is not None:
             netloc = urlquote(self.username, '@/:') + "@" + netloc
 
-        return urlparse.urlunsplit(('ssh', netloc, path, '', ''))
+        return urlunsplit(('ssh', netloc, path, '', ''))
 
     @classmethod
     def from_parsedurl(cls, parsedurl, **kwargs):
@@ -1419,7 +1423,8 @@ def default_user_agent_string():
     return "git/dulwich/%s" % ".".join([str(x) for x in dulwich.__version__])
 
 
-def default_urllib3_manager(config, **override_kwargs):
+def default_urllib3_manager(config, pool_manager_cls=None,
+                            proxy_manager_cls=None, **override_kwargs):
     """Return `urllib3` connection pool manager.
 
     Honour detected proxy configurations.
@@ -1429,8 +1434,9 @@ def default_urllib3_manager(config, **override_kwargs):
       kwargs: Additional arguments for urllib3.ProxyManager
 
     Returns:
-      urllib3.ProxyManager` instance for proxy configurations,
-      `urllib3.PoolManager` otherwise.
+      `pool_manager_cls` (defaults to `urllib3.ProxyManager`) instance for
+      proxy configurations, `proxy_manager_cls` (defaults to
+      `urllib3.PoolManager`) instance otherwise.
 
     """
     proxy_server = user_agent = None
@@ -1487,14 +1493,17 @@ def default_urllib3_manager(config, **override_kwargs):
     import urllib3
 
     if proxy_server is not None:
+        if proxy_manager_cls is None:
+            proxy_manager_cls = urllib3.ProxyManager
         # `urllib3` requires a `str` object in both Python 2 and 3, while
         # `ConfigDict` coerces entries to `bytes` on Python 3. Compensate.
         if not isinstance(proxy_server, str):
             proxy_server = proxy_server.decode()
-        manager = urllib3.ProxyManager(proxy_server, headers=headers,
-                                       **kwargs)
+        manager = proxy_manager_cls(proxy_server, headers=headers, **kwargs)
     else:
-        manager = urllib3.PoolManager(headers=headers, **kwargs)
+        if pool_manager_cls is None:
+            pool_manager_cls = urllib3.PoolManager
+        manager = pool_manager_cls(headers=headers, **kwargs)
 
     return manager
 
@@ -1540,7 +1549,7 @@ class HttpGitClient(GitClient):
         if parsedurl.username:
             netloc = "%s@%s" % (parsedurl.username, netloc)
         parsedurl = parsedurl._replace(netloc=netloc)
-        return cls(urlparse.urlunparse(parsedurl), **kwargs)
+        return cls(urlunparse(parsedurl), **kwargs)
 
     def __repr__(self):
         return "%s(%r, dumb=%r)" % (
@@ -1548,11 +1557,10 @@ class HttpGitClient(GitClient):
 
     def _get_url(self, path):
         if not isinstance(path, str):
-            # TODO(jelmer): this is unrelated to the local filesystem;
-            # This is not necessarily the right encoding to decode the path
-            # with.
-            path = path.decode(sys.getfilesystemencoding())
-        return urlparse.urljoin(self._base_url, path).rstrip("/") + "/"
+            # urllib3.util.url._encode_invalid_chars() converts the path back
+            # to bytes using the utf-8 codec.
+            path = path.decode('utf-8')
+        return urljoin(self._base_url, path).rstrip("/") + "/"
 
     def _http_request(self, url, headers=None, data=None,
                       allow_compression=False):
@@ -1600,9 +1608,14 @@ class HttpGitClient(GitClient):
         read = BytesIO(resp.data).read
 
         resp.content_type = resp.getheader("Content-Type")
-        resp_url = resp.geturl()
-        resp.redirect_location = resp_url if resp_url != url else ''
-
+        # Check if geturl() is available (urllib3 version >= 1.23)
+        try:
+            resp_url = resp.geturl()
+        except AttributeError:
+            # get_redirect_location() is available for urllib3 >= 1.1
+            resp.redirect_location = resp.get_redirect_location()
+        else:
+            resp.redirect_location = resp_url if resp_url != url else ''
         return resp, read
 
     def _discover_references(self, service, base_url):
@@ -1611,7 +1624,7 @@ class HttpGitClient(GitClient):
         headers = {"Accept": "*/*"}
         if self.dumb is not True:
             tail += "?service=%s" % service.decode('ascii')
-        url = urlparse.urljoin(base_url, tail)
+        url = urljoin(base_url, tail)
         resp, read = self._http_request(url, headers, allow_compression=True)
 
         if resp.redirect_location:
@@ -1643,7 +1656,7 @@ class HttpGitClient(GitClient):
 
     def _smart_request(self, service, url, data):
         assert url[-1] == "/"
-        url = urlparse.urljoin(url, service)
+        url = urljoin(url, service)
         result_content_type = "application/x-%s-result" % service
         headers = {
             "Content-Type": "application/x-%s-request" % service,
@@ -1663,7 +1676,7 @@ class HttpGitClient(GitClient):
         Args:
           path: Repository path (as bytestring)
           update_refs: Function to determine changes to remote refs.
-        Receive dict with existing remote refs, returns dict with
+        Receives dict with existing remote refs, returns dict with
         changed refs (name -> sha, where sha=ZERO_SHA for deletions)
           generate_pack_data: Function that can return a tuple
         with number of elements and pack data to upload.
@@ -1788,7 +1801,7 @@ def get_transport_and_path_from_url(url, config=None, **kwargs):
       Tuple with client instance and relative path.
 
     """
-    parsed = urlparse.urlparse(url)
+    parsed = urlparse(url)
     if parsed.scheme == 'git':
         return (TCPGitClient.from_parsedurl(parsed, **kwargs),
                 parsed.path)
@@ -1856,3 +1869,25 @@ def get_transport_and_path(location, **kwargs):
         return default_local_git_client_cls(**kwargs), location
     else:
         return SSHGitClient(hostname, username=username, **kwargs), path
+
+
+DEFAULT_GIT_CREDENTIALS_PATHS = [
+    os.path.expanduser('~/.git-credentials'),
+    get_xdg_config_home_path('git', 'credentials')]
+
+
+def get_credentials_from_store(scheme, hostname, username=None,
+                               fnames=DEFAULT_GIT_CREDENTIALS_PATHS):
+    for fname in fnames:
+        try:
+            with open(fname, 'rb') as f:
+                for line in f:
+                    parsed_line = urlparse(line)
+                    if (parsed_line.scheme == scheme and
+                            parsed_line.hostname == hostname and
+                            (username is None or
+                                parsed_line.username == username)):
+                        return parsed_line.username, parsed_line.password
+        except FileNotFoundError:
+            # If the file doesn't exist, try the next one.
+            continue

dulwich/config.py

@@ -26,7 +26,6 @@ TODO:
    subsections
 """
 
-import errno
 import os
 import sys
 
@@ -487,6 +486,13 @@ class ConfigFile(ConfigDict):
                 f.write(b"\t" + key + b" = " + value + b"\n")
 
 
+def get_xdg_config_home_path(*path_segments):
+    xdg_config_home = os.environ.get(
+        "XDG_CONFIG_HOME", os.path.expanduser("~/.config/"),
+    )
+    return os.path.join(xdg_config_home, *path_segments)
+
+
 class StackedConfig(Config):
     """Configuration which reads from multiple config files.."""
 
@@ -509,11 +515,7 @@ class StackedConfig(Config):
         """
         paths = []
         paths.append(os.path.expanduser("~/.gitconfig"))
-
-        xdg_config_home = os.environ.get(
-            "XDG_CONFIG_HOME", os.path.expanduser("~/.config/"),
-        )
-        paths.append(os.path.join(xdg_config_home, "git", "config"))
+        paths.append(get_xdg_config_home_path("git", "config"))
 
         if "GIT_CONFIG_NOSYSTEM" not in os.environ:
             paths.append("/etc/gitconfig")
@@ -522,11 +524,8 @@ class StackedConfig(Config):
         for path in paths:
             try:
                 cf = ConfigFile.from_path(path)
-            except (IOError, OSError) as e:
-                if e.errno != errno.ENOENT:
-                    raise
-                else:
-                    continue
+            except FileNotFoundError:
+                continue
             backends.append(cf)
         return backends
 

dulwich/contrib/diffstat.py

@@ -0,0 +1,340 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
+
+# Copyright (c) 2020 Kevin B. Hendricks, Stratford Ontario Canada
+# All rights reserved.
+#
+# This diffstat code was extracted and heavily modified from:
+#
+#  https://github.com/techtonik/python-patch
+#      Under the following license:
+#
+#  Patch utility to apply unified diffs
+#  Brute-force line-by-line non-recursive parsing
+#
+# Copyright (c) 2008-2016 anatoly techtonik
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import re
+
+# only needs to detect git style diffs as this is for
+# use with dulwich
+
+_git_header_name = re.compile(br'diff --git a/(.*) b/(.*)')
+
+_GIT_HEADER_START = b'diff --git a/'
+_GIT_BINARY_START = b'Binary file'
+_GIT_RENAMEFROM_START = b'rename from'
+_GIT_RENAMETO_START = b'rename to'
+_GIT_CHUNK_START = b'@@'
+_GIT_ADDED_START = b'+'
+_GIT_DELETED_START = b'-'
+_GIT_UNCHANGED_START = b' '
+
+# emulate original full Patch class by just extracting
+# filename and minimal chunk added/deleted information to
+# properly interface with diffstat routine
+
+
+def _parse_patch(lines):
+    """An internal routine to parse a git style diff or patch to generate
+       diff stats
+    Args:
+      lines: list of byte strings "lines" from the diff to be parsed
+    Returns: A tuple (names, nametypes, counts) of three lists:
+             names = list of repo relative file paths
+             nametypes - list of booolean values indicating if file
+                         is binary (True means binary file)
+             counts = list of tuples of (added, deleted) counts for that file
+    """
+    names = []
+    nametypes = []
+    counts = []
+    in_patch_chunk = in_git_header = binaryfile = False
+    currentfile = None
+    added = deleted = 0
+    for line in lines:
+        if line.startswith(_GIT_HEADER_START):
+            if currentfile is not None:
+                names.append(currentfile)
+                nametypes.append(binaryfile)
+                counts.append((added, deleted))
+            currentfile = _git_header_name.search(line).group(2)
+            binaryfile = False
+            added = deleted = 0
+            in_git_header = True
+            in_patch_chunk = False
+        elif line.startswith(_GIT_BINARY_START) and in_git_header:
+            binaryfile = True
+            in_git_header = False
+        elif line.startswith(_GIT_RENAMEFROM_START) and in_git_header:
+            currentfile = line[12:]
+        elif line.startswith(_GIT_RENAMETO_START) and in_git_header:
+            currentfile += b' => %s' % line[10:]
+        elif line.startswith(_GIT_CHUNK_START) and \
+                (in_patch_chunk or in_git_header):
+            in_patch_chunk = True
+            in_git_header = False
+        elif line.startswith(_GIT_ADDED_START) and in_patch_chunk:
+            added += 1
+        elif line.startswith(_GIT_DELETED_START) and in_patch_chunk:
+            deleted += 1
+        elif not line.startswith(_GIT_UNCHANGED_START) and in_patch_chunk:
+            in_patch_chunk = False
+    # handle end of input
+    if currentfile is not None:
+        names.append(currentfile)
+        nametypes.append(binaryfile)
+        counts.append((added, deleted))
+    return names, nametypes, counts
+
+
+# note must all done using bytes not string because on linux filenames
+# may not be encodable even to utf-8
+def diffstat(lines, max_width=80):
+    """Generate summary statistics from a git style diff ala
+       (git diff tag1 tag2 --stat)
+    Args:
+      lines: list of byte string "lines" from the diff to be parsed
+      max_width: maximum line length for generating the summary
+                 statistics (default 80)
+    Returns: A byte string that lists the changed files with change
+             counts and histogram
+    """
+    names, nametypes, counts = _parse_patch(lines)
+    insert = []
+    delete = []
+    namelen = 0
+    maxdiff = 0  # max changes for any file used for histogram width calc
+    for i, filename in enumerate(names):
+        i, d = counts[i]
+        insert.append(i)
+        delete.append(d)
+        namelen = max(namelen, len(filename))
+        maxdiff = max(maxdiff, i+d)
+    output = b''
+    statlen = len(str(maxdiff))  # stats column width
+    for i, n in enumerate(names):
+        binaryfile = nametypes[i]
+        # %-19s | %-4d %s
+        # note b'%d' % namelen is not supported until Python 3.5
+        # To convert an int to a format width specifier for byte
+        # strings use str(namelen).encode('ascii')
+        format = b' %-' + str(namelen).encode('ascii') + \
+            b's | %' + str(statlen).encode('ascii') + b's %s\n'
+        binformat = b' %-' + str(namelen).encode('ascii') + b's | %s\n'
+        if not binaryfile:
+            hist = b''
+            # -- calculating histogram --
+            width = len(format % (b'', b'', b''))
+            histwidth = max(2, max_width - width)
+            if maxdiff < histwidth:
+                hist = b'+'*insert[i] + b'-'*delete[i]
+            else:
+                iratio = (float(insert[i]) / maxdiff) * histwidth
+                dratio = (float(delete[i]) / maxdiff) * histwidth
+                iwidth = dwidth = 0
+                # make sure every entry that had actual insertions gets
+                # at least one +
+                if insert[i] > 0:
+                    iwidth = int(iratio)
+                    if iwidth == 0 and 0 < iratio < 1:
+                        iwidth = 1
+                # make sure every entry that had actual deletions gets
+                # at least one -
+                if delete[i] > 0:
+                    dwidth = int(dratio)
+                    if dwidth == 0 and 0 < dratio < 1:
+                        dwidth = 1
+                hist = b'+'*int(iwidth) + b'-'*int(dwidth)
+            output += (format % (bytes(names[i]),
+                                 str(insert[i] + delete[i]).encode('ascii'),
+                                 hist))
+        else:
+            output += (binformat % (bytes(names[i]), b'Bin'))
+
+    output += (b' %d files changed, %d insertions(+), %d deletions(-)'
+               % (len(names), sum(insert), sum(delete)))
+    return output
+
+
+def main():
+    argv = sys.argv
+    # allow diffstat.py to also be used from the comand line
+    if len(sys.argv) > 1:
+        diffpath = argv[1]
+        data = b''
+        with open(diffpath, 'rb') as f:
+            data = f.read()
+        lines = data.split(b'\n')
+        result = diffstat(lines)
+        print(result.decode('utf-8'))
+        return 0
+
+    # if no path argument to a diff file is passed in, run
+    # a self test. The test case includes tricky things like
+    # a diff of diff, binary files, renames with futher changes
+    # added files and removed files.
+    # All extracted from Sigil-Ebook/Sigil's github repo with
+    # full permission to use under this license.
+    selftest = b"""
+diff --git a/docs/qt512.7_remove_bad_workaround.patch b/docs/qt512.7_remove_bad_workaround.patch
+new file mode 100644
+index 00000000..64e34192
+--- /dev/null
++++ b/docs/qt512.7_remove_bad_workaround.patch
+@@ -0,0 +1,15 @@
++--- qtbase/src/gui/kernel/qwindow.cpp.orig     2019-12-12 09:15:59.000000000 -0500
+++++ qtbase/src/gui/kernel/qwindow.cpp  2020-01-10 10:36:53.000000000 -0500
++@@ -218,12 +218,6 @@
++     QGuiApplicationPrivate::window_list.removeAll(this);
++     if (!QGuiApplicationPrivate::is_app_closing)
++         QGuiApplicationPrivate::instance()->modalWindowList.removeOne(this);
++-
++-    // focus_window is normally cleared in destroy(), but the window may in
++-    // some cases end up becoming the focus window again. Clear it again
++-    // here as a workaround. See QTBUG-75326.
++-    if (QGuiApplicationPrivate::focus_window == this)
++-        QGuiApplicationPrivate::focus_window = 0;
++ }
++
++ void QWindowPrivate::init(QScreen *targetScreen)
+diff --git a/docs/testplugin_v017.zip b/docs/testplugin_v017.zip
+new file mode 100644
+index 00000000..a4cf4c4c
+Binary files /dev/null and b/docs/testplugin_v017.zip differ
+diff --git a/ci_scripts/macgddeploy.py b/ci_scripts/gddeploy.py
+similarity index 73%
+rename from ci_scripts/macgddeploy.py
+rename to ci_scripts/gddeploy.py
+index a512d075..f9dacd33 100644
+--- a/ci_scripts/macgddeploy.py
++++ b/ci_scripts/gddeploy.py
+@@ -1,19 +1,32 @@
+ #!/usr/bin/env python3
+
+ import os
++import sys
+ import subprocess
+ import datetime
+ import shutil
++import glob
+
+ gparent = os.path.expandvars('$GDRIVE_DIR')
+ grefresh_token = os.path.expandvars('$GDRIVE_REFRESH_TOKEN')
+
+-travis_branch = os.path.expandvars('$TRAVIS_BRANCH')
+-travis_commit = os.path.expandvars('$TRAVIS_COMMIT')
+-travis_build_number = os.path.expandvars('$TRAVIS_BUILD_NUMBER')
++if sys.platform.lower().startswith('darwin'):
++    travis_branch = os.path.expandvars('$TRAVIS_BRANCH')
++    travis_commit = os.path.expandvars('$TRAVIS_COMMIT')
++    travis_build_number = os.path.expandvars('$TRAVIS_BUILD_NUMBER')
++
++    origfilename = './bin/Sigil.tar.xz'
++    newfilename = './bin/Sigil-{}-{}-build_num-{}.tar.xz'.format(travis_branch, travis_commit[:7],travis_build_numbe\
+r)
++else:
++    appveyor_branch = os.path.expandvars('$APPVEYOR_REPO_BRANCH')
++    appveyor_commit = os.path.expandvars('$APPVEYOR_REPO_COMMIT')
++    appveyor_build_number = os.path.expandvars('$APPVEYOR_BUILD_NUMBER')
++    names = glob.glob('.\\installer\\Sigil-*-Setup.exe')
++    if not names:
++        exit(1)
++    origfilename = names[0]
++    newfilename = '.\\installer\\Sigil-{}-{}-build_num-{}-Setup.exe'.format(appveyor_branch, appveyor_commit[:7], ap\
+pveyor_build_number)
+
+-origfilename = './bin/Sigil.tar.xz'
+-newfilename = './bin/Sigil-{}-{}-build_num-{}.tar.xz'.format(travis_branch, travis_commit[:7],travis_build_number)
+ shutil.copy2(origfilename, newfilename)
+
+ folder_name = datetime.date.today()
+diff --git a/docs/qt512.6_backport_009abcd_fix.patch b/docs/qt512.6_backport_009abcd_fix.patch
+deleted file mode 100644
+index f4724347..00000000
+--- a/docs/qt512.6_backport_009abcd_fix.patch
++++ /dev/null
+@@ -1,26 +0,0 @@
+---- qtbase/src/widgets/kernel/qwidget.cpp.orig 2019-11-08 10:57:07.000000000 -0500
+-+++ qtbase/src/widgets/kernel/qwidget.cpp      2019-12-11 12:32:24.000000000 -0500
+-@@ -8934,6 +8934,23 @@
+-         }
+-     }
+-     switch (event->type()) {
+-+    case QEvent::PlatformSurface: {
+-+        // Sync up QWidget's view of whether or not the widget has been created
+-+        switch (static_cast<QPlatformSurfaceEvent*>(event)->surfaceEventType()) {
+-+        case QPlatformSurfaceEvent::SurfaceCreated:
+-+            if (!testAttribute(Qt::WA_WState_Created))
+-+                create();
+-+            break;
+-+        case QPlatformSurfaceEvent::SurfaceAboutToBeDestroyed:
+-+            if (testAttribute(Qt::WA_WState_Created)) {
+-+                // Child windows have already been destroyed by QWindow,
+-+                // so we skip them here.
+-+                destroy(false, false);
+-+            }
+-+            break;
+-+        }
+-+        break;
+-+    }
+-     case QEvent::MouseMove:
+-         mouseMoveEvent((QMouseEvent*)event);
+-         break;
+diff --git a/docs/Building_Sigil_On_MacOSX.txt b/docs/Building_Sigil_On_MacOSX.txt
+index 3b41fd80..64914c78 100644
+--- a/docs/Building_Sigil_On_MacOSX.txt
++++ b/docs/Building_Sigil_On_MacOSX.txt
+@@ -113,7 +113,7 @@ install_name_tool -add_rpath @loader_path/../../Frameworks ./bin/Sigil.app/Content
+ 
+ # To test if the newly bundled python 3 version of Sigil is working properly ypou can do the following:
+ 
+-1. download testplugin_v014.zip from https://github.com/Sigil-Ebook/Sigil/tree/master/docs
++1. download testplugin_v017.zip from https://github.com/Sigil-Ebook/Sigil/tree/master/docs
+ 2. open Sigil.app to the normal nearly blank template epub it generates when opened
+ 3. use Plugins->Manage Plugins menu and make sure the "Use Bundled Python" checkbox is checked
+ 4. use the "Add Plugin" button to navigate to and add testplugin.zip and then hit "Okay" to exit the Manage Plugins Dialog
+"""     # noqa: E501 W293
+
+    testoutput = b""" docs/qt512.7_remove_bad_workaround.patch            | 15 ++++++++++++
+ docs/testplugin_v017.zip                            | Bin
+ ci_scripts/macgddeploy.py => ci_scripts/gddeploy.py |  0 
+ docs/qt512.6_backport_009abcd_fix.patch             | 26 ---------------------
+ docs/Building_Sigil_On_MacOSX.txt                   |  2 +-
+ 5 files changed, 16 insertions(+), 27 deletions(-)"""  # noqa: W291
+
+    # return 0 on success otherwise return -1
+    result = diffstat(selftest.split(b'\n'))
+    if result == testoutput:
+        print("self test passed")
+        return 0
+    print("self test failed")
+    print("Received:")
+    print(result.decode('utf-8'))
+    print("Expected:")
+    print(testoutput.decode('utf-8'))
+    return -1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

dulwich/contrib/swift.py

@@ -32,16 +32,10 @@ import zlib
 import tempfile
 import posixpath
 
-try:
-    import urlparse
-except ImportError:
-    import urllib.parse as urlparse
+import urllib.parse as urlparse
 
 from io import BytesIO
-try:
-    from ConfigParser import ConfigParser
-except ImportError:
-    from configparser import ConfigParser
+from configparser import ConfigParser
 from geventhttpclient import HTTPClient
 
 from dulwich.greenthreads import (
@@ -92,12 +86,7 @@ from dulwich.server import (
     TCPGitServer,
     )
 
-try:
-    from simplejson import loads as json_loads
-    from simplejson import dumps as json_dumps
-except ImportError:
-    from json import loads as json_loads
-    from json import dumps as json_dumps
+import json
 
 import sys
 
@@ -223,7 +212,7 @@ def pack_info_create(pack_data, pack_index):
         # Tag
         elif obj.type_num == Tag.type_num:
             info[obj.id] = (obj.type_num, obj.object[1])
-    return zlib.compress(json_dumps(info))
+    return zlib.compress(json.dumps(info))
 
 
 def load_pack_info(filename, scon=None, file=None):
@@ -234,7 +223,7 @@ def load_pack_info(filename, scon=None, file=None):
     if not f:
         return None
     try:
-        return json_loads(zlib.decompress(f.read()))
+        return json.loads(zlib.decompress(f.read()))
     finally:
         f.close()
 
@@ -323,7 +312,7 @@ class SwiftConnector(object):
                                  'password': self.password,
                              },
                              'tenantName': self.tenant}
-        auth_json = json_dumps(auth_dict)
+        auth_json = json.dumps(auth_dict)
         headers = {'Content-Type': 'application/json'}
         auth_httpclient = HTTPClient.from_url(
             self.auth_url,
@@ -343,7 +332,7 @@ class SwiftConnector(object):
                                  % (str(auth_httpclient.get_base_url()) +
                                     path, ret.status_code,
                                     str(ret.items())))
-        auth_ret_json = json_loads(ret.read())
+        auth_ret_json = json.loads(ret.read())
         token = auth_ret_json['access']['token']['id']
         catalogs = auth_ret_json['access']['serviceCatalog']
         object_store = [o_store for o_store in catalogs if
@@ -393,7 +382,7 @@ class SwiftConnector(object):
             raise SwiftException('GET request failed with error code %s'
                                  % ret.status_code)
         content = ret.read()
-        return json_loads(content)
+        return json.loads(content)
 
     def get_object_stat(self, name):
         """Retrieve object stat
@@ -816,8 +805,7 @@ class SwiftObjectStore(PackBasedObjectStore):
         entries.sort()
         pack_base_name = posixpath.join(
             self.pack_dir,
-            'pack-' + iter_sha1(e[0] for e in entries).decode(
-                sys.getfilesystemencoding()))
+            'pack-' + os.fsdecode(iter_sha1(e[0] for e in entries)))
         self.scon.put_object(pack_base_name + '.pack', f)
 
         # Write the index.

dulwich/contrib/test_swift.py

@@ -25,13 +25,8 @@
 import posixpath
 
 from time import time
-from io import BytesIO
-try:
-    from StringIO import StringIO
-except ImportError:
-    from io import StringIO
+from io import BytesIO, StringIO
 
-import sys
 from unittest import skipIf
 
 from dulwich.tests import (
@@ -40,9 +35,6 @@ from dulwich.tests import (
 from dulwich.tests.test_object_store import (
     ObjectStoreTests,
     )
-from dulwich.tests.utils import (
-    build_pack,
-    )
 from dulwich.objects import (
     Blob,
     Commit,
@@ -50,17 +42,8 @@ from dulwich.objects import (
     Tag,
     parse_timezone,
     )
-from dulwich.pack import (
-    REF_DELTA,
-    write_pack_index_v2,
-    PackData,
-    load_pack_index_file,
-    )
 
-try:
-    from simplejson import dumps as json_dumps
-except ImportError:
-    from json import dumps as json_dumps
+import json
 
 missing_libs = []
 
@@ -81,8 +64,6 @@ except ImportError:
 
 skipmsg = "Required libraries are not installed (%r)" % missing_libs
 
-skipIfPY3 = skipIf(sys.version_info[0] == 3,
-                   "SWIFT module not yet ported to python3.")
 
 if not missing_libs:
     from dulwich.contrib import swift
@@ -163,7 +144,7 @@ def fake_auth_request_v2(*args, **kwargs):
                        ]
                        }
             }
-    ret = Response(status=200, content=json_dumps(resp))
+    ret = Response(status=200, content=json.dumps(resp))
     return ret
 
 
@@ -259,128 +240,6 @@ class FakeSwiftConnector(object):
         return {'content-length': len(self.store[name])}
 
 
-@skipIf(missing_libs, skipmsg)
-@skipIfPY3
-class TestSwiftObjectStore(TestCase):
-
-    def setUp(self):
-        super(TestSwiftObjectStore, self).setUp()
-        self.conf = swift.load_conf(file=StringIO(config_file %
-                                                  def_config_file))
-        self.fsc = FakeSwiftConnector('fakerepo', conf=self.conf)
-
-    def _put_pack(self, sos, commit_amount=1, marker='Default'):
-        odata = create_commits(length=commit_amount, marker=marker)
-        data = [(d.type_num, d.as_raw_string()) for d in odata]
-        f = BytesIO()
-        build_pack(f, data, store=sos)
-        sos.add_thin_pack(f.read, None)
-        return odata
-
-    def test_load_packs(self):
-        store = {'fakerepo/objects/pack/pack-'+'1'*40+'.idx': '',
-                 'fakerepo/objects/pack/pack-'+'1'*40+'.pack': '',
-                 'fakerepo/objects/pack/pack-'+'1'*40+'.info': '',
-                 'fakerepo/objects/pack/pack-'+'2'*40+'.idx': '',
-                 'fakerepo/objects/pack/pack-'+'2'*40+'.pack': '',
-                 'fakerepo/objects/pack/pack-'+'2'*40+'.info': ''}
-        fsc = FakeSwiftConnector('fakerepo', conf=self.conf, store=store)
-        sos = swift.SwiftObjectStore(fsc)
-        packs = sos.packs
-        self.assertEqual(len(packs), 2)
-        for pack in packs:
-            self.assertTrue(isinstance(pack, swift.SwiftPack))
-
-    def test_add_thin_pack(self):
-        sos = swift.SwiftObjectStore(self.fsc)
-        self._put_pack(sos, 1, 'Default')
-        self.assertEqual(len(self.fsc.store), 3)
-
-    def test_find_missing_objects(self):
-        commit_amount = 3
-        sos = swift.SwiftObjectStore(self.fsc)
-        odata = self._put_pack(sos, commit_amount, 'Default')
-        head = odata[-1].id
-        i = sos.iter_shas(sos.find_missing_objects([],
-                                                   [head, ],
-                                                   progress=None,
-                                                   get_tagged=None))
-        self.assertEqual(len(i), commit_amount * 3)
-        shas = [d.id for d in odata]
-        for sha, path in i:
-            self.assertIn(sha.id, shas)
-
-    def test_find_missing_objects_with_tag(self):
-        commit_amount = 3
-        sos = swift.SwiftObjectStore(self.fsc)
-        odata = self._put_pack(sos, commit_amount, 'Default')
-        head = odata[-1].id
-        peeled_sha = dict([(sha.object[1], sha.id)
-                           for sha in odata if isinstance(sha, Tag)])
-
-        def get_tagged():
-            return peeled_sha
-        i = sos.iter_shas(sos.find_missing_objects([],
-                                                   [head, ],
-                                                   progress=None,
-                                                   get_tagged=get_tagged))
-        self.assertEqual(len(i), commit_amount * 4)
-        shas = [d.id for d in odata]
-        for sha, path in i:
-            self.assertIn(sha.id, shas)
-
-    def test_find_missing_objects_with_common(self):
-        commit_amount = 3
-        sos = swift.SwiftObjectStore(self.fsc)
-        odata = self._put_pack(sos, commit_amount, 'Default')
-        head = odata[-1].id
-        have = odata[7].id
-        i = sos.iter_shas(sos.find_missing_objects([have, ],
-                                                   [head, ],
-                                                   progress=None,
-                                                   get_tagged=None))
-        self.assertEqual(len(i), 3)
-
-    def test_find_missing_objects_multiple_packs(self):
-        sos = swift.SwiftObjectStore(self.fsc)
-        commit_amount_a = 3
-        odataa = self._put_pack(sos, commit_amount_a, 'Default1')
-        heada = odataa[-1].id
-        commit_amount_b = 2
-        odatab = self._put_pack(sos, commit_amount_b, 'Default2')
-        headb = odatab[-1].id
-        i = sos.iter_shas(sos.find_missing_objects([],
-                                                   [heada, headb],
-                                                   progress=None,
-                                                   get_tagged=None))
-        self.assertEqual(len(self.fsc.store), 6)
-        self.assertEqual(len(i),
-                         commit_amount_a * 3 +
-                         commit_amount_b * 3)
-        shas = [d.id for d in odataa]
-        shas.extend([d.id for d in odatab])
-        for sha, path in i:
-            self.assertIn(sha.id, shas)
-
-    def test_add_thin_pack_ext_ref(self):
-        sos = swift.SwiftObjectStore(self.fsc)
-        odata = self._put_pack(sos, 1, 'Default1')
-        ref_blob_content = odata[0].as_raw_string()
-        ref_blob_id = odata[0].id
-        new_blob = Blob.from_string(ref_blob_content.replace('blob',
-                                                             'yummy blob'))
-        blob, tree, tag, cmt = \
-            create_commit([], marker='Default2', blob=new_blob)
-        data = [(REF_DELTA, (ref_blob_id, blob.as_raw_string())),
-                (tree.type_num, tree.as_raw_string()),
-                (cmt.type_num, cmt.as_raw_string()),
-                (tag.type_num, tag.as_raw_string())]
-        f = BytesIO()
-        build_pack(f, data, store=sos)
-        sos.add_thin_pack(f.read, None)
-        self.assertEqual(len(self.fsc.store), 6)
-
-
 @skipIf(missing_libs, skipmsg)
 class TestSwiftRepo(TestCase):
 
@@ -432,51 +291,6 @@ class TestSwiftRepo(TestCase):
         self.assertIn('fakeroot/description', fsc.store)
 
 
-@skipIf(missing_libs, skipmsg)
-@skipIfPY3
-class TestPackInfoLoadDump(TestCase):
-
-    def setUp(self):
-        super(TestPackInfoLoadDump, self).setUp()
-        conf = swift.load_conf(file=StringIO(config_file %
-                                             def_config_file))
-        sos = swift.SwiftObjectStore(
-            FakeSwiftConnector('fakerepo', conf=conf))
-        commit_amount = 10
-        self.commits = create_commits(length=commit_amount, marker="m")
-        data = [(d.type_num, d.as_raw_string()) for d in self.commits]
-        f = BytesIO()
-        fi = BytesIO()
-        expected = build_pack(f, data, store=sos)
-        entries = [(sha, ofs, checksum) for
-                   ofs, _, _, sha, checksum in expected]
-        self.pack_data = PackData.from_file(file=f, size=None)
-        write_pack_index_v2(
-            fi, entries, self.pack_data.calculate_checksum())
-        fi.seek(0)
-        self.pack_index = load_pack_index_file('', fi)
-
-#    def test_pack_info_perf(self):
-#        dump_time = []
-#        load_time = []
-#        for i in range(0, 100):
-#            start = time()
-#            dumps = swift.pack_info_create(self.pack_data, self.pack_index)
-#            dump_time.append(time() - start)
-#        for i in range(0, 100):
-#            start = time()
-#            pack_infos = swift.load_pack_info('', file=BytesIO(dumps))
-#            load_time.append(time() - start)
-#        print sum(dump_time) / float(len(dump_time))
-#        print sum(load_time) / float(len(load_time))
-
-    def test_pack_info(self):
-        dumps = swift.pack_info_create(self.pack_data, self.pack_index)
-        pack_infos = swift.load_pack_info('', file=BytesIO(dumps))
-        for obj in self.commits:
-            self.assertIn(obj.id, pack_infos)
-
-
 @skipIf(missing_libs, skipmsg)
 class TestSwiftInfoRefsContainer(TestCase):
 
@@ -581,7 +395,7 @@ class TestSwiftConnector(TestCase):
 
     def test_get_container_objects(self):
         with patch('geventhttpclient.HTTPClient.request',
-                   lambda *args: Response(content=json_dumps(
+                   lambda *args: Response(content=json.dumps(
                        (({'name': 'a'}, {'name': 'b'}))))):
             self.assertEqual(len(self.conn.get_container_objects()), 2)
 

dulwich/contrib/test_swift_smoke.py

@@ -140,9 +140,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         swift.SwiftRepo.init_bare(self.scon, self.conf)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack(self.fakerepo,
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack(self.fakerepo, determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo("fakerepo", self.conf)
         remote_sha = swift_repo.refs.read_loose_ref('refs/heads/master')
         self.assertEqual(sha, remote_sha)
@@ -160,9 +159,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         swift.SwiftRepo.init_bare(self.scon, self.conf)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack("/fakerepo",
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack("/fakerepo", determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo(self.fakerepo, self.conf)
         remote_sha = swift_repo.refs.read_loose_ref('refs/heads/mybranch')
         self.assertEqual(sha, remote_sha)
@@ -187,9 +185,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         swift.SwiftRepo.init_bare(self.scon, self.conf)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack(self.fakerepo,
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack(self.fakerepo, determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo("fakerepo", self.conf)
         for branch in ('master', 'mybranch', 'pullr-108'):
             remote_shas[branch] = swift_repo.refs.read_loose_ref(
@@ -212,9 +209,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         swift.SwiftRepo.init_bare(self.scon, self.conf)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack(self.fakerepo,
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack(self.fakerepo, determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo("fakerepo", self.conf)
         commit_sha = swift_repo.refs.read_loose_ref('refs/heads/master')
         otype, data = swift_repo.object_store.get_raw(commit_sha)
@@ -259,9 +255,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         local_repo.stage(files)
         local_repo.do_commit('Test commit', 'fbo@localhost',
                              ref='refs/heads/master')
-        tcp_client.send_pack("/fakerepo",
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack("/fakerepo", determine_wants,
+                             local_repo.generate_pack_data)
 
     def test_push_remove_branch(self):
         def determine_wants(*args):
@@ -275,9 +270,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         local_repo = repo.Repo(self.temp_d)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack(self.fakerepo,
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack(self.fakerepo, determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo("fakerepo", self.conf)
         self.assertNotIn('refs/heads/pullr-108', swift_repo.refs.allkeys())
 
@@ -302,9 +296,8 @@ class SwiftRepoSmokeTest(unittest.TestCase):
         swift.SwiftRepo.init_bare(self.scon, self.conf)
         tcp_client = client.TCPGitClient(self.server_address,
                                          port=self.port)
-        tcp_client.send_pack(self.fakerepo,
-                             determine_wants,
-                             local_repo.object_store.generate_pack_data)
+        tcp_client.send_pack(self.fakerepo, determine_wants,
+                             local_repo.generate_pack_data)
         swift_repo = swift.SwiftRepo(self.fakerepo, self.conf)
         tag_sha = swift_repo.refs.read_loose_ref('refs/tags/v1.0')
         otype, data = swift_repo.object_store.get_raw(tag_sha)

dulwich/diff_tree.py

@@ -19,7 +19,7 @@
 #
 
 """Utilities for diffing files and trees."""
-import sys
+
 from collections import (
     defaultdict,
     namedtuple,
@@ -315,8 +315,7 @@ def _count_blocks(obj):
     block_getvalue = block.getvalue
 
     for c in chain(*obj.as_raw_chunks()):
-        if sys.version_info[0] == 3:
-            c = c.to_bytes(1, 'big')
+        c = c.to_bytes(1, 'big')
         block_write(c)
         n += 1
         if c == b'\n' or n == _BLOCK_SIZE:
@@ -619,6 +618,10 @@ _merge_entries_py = _merge_entries
 _count_blocks_py = _count_blocks
 try:
     # Try to import C versions
-    from dulwich._diff_tree import _is_tree, _merge_entries, _count_blocks
+    from dulwich._diff_tree import (  # type: ignore
+        _is_tree,
+        _merge_entries,
+        _count_blocks,
+        )
 except ImportError:
     pass

dulwich/errors.py

@@ -162,10 +162,6 @@ class ObjectFormatException(FileFormatException):
     """Indicates an error parsing an object."""
 
 
-class EmptyFileException(FileFormatException):
-    """An unexpectedly empty file was encountered."""
-
-
 class NoIndexPresent(Exception):
     """No index is present."""
 

dulwich/fastexport.py

@@ -21,8 +21,6 @@
 
 """Fast export/import functionality."""
 
-import sys
-
 from dulwich.index import (
     commit_tree,
     )
@@ -32,10 +30,6 @@ from dulwich.objects import (
     Tag,
     ZERO_SHA,
     )
-from fastimport import __version__ as fastimport_version
-if (fastimport_version <= (0, 9, 5) and
-        sys.version_info[0] == 3 and sys.version_info[1] < 5):
-    raise ImportError("Older versions of fastimport don't support python3<3.5")
 from fastimport import (  # noqa: E402
     commands,
     errors as fastimport_errors,

dulwich/file.py

@@ -20,7 +20,6 @@
 
 """Safe access to git files."""
 
-import errno
 import io
 import os
 import sys
@@ -31,9 +30,8 @@ def ensure_dir_exists(dirname):
     """Ensure a directory exists, creating if necessary."""
     try:
         os.makedirs(dirname)
-    except OSError as e:
-        if e.errno != errno.EEXIST:
-            raise
+    except FileExistsError:
+        pass
 
 
 def _fancy_rename(oldname, newname):
@@ -127,10 +125,8 @@ class _GitFile(object):
                 self._lockfilename,
                 os.O_RDWR | os.O_CREAT | os.O_EXCL |
                 getattr(os, "O_BINARY", 0))
-        except OSError as e:
-            if e.errno == errno.EEXIST:
-                raise FileLocked(filename, self._lockfilename)
-            raise
+        except FileExistsError:
+            raise FileLocked(filename, self._lockfilename)
         self._file = os.fdopen(fd, mode, bufsize)
         self._closed = False
 
@@ -148,10 +144,8 @@ class _GitFile(object):
         try:
             os.remove(self._lockfilename)
             self._closed = True
-        except OSError as e:
+        except FileNotFoundError:
             # The file may have been removed already, which is ok.
-            if e.errno != errno.ENOENT:
-                raise
             self._closed = True
 
     def close(self):

dulwich/hooks.py

@@ -22,7 +22,6 @@
 
 import os
 import subprocess
-import sys
 import tempfile
 
 from dulwich.errors import (
@@ -82,11 +81,6 @@ class ShellHook(Hook):
 
         self.cwd = cwd
 
-        if sys.version_info[0] == 2 and sys.platform == 'win32':
-            # Python 2 on windows does not support unicode file paths
-            # http://bugs.python.org/issue1759845
-            self.filepath = self.filepath.encode(sys.getfilesystemencoding())
-
     def execute(self, *args):
         """Execute the hook with given args"""
 
@@ -103,8 +97,8 @@ class ShellHook(Hook):
             if ret != 0:
                 if (self.post_exec_callback is not None):
                     self.post_exec_callback(0, *args)
-                raise HookError("Hook %s exited with non-zero status"
-                                % (self.name))
+                raise HookError("Hook %s exited with non-zero status %d"
+                                % (self.name, ret))
             if (self.post_exec_callback is not None):
                 return self.post_exec_callback(1, *args)
         except OSError:  # no file. silent failure.
@@ -160,3 +154,43 @@ class CommitMsgShellHook(ShellHook):
 
         ShellHook.__init__(self, 'commit-msg', filepath, 1,
                            prepare_msg, clean_msg, controldir)
+
+
+class PostReceiveShellHook(ShellHook):
+    """post-receive shell hook"""
+
+    def __init__(self, controldir):
+        self.controldir = controldir
+        filepath = os.path.join(controldir, 'hooks', 'post-receive')
+        ShellHook.__init__(self, 'post-receive', filepath, 0)
+
+    def execute(self, client_refs):
+        # do nothing if the script doesn't exist
+        if not os.path.exists(self.filepath):
+            return None
+
+        try:
+            env = os.environ.copy()
+            env['GIT_DIR'] = self.controldir
+
+            p = subprocess.Popen(
+                self.filepath,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                env=env
+            )
+
+            # client_refs is a list of (oldsha, newsha, ref)
+            in_data = '\n'.join([' '.join(ref) for ref in client_refs])
+
+            out_data, err_data = p.communicate(in_data)
+
+            if (p.returncode != 0) or err_data:
+                err_fmt = "post-receive exit code: %d\n" \
+                    + "stdout:\n%s\nstderr:\n%s"
+                err_msg = err_fmt % (p.returncode, out_data, err_data)
+                raise HookError(err_msg)
+            return out_data
+        except OSError as err:
+            raise HookError(repr(err))

dulwich/ignore.py

@@ -24,7 +24,8 @@ For details for the matching rules, see https://git-scm.com/docs/gitignore
 
 import os.path
 import re
-import sys
+
+from dulwich.config import get_xdg_config_home_path
 
 
 def _translate_segment(segment):
@@ -161,7 +162,7 @@ class Pattern(object):
         return self.pattern
 
     def __str__(self):
-        return self.pattern.decode(sys.getfilesystemencoding())
+        return os.fsdecode(self.pattern)
 
     def __eq__(self, other):
         return (type(self) == type(other) and
@@ -203,7 +204,7 @@ class IgnoreFilter(object):
           Iterator over  iterators
         """
         if not isinstance(path, bytes):
-            path = path.encode(sys.getfilesystemencoding())
+            path = os.fsencode(path)
         for pattern in self._patterns:
             if pattern.match(path):
                 yield pattern
@@ -271,8 +272,7 @@ def default_user_ignore_filter_path(config):
     except KeyError:
         pass
 
-    xdg_config_home = os.environ.get("XDG_CONFIG_HOME", "~/.config/")
-    return os.path.join(xdg_config_home, 'git', 'ignore')
+    return get_xdg_config_home_path('git', 'ignore')
 
 
 class IgnoreFilterManager(object):

dulwich/index.py

@@ -21,7 +21,6 @@
 """Parser for the git index file format."""
 
 import collections
-import errno
 import os
 import stat
 import struct
@@ -454,7 +453,8 @@ def index_entry_from_stat(stat_val, hex_sha, flags, mode=None):
             stat_val.st_gid, stat_val.st_size, hex_sha, flags)
 
 
-def build_file_from_blob(blob, mode, target_path, honor_filemode=True):
+def build_file_from_blob(blob, mode, target_path, honor_filemode=True,
+                         tree_encoding='utf-8'):
     """Build a file or symlink on disk based on a Git object.
 
     Args:
@@ -467,20 +467,15 @@ def build_file_from_blob(blob, mode, target_path, honor_filemode=True):
     """
     try:
         oldstat = os.lstat(target_path)
-    except OSError as e:
-        if e.errno == errno.ENOENT:
-            oldstat = None
-        else:
-            raise
+    except FileNotFoundError:
+        oldstat = None
     contents = blob.as_raw_string()
     if stat.S_ISLNK(mode):
         # FIXME: This will fail on Windows. What should we do instead?
         if oldstat:
             os.unlink(target_path)
-        if sys.platform == 'win32' and sys.version_info[0] == 3:
+        if sys.platform == 'win32':
             # os.readlink on Python3 on Windows requires a unicode string.
-            # TODO(jelmer): Don't assume tree_encoding == fs_encoding
-            tree_encoding = sys.getfilesystemencoding()
             contents = contents.decode(tree_encoding)
             target_path = target_path.decode(tree_encoding)
         os.symlink(contents, target_path)
@@ -547,7 +542,7 @@ def build_index_from_tree(root_path, index_path, object_store, tree_id,
 
     index = Index(index_path)
     if not isinstance(root_path, bytes):
-        root_path = root_path.encode(sys.getfilesystemencoding())
+        root_path = os.fsencode(root_path)
 
     for entry in object_store.iter_tree_contents(tree_id):
         if not validate_path(entry.path, validate_path_element):
@@ -567,6 +562,7 @@ def build_index_from_tree(root_path, index_path, object_store, tree_id,
             obj = object_store[entry.sha]
             st = build_file_from_blob(
                 obj, entry.mode, full_path, honor_filemode=honor_filemode)
+
         # Add file to index
         if not honor_filemode or S_ISGITLINK(entry.mode):
             # we can not use tuple slicing to build a new tuple,
@@ -581,7 +577,7 @@ def build_index_from_tree(root_path, index_path, object_store, tree_id,
     index.write()
 
 
-def blob_from_path_and_stat(fs_path, st):
+def blob_from_path_and_stat(fs_path, st, tree_encoding='utf-8'):
     """Create a blob from a path and a stat object.
 
     Args:
@@ -591,18 +587,16 @@ def blob_from_path_and_stat(fs_path, st):
     """
     assert isinstance(fs_path, bytes)
     blob = Blob()
-    if not stat.S_ISLNK(st.st_mode):
-        with open(fs_path, 'rb') as f:
-            blob.data = f.read()
-    else:
-        if sys.platform == 'win32' and sys.version_info[0] == 3:
+    if stat.S_ISLNK(st.st_mode):
+        if sys.platform == 'win32':
             # os.readlink on Python3 on Windows requires a unicode string.
-            # TODO(jelmer): Don't assume tree_encoding == fs_encoding
-            tree_encoding = sys.getfilesystemencoding()
-            fs_path = fs_path.decode(tree_encoding)
+            fs_path = os.fsdecode(fs_path)
             blob.data = os.readlink(fs_path).encode(tree_encoding)
         else:
             blob.data = os.readlink(fs_path)
+    else:
+        with open(fs_path, 'rb') as f:
+            blob.data = f.read()
     return blob
 
 
@@ -618,7 +612,7 @@ def read_submodule_head(path):
     # Repo currently expects a "str", so decode if necessary.
     # TODO(jelmer): Perhaps move this into Repo() ?
     if not isinstance(path, str):
-        path = path.decode(sys.getfilesystemencoding())
+        path = os.fsdecode(path)
     try:
         repo = Repo(path)
     except NotGitRepository:
@@ -664,7 +658,7 @@ def get_unstaged_changes(index, root_path, filter_blob_callback=None):
     """
     # For each entry in the index check the sha1 & ensure not staged
     if not isinstance(root_path, bytes):
-        root_path = root_path.encode(sys.getfilesystemencoding())
+        root_path = os.fsencode(root_path)
 
     for tree_path, entry in index.iteritems():
         full_path = _tree_to_fs_path(root_path, tree_path)
@@ -675,17 +669,17 @@ def get_unstaged_changes(index, root_path, filter_blob_callback=None):
                     yield tree_path
                 continue
 
+            if not stat.S_ISREG(st.st_mode) and not stat.S_ISLNK(st.st_mode):
+                continue
+
             blob = blob_from_path_and_stat(full_path, st)
 
             if filter_blob_callback is not None:
                 blob = filter_blob_callback(blob, tree_path)
-        except EnvironmentError as e:
-            if e.errno == errno.ENOENT:
-                # The file was removed, so we assume that counts as
-                # different from whatever file used to exist.
-                yield tree_path
-            else:
-                raise
+        except FileNotFoundError:
+            # The file was removed, so we assume that counts as
+            # different from whatever file used to exist.
+            yield tree_path
         else:
             if blob.id != entry.sha:
                 yield tree_path
@@ -711,19 +705,16 @@ def _tree_to_fs_path(root_path, tree_path):
     return os.path.join(root_path, sep_corrected_path)
 
 
-def _fs_to_tree_path(fs_path, fs_encoding=None):
+def _fs_to_tree_path(fs_path):
     """Convert a file system path to a git tree path.
 
     Args:
       fs_path: File system path.
-      fs_encoding: File system encoding
 
     Returns:  Git tree path as bytes
     """
-    if fs_encoding is None:
-        fs_encoding = sys.getfilesystemencoding()
     if not isinstance(fs_path, bytes):
-        fs_path_bytes = fs_path.encode(fs_encoding)
+        fs_path_bytes = os.fsencode(fs_path)
     else:
         fs_path_bytes = fs_path
     if os_sep_bytes != b'/':
@@ -757,10 +748,13 @@ def index_entry_from_path(path, object_store=None):
                 st, head, 0, mode=S_IFGITLINK)
         return None
 
-    blob = blob_from_path_and_stat(path, st)
-    if object_store is not None:
-        object_store.add_object(blob)
-    return index_entry_from_stat(st, blob.id, 0)
+    if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode):
+        blob = blob_from_path_and_stat(path, st)
+        if object_store is not None:
+            object_store.add_object(blob)
+        return index_entry_from_stat(st, blob.id, 0)
+
+    return None
 
 
 def iter_fresh_entries(paths, root_path, object_store=None):
@@ -776,11 +770,8 @@ def iter_fresh_entries(paths, root_path, object_store=None):
         p = _tree_to_fs_path(root_path, path)
         try:
             entry = index_entry_from_path(p, object_store=object_store)
-        except EnvironmentError as e:
-            if e.errno in (errno.ENOENT, errno.EISDIR):
-                entry = None
-            else:
-                raise
+        except (FileNotFoundError, IsADirectoryError):
+            entry = None
         yield path, entry
 
 

dulwich/lfs.py

@@ -0,0 +1,75 @@
+# lfs.py -- Implementation of the LFS
+# Copyright (C) 2020 Jelmer Vernooij
+#
+# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+# General Public License as public by the Free Software Foundation; version 2.0
+# or (at your option) any later version. You can redistribute it and/or
+# modify it under the terms of either of these two licenses.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# You should have received a copy of the licenses; if not, see
+# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+# License, Version 2.0.
+#
+
+import hashlib
+import os
+import tempfile
+
+
+class LFSStore(object):
+    """Stores objects on disk, indexed by SHA256."""
+
+    def __init__(self, path):
+        self.path = path
+
+    @classmethod
+    def create(cls, lfs_dir):
+        if not os.path.isdir(lfs_dir):
+            os.mkdir(lfs_dir)
+        os.mkdir(os.path.join(lfs_dir, 'tmp'))
+        os.mkdir(os.path.join(lfs_dir, 'objects'))
+        return cls(lfs_dir)
+
+    @classmethod
+    def from_repo(cls, repo, create=False):
+        lfs_dir = os.path.join(repo.controldir, 'lfs')
+        if create:
+            return cls.create(lfs_dir)
+        return cls(lfs_dir)
+
+    def _sha_path(self, sha):
+        return os.path.join(self.path, 'objects', sha[0:2], sha[2:4], sha)
+
+    def open_object(self, sha):
+        """Open an object by sha."""
+        try:
+            return open(self._sha_path(sha), 'rb')
+        except FileNotFoundError:
+            raise KeyError(sha)
+
+    def write_object(self, chunks):
+        """Write an object.
+
+        Returns: object SHA
+        """
+        sha = hashlib.sha256()
+        tmpdir = os.path.join(self.path, 'tmp')
+        with tempfile.NamedTemporaryFile(
+                dir=tmpdir, mode='wb', delete=False) as f:
+            for chunk in chunks:
+                sha.update(chunk)
+                f.write(chunk)
+            f.flush()
+            tmppath = f.name
+        path = self._sha_path(sha.hexdigest())
+        if not os.path.exists(os.path.dirname(path)):
+            os.makedirs(os.path.dirname(path))
+        os.rename(tmppath, path)
+        return sha.hexdigest()

dulwich/object_store.py

@@ -23,7 +23,6 @@
 """Git object store interfaces and implementation."""
 
 from io import BytesIO
-import errno
 import os
 import stat
 import sys
@@ -48,6 +47,7 @@ from dulwich.objects import (
     hex_to_filename,
     S_ISGITLINK,
     object_class,
+    valid_hexsha,
     )
 from dulwich.pack import (
     Pack,
@@ -152,7 +152,9 @@ class BaseObjectStore(object):
             return
         f, commit, abort = self.add_pack()
         try:
-            write_pack_data(f, count, pack_data, progress)
+            write_pack_data(
+                f, count, pack_data, progress,
+                compression_level=self.pack_compression_level)
         except BaseException:
             abort()
             raise
@@ -199,7 +201,7 @@ class BaseObjectStore(object):
                  not stat.S_ISDIR(entry.mode)) or include_trees):
                 yield entry
 
-    def find_missing_objects(self, haves, wants, progress=None,
+    def find_missing_objects(self, haves, wants, shallow=None, progress=None,
                              get_tagged=None,
                              get_parents=lambda commit: commit.parents,
                              depth=None):
@@ -208,6 +210,7 @@ class BaseObjectStore(object):
         Args:
           haves: Iterable over SHAs already in common.
           wants: Iterable over SHAs of objects to fetch.
+          shallow: Set of shallow commit SHA1s to skip
           progress: Simple progress function that will be called with
             updated progress strings.
           get_tagged: Function that returns a dict of pointed-to sha ->
@@ -216,8 +219,8 @@ class BaseObjectStore(object):
             commit.
         Returns: Iterator over (sha, path) pairs.
         """
-        finder = MissingObjectFinder(self, haves, wants, progress, get_tagged,
-                                     get_parents=get_parents)
+        finder = MissingObjectFinder(self, haves, wants, shallow, progress,
+                                     get_tagged, get_parents=get_parents)
         return iter(finder.next, None)
 
     def find_common_revisions(self, graphwalker):
@@ -236,28 +239,32 @@ class BaseObjectStore(object):
             sha = next(graphwalker)
         return haves
 
-    def generate_pack_contents(self, have, want, progress=None):
+    def generate_pack_contents(self, have, want, shallow=None, progress=None):
         """Iterate over the contents of a pack file.
 
         Args:
           have: List of SHA1s of objects that should not be sent
           want: List of SHA1s of objects that should be sent
+          shallow: Set of shallow commit SHA1s to skip
           progress: Optional progress reporting method
         """
-        return self.iter_shas(self.find_missing_objects(have, want, progress))
+        missing = self.find_missing_objects(have, want, shallow, progress)
+        return self.iter_shas(missing)
 
-    def generate_pack_data(self, have, want, progress=None, ofs_delta=True):
+    def generate_pack_data(self, have, want, shallow=None, progress=None,
+                           ofs_delta=True):
         """Generate pack data objects for a set of wants/haves.
 
         Args:
           have: List of SHA1s of objects that should not be sent
           want: List of SHA1s of objects that should be sent
+          shallow: Set of shallow commit SHA1s to skip
           ofs_delta: Whether OFS deltas can be included
           progress: Optional progress reporting method
         """
         # TODO(jelmer): More efficient implementation
         return pack_objects_to_data(
-            self.generate_pack_contents(have, want, progress))
+            self.generate_pack_contents(have, want, shallow, progress))
 
     def peel_sha(self, sha):
         """Peel all tags from a SHA.
@@ -275,7 +282,7 @@ class BaseObjectStore(object):
             obj = self[sha]
         return obj
 
-    def _collect_ancestors(self, heads, common=set(),
+    def _collect_ancestors(self, heads, common=set(), shallow=set(),
                            get_parents=lambda commit: commit.parents):
         """Collect all ancestors of heads up to (excluding) those in common.
 
@@ -299,6 +306,8 @@ class BaseObjectStore(object):
                 bases.add(e)
             elif e not in commits:
                 commits.add(e)
+                if e in shallow:
+                    continue
                 cmt = self[e]
                 queue.extend(get_parents(cmt))
         return (commits, bases)
@@ -310,8 +319,9 @@ class BaseObjectStore(object):
 
 class PackBasedObjectStore(BaseObjectStore):
 
-    def __init__(self):
+    def __init__(self, pack_compression_level=-1):
         self._pack_cache = {}
+        self.pack_compression_level = pack_compression_level
 
     @property
     def alternates(self):
@@ -512,20 +522,45 @@ class PackBasedObjectStore(BaseObjectStore):
 class DiskObjectStore(PackBasedObjectStore):
     """Git-style object store that exists on disk."""
 
-    def __init__(self, path):
+    def __init__(self, path, loose_compression_level=-1,
+                 pack_compression_level=-1):
         """Open an object store.
 
         Args:
           path: Path of the object store.
+          loose_compression_level: zlib compression level for loose objects
+          pack_compression_level: zlib compression level for pack objects
         """
-        super(DiskObjectStore, self).__init__()
+        super(DiskObjectStore, self).__init__(
+            pack_compression_level=pack_compression_level)
         self.path = path
         self.pack_dir = os.path.join(self.path, PACKDIR)
         self._alternates = None
+        self.loose_compression_level = loose_compression_level
+        self.pack_compression_level = pack_compression_level
 
     def __repr__(self):
         return "<%s(%r)>" % (self.__class__.__name__, self.path)
 
+    @classmethod
+    def from_config(cls, path, config):
+        try:
+            default_compression_level = int(config.get(
+                (b'core', ), b'compression').decode())
+        except KeyError:
+            default_compression_level = -1
+        try:
+            loose_compression_level = int(config.get(
+                (b'core', ), b'looseCompression').decode())
+        except KeyError:
+            loose_compression_level = default_compression_level
+        try:
+            pack_compression_level = int(config.get(
+                (b'core', ), 'packCompression').decode())
+        except KeyError:
+            pack_compression_level = default_compression_level
+        return cls(path, loose_compression_level, pack_compression_level)
+
     @property
     def alternates(self):
         if self._alternates is not None:
@@ -538,40 +573,35 @@ class DiskObjectStore(PackBasedObjectStore):
     def _read_alternate_paths(self):
         try:
             f = GitFile(os.path.join(self.path, INFODIR, "alternates"), 'rb')
-        except (OSError, IOError) as e:
-            if e.errno == errno.ENOENT:
-                return
-            raise
+        except FileNotFoundError:
+            return
         with f:
             for line in f.readlines():
                 line = line.rstrip(b"\n")
                 if line[0] == b"#":
                     continue
                 if os.path.isabs(line):
-                    yield line.decode(sys.getfilesystemencoding())
+                    yield os.fsdecode(line)
                 else:
-                    yield os.path.join(self.path, line).decode(
-                        sys.getfilesystemencoding())
+                    yield os.fsdecode(os.path.join(self.path, line))
 
     def add_alternate_path(self, path):
         """Add an alternate path to this object store.
         """
         try:
             os.mkdir(os.path.join(self.path, INFODIR))
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
+        except FileExistsError:
+            pass
         alternates_path = os.path.join(self.path, INFODIR, "alternates")
         with GitFile(alternates_path, 'wb') as f:
             try:
                 orig_f = open(alternates_path, 'rb')
-            except (OSError, IOError) as e:
-                if e.errno != errno.ENOENT:
-                    raise
+            except FileNotFoundError:
+                pass
             else:
                 with orig_f:
                     f.write(orig_f.read())
-            f.write(path.encode(sys.getfilesystemencoding()) + b"\n")
+            f.write(os.fsencode(path) + b"\n")
 
         if not os.path.isabs(path):
             path = os.path.join(self.path, path)
@@ -581,11 +611,9 @@ class DiskObjectStore(PackBasedObjectStore):
         """Read and iterate over new pack files and cache them."""
         try:
             pack_dir_contents = os.listdir(self.pack_dir)
-        except OSError as e:
-            if e.errno == errno.ENOENT:
-                self.close()
-                return []
-            raise
+        except FileNotFoundError:
+            self.close()
+            return []
         pack_files = set()
         for name in pack_dir_contents:
             if name.startswith("pack-") and name.endswith(".pack"):
@@ -617,16 +645,17 @@ class DiskObjectStore(PackBasedObjectStore):
             if len(base) != 2:
                 continue
             for rest in os.listdir(os.path.join(self.path, base)):
-                yield (base+rest).encode(sys.getfilesystemencoding())
+                sha = os.fsencode(base+rest)
+                if not valid_hexsha(sha):
+                    continue
+                yield sha
 
     def _get_loose_object(self, sha):
         path = self._get_shafile_path(sha)
         try:
             return ShaFile.from_path(path)
-        except (OSError, IOError) as e:
-            if e.errno == errno.ENOENT:
-                return None
-            raise
+        except FileNotFoundError:
+            return None
 
     def _remove_loose_object(self, sha):
         os.remove(self._get_shafile_path(sha))
@@ -678,7 +707,9 @@ class DiskObjectStore(PackBasedObjectStore):
             assert len(ext_sha) == 20
             type_num, data = self.get_raw(ext_sha)
             offset = f.tell()
-            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
+            crc32 = write_pack_object(
+                f, type_num, data, sha=new_sha,
+                compression_level=self.pack_compression_level)
             entries.append((ext_sha, offset, crc32))
         pack_sha = new_sha.digest()
         f.write(pack_sha)
@@ -693,9 +724,8 @@ class DiskObjectStore(PackBasedObjectStore):
             # removal, silently passing if the target does not exist.
             try:
                 os.remove(target_pack)
-            except (IOError, OSError) as e:
-                if e.errno != errno.ENOENT:
-                    raise
+            except FileNotFoundError:
+                pass
         os.rename(path, target_pack)
 
         # Write the index.
@@ -760,9 +790,8 @@ class DiskObjectStore(PackBasedObjectStore):
             # removal, silently passing if the target does not exist.
             try:
                 os.remove(target_pack)
-            except (IOError, OSError) as e:
-                if e.errno != errno.ENOENT:
-                    raise
+            except FileNotFoundError:
+                pass
         os.rename(path, target_pack)
         final_pack = Pack(basename)
         self._add_cached_pack(basename, final_pack)
@@ -803,21 +832,20 @@ class DiskObjectStore(PackBasedObjectStore):
         dir = os.path.dirname(path)
         try:
             os.mkdir(dir)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
+        except FileExistsError:
+            pass
         if os.path.exists(path):
             return  # Already there, no need to write again
         with GitFile(path, 'wb') as f:
-            f.write(obj.as_legacy_object())
+            f.write(obj.as_legacy_object(
+                compression_level=self.loose_compression_level))
 
     @classmethod
     def init(cls, path):
         try:
             os.mkdir(path)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
+        except FileExistsError:
+            pass
         os.mkdir(os.path.join(path, "info"))
         os.mkdir(os.path.join(path, PACKDIR))
         return cls(path)
@@ -829,6 +857,7 @@ class MemoryObjectStore(BaseObjectStore):
     def __init__(self):
         super(MemoryObjectStore, self).__init__()
         self._data = {}
+        self.pack_compression_level = -1
 
     def _to_hexsha(self, sha):
         if len(sha) == 40:
@@ -928,7 +957,8 @@ class MemoryObjectStore(BaseObjectStore):
         for ext_sha in indexer.ext_refs():
             assert len(ext_sha) == 20
             type_num, data = self.get_raw(ext_sha)
-            write_pack_object(f, type_num, data, sha=new_sha)
+            write_pack_object(
+                f, type_num, data, sha=new_sha)
         pack_sha = new_sha.digest()
         f.write(pack_sha)
 
@@ -1130,9 +1160,11 @@ class MissingObjectFinder(object):
       tagged: dict of pointed-to sha -> tag sha for including tags
     """
 
-    def __init__(self, object_store, haves, wants, progress=None,
+    def __init__(self, object_store, haves, wants, shallow=None, progress=None,
                  get_tagged=None, get_parents=lambda commit: commit.parents):
         self.object_store = object_store
+        if shallow is None:
+            shallow = set()
         self._get_parents = get_parents
         # process Commits and Tags differently
         # Note, while haves may list commits/tags not available locally,
@@ -1146,12 +1178,13 @@ class MissingObjectFinder(object):
         # all_ancestors is a set of commits that shall not be sent
         # (complete repository up to 'haves')
         all_ancestors = object_store._collect_ancestors(
-            have_commits, get_parents=self._get_parents)[0]
+            have_commits, shallow=shallow, get_parents=self._get_parents)[0]
         # all_missing - complete set of commits between haves and wants
         # common - commits from all_ancestors we hit into while
         # traversing parent hierarchy of wants
         missing_commits, common_commits = object_store._collect_ancestors(
-            want_commits, all_ancestors, get_parents=self._get_parents)
+            want_commits, all_ancestors, shallow=shallow,
+            get_parents=self._get_parents)
         self.sha_done = set()
         # Now, fill sha_done with commits and revisions of
         # files and directories known to be both locally
@@ -1382,4 +1415,4 @@ def read_packs_file(f):
         (kind, name) = line.split(b" ", 1)
         if kind != b"P":
             continue
-        yield name.decode(sys.getfilesystemencoding())
+        yield os.fsdecode(name)

dulwich/objects.py

@@ -27,7 +27,12 @@ from collections import namedtuple
 import os
 import posixpath
 import stat
-import sys
+from typing import (
+    Optional,
+    Dict,
+    Union,
+    Type,
+    )
 import warnings
 import zlib
 from hashlib import sha1
@@ -39,7 +44,7 @@ from dulwich.errors import (
     NotTagError,
     NotTreeError,
     ObjectFormatException,
-    EmptyFileException,
+    FileFormatException,
     )
 from dulwich.file import GitFile
 
@@ -70,6 +75,10 @@ MAX_TIME = 9223372036854775807  # (2**63) - 1 - signed long int max
 BEGIN_PGP_SIGNATURE = b"-----BEGIN PGP SIGNATURE-----"
 
 
+class EmptyFileException(FileFormatException):
+    """An unexpectedly empty file was encountered."""
+
+
 def S_ISGITLINK(m):
     """Check if a mode indicates a submodule.
 
@@ -142,13 +151,13 @@ def filename_to_hex(filename):
     return hex
 
 
-def object_header(num_type, length):
+def object_header(num_type: int, length: int) -> bytes:
     """Return an object header for the given numeric type and text length."""
     return (object_class(num_type).type_name +
             b' ' + str(length).encode('ascii') + b'\0')
 
 
-def serializable_property(name, docstring=None):
+def serializable_property(name: str, docstring: Optional[str] = None):
     """A property that helps tracking whether serialization is necessary.
     """
     def set(obj, value):
@@ -249,6 +258,9 @@ class ShaFile(object):
 
     __slots__ = ('_chunked_text', '_sha', '_needs_serialization')
 
+    type_name = None  # type: bytes
+    type_num = None  # type: int
+
     @staticmethod
     def _parse_legacy_object_header(magic, f):
         """Parse a legacy object, creating it but not reading the file."""
@@ -282,21 +294,22 @@ class ShaFile(object):
             raise ObjectFormatException("Invalid object header, no \\0")
         self.set_raw_string(text[header_end+1:])
 
-    def as_legacy_object_chunks(self):
+    def as_legacy_object_chunks(self, compression_level=-1):
         """Return chunks representing the object in the experimental format.
 
         Returns: List of strings
         """
-        compobj = zlib.compressobj()
+        compobj = zlib.compressobj(compression_level)
         yield compobj.compress(self._header())
         for chunk in self.as_raw_chunks():
             yield compobj.compress(chunk)
         yield compobj.flush()
 
-    def as_legacy_object(self):
+    def as_legacy_object(self, compression_level=-1):
         """Return string representing the object in the experimental format.
         """
-        return b''.join(self.as_legacy_object_chunks())
+        return b''.join(self.as_legacy_object_chunks(
+            compression_level=compression_level))
 
     def as_raw_chunks(self):
         """Return chunks with serialization of the object.
@@ -316,14 +329,9 @@ class ShaFile(object):
         """
         return b''.join(self.as_raw_chunks())
 
-    if sys.version_info[0] >= 3:
-        def __bytes__(self):
-            """Return raw string serialization of this object."""
-            return self.as_raw_string()
-    else:
-        def __str__(self):
-            """Return raw string serialization of this object."""
-            return self.as_raw_string()
+    def __bytes__(self):
+        """Return raw string serialization of this object."""
+        return self.as_raw_string()
 
     def __hash__(self):
         """Return unique hash for this object."""
@@ -586,7 +594,7 @@ class Blob(ShaFile):
         self.set_raw_string(data)
 
     data = property(_get_data, _set_data,
-                    "The text contained within the blob object.")
+                    doc="The text contained within the blob object.")
 
     def _get_chunked(self):
         return self._chunked_text
@@ -602,7 +610,7 @@ class Blob(ShaFile):
 
     chunked = property(
         _get_chunked, _set_chunked,
-        "The text within the blob object, as chunks (not necessarily lines).")
+        doc="The text in the blob object, as chunks (not necessarily lines)")
 
     @classmethod
     def from_path(cls, path):
@@ -1417,7 +1425,7 @@ OBJECT_CLASSES = (
     Tag,
     )
 
-_TYPE_MAP = {}
+_TYPE_MAP = {}  # type: Dict[Union[bytes, int], Type[ShaFile]]
 
 for cls in OBJECT_CLASSES:
     _TYPE_MAP[cls.type_name] = cls
@@ -1429,6 +1437,6 @@ _parse_tree_py = parse_tree
 _sorted_tree_items_py = sorted_tree_items
 try:
     # Try to import C versions
-    from dulwich._objects import parse_tree, sorted_tree_items
+    from dulwich._objects import parse_tree, sorted_tree_items  # type: ignore
 except ImportError:
     pass

dulwich/pack.py

@@ -43,12 +43,6 @@ import difflib
 import struct
 
 from itertools import chain
-try:
-    from itertools import imap, izip
-except ImportError:
-    # Python3
-    imap = map
-    izip = zip
 
 import os
 import sys
@@ -363,8 +357,8 @@ class PackIndex(object):
         if not isinstance(other, PackIndex):
             return False
 
-        for (name1, _, _), (name2, _, _) in izip(self.iterentries(),
-                                                 other.iterentries()):
+        for (name1, _, _), (name2, _, _) in zip(self.iterentries(),
+                                                other.iterentries()):
             if name1 != name2:
                 return False
         return True
@@ -378,7 +372,7 @@ class PackIndex(object):
 
     def __iter__(self):
         """Iterate over the SHAs in this pack."""
-        return imap(sha_to_hex, self._itersha())
+        return map(sha_to_hex, self._itersha())
 
     def iterentries(self):
         """Iterate over the entries in this pack index.
@@ -710,7 +704,7 @@ def chunks_length(chunks):
     if isinstance(chunks, bytes):
         return len(chunks)
     else:
-        return sum(imap(len, chunks))
+        return sum(map(len, chunks))
 
 
 def unpack_object(read_all, read_some=None, compute_crc32=False,
@@ -1531,13 +1525,14 @@ def pack_object_header(type_num, delta_base, size):
     return bytearray(header)
 
 
-def write_pack_object(f, type, object, sha=None):
+def write_pack_object(f, type, object, sha=None, compression_level=-1):
     """Write pack object to a file.
 
     Args:
       f: File to write to
       type: Numeric type of the object
       object: Object to write
+      compression_level: the zlib compression level
     Returns: Tuple with offset at which the object was written, and crc32
     """
     if type in DELTA_TYPES:
@@ -1545,7 +1540,7 @@ def write_pack_object(f, type, object, sha=None):
     else:
         delta_base = None
     header = bytes(pack_object_header(type, delta_base, len(object)))
-    comp_data = zlib.compress(object)
+    comp_data = zlib.compress(object, compression_level)
     crc32 = 0
     for data in (header, comp_data):
         f.write(data)
@@ -1555,7 +1550,8 @@ def write_pack_object(f, type, object, sha=None):
     return crc32 & 0xffffffff
 
 
-def write_pack(filename, objects, deltify=None, delta_window_size=None):
+def write_pack(filename, objects, deltify=None, delta_window_size=None,
+               compression_level=-1):
     """Write a new pack data file.
 
     Args:
@@ -1564,11 +1560,13 @@ def write_pack(filename, objects, deltify=None, delta_window_size=None):
         Should provide __len__
       window_size: Delta window size
       deltify: Whether to deltify pack objects
+      compression_level: the zlib compression level
     Returns: Tuple with checksum of pack file and index file
     """
     with GitFile(filename + '.pack', 'wb') as f:
         entries, data_sum = write_pack_objects(
-            f, objects, delta_window_size=delta_window_size, deltify=deltify)
+            f, objects, delta_window_size=delta_window_size, deltify=deltify,
+            compression_level=compression_level)
     entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
     with GitFile(filename + '.idx', 'wb') as f:
         return data_sum, write_pack_index_v2(f, entries, data_sum)
@@ -1632,7 +1630,8 @@ def pack_objects_to_data(objects):
              for (o, path) in objects))
 
 
-def write_pack_objects(f, objects, delta_window_size=None, deltify=None):
+def write_pack_objects(f, objects, delta_window_size=None, deltify=None,
+                       compression_level=-1):
     """Write a new pack data file.
 
     Args:
@@ -1642,6 +1641,7 @@ def write_pack_objects(f, objects, delta_window_size=None, deltify=None):
       window_size: Sliding window size for searching for deltas;
                         Set to None for default window size.
       deltify: Whether to deltify objects
+      compression_level: the zlib compression level to use
     Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
     """
     if deltify is None:
@@ -1654,10 +1654,13 @@ def write_pack_objects(f, objects, delta_window_size=None, deltify=None):
     else:
         pack_contents_count, pack_contents = pack_objects_to_data(objects)
 
-    return write_pack_data(f, pack_contents_count, pack_contents)
+    return write_pack_data(
+        f, pack_contents_count, pack_contents,
+        compression_level=compression_level)
 
 
-def write_pack_data(f, num_records, records, progress=None):
+def write_pack_data(
+        f, num_records, records, progress=None, compression_level=-1):
     """Write a new pack data file.
 
     Args:
@@ -1665,6 +1668,7 @@ def write_pack_data(f, num_records, records, progress=None):
       num_records: Number of records
       records: Iterator over type_num, object_id, delta_base, raw
       progress: Function to report progress to
+      compression_level: the zlib compression level
     Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
     """
     # Write the pack
@@ -1686,7 +1690,8 @@ def write_pack_data(f, num_records, records, progress=None):
             else:
                 type_num = OFS_DELTA
                 raw = (offset - base_offset, raw)
-        crc32 = write_pack_object(f, type_num, raw)
+        crc32 = write_pack_object(
+            f, type_num, raw, compression_level=compression_level)
         entries[object_id] = (offset, crc32)
     return entries, f.write_sha()
 
@@ -2083,6 +2088,9 @@ class Pack(object):
 
 
 try:
-    from dulwich._pack import apply_delta, bisect_find_sha  # noqa: F811
+    from dulwich._pack import (  # type: ignore # noqa: F811
+        apply_delta,
+        bisect_find_sha,
+        )
 except ImportError:
     pass

dulwich/porcelain.py

@@ -64,6 +64,7 @@ from contextlib import (
 from io import BytesIO, RawIOBase
 import datetime
 import os
+from pathlib import Path
 import posixpath
 import shutil
 import stat
@@ -156,14 +157,10 @@ class NoneStream(RawIOBase):
         return None
 
 
-if sys.version_info[0] == 2:
-    default_bytes_out_stream = sys.stdout or NoneStream()
-    default_bytes_err_stream = sys.stderr or NoneStream()
-else:
-    default_bytes_out_stream = (
-        getattr(sys.stdout, 'buffer', None) or NoneStream())
-    default_bytes_err_stream = (
-        getattr(sys.stderr, 'buffer', None) or NoneStream())
+default_bytes_out_stream = (
+    getattr(sys.stdout, 'buffer', None) or NoneStream())
+default_bytes_err_stream = (
+    getattr(sys.stderr, 'buffer', None) or NoneStream())
 
 
 DEFAULT_ENCODING = 'utf-8'
@@ -196,7 +193,7 @@ def open_repo_closing(path_or_repo):
     return closing(Repo(path_or_repo))
 
 
-def path_to_tree_path(repopath, path):
+def path_to_tree_path(repopath, path, tree_encoding=DEFAULT_ENCODING):
     """Convert a path to a path usable in an index, e.g. bytes and relative to
     the repository root.
 
@@ -205,16 +202,13 @@ def path_to_tree_path(repopath, path):
       path: A path, absolute or relative to the cwd
     Returns: A path formatted for use in e.g. an index
     """
-    if not isinstance(path, bytes):
-        path = path.encode(sys.getfilesystemencoding())
-    if not isinstance(repopath, bytes):
-        repopath = repopath.encode(sys.getfilesystemencoding())
-    treepath = os.path.relpath(path, repopath)
-    if treepath.startswith(b'..'):
-        raise ValueError('Path not in repo')
-    if os.path.sep != '/':
-        treepath = treepath.replace(os.path.sep.encode('ascii'), b'/')
-    return treepath
+    path = Path(path).resolve()
+    repopath = Path(repopath).resolve()
+    relpath = path.relative_to(repopath)
+    if sys.platform == 'win32':
+        return str(relpath).replace(os.path.sep, '/').encode(tree_encoding)
+    else:
+        return bytes(relpath)
 
 
 def archive(repo, committish=None, outstream=default_bytes_out_stream,
@@ -400,17 +394,18 @@ def add(repo=".", paths=None):
     """
     ignored = set()
     with open_repo_closing(repo) as r:
+        repo_path = Path(r.path).resolve()
         ignore_manager = IgnoreFilterManager.from_repo(r)
         if not paths:
             paths = list(
-                get_untracked_paths(os.getcwd(), r.path, r.open_index()))
+                get_untracked_paths(
+                    str(Path(os.getcwd()).resolve()),
+                    str(repo_path), r.open_index()))
         relpaths = []
         if not isinstance(paths, list):
             paths = [paths]
         for p in paths:
-            relpath = os.path.relpath(p, r.path)
-            if relpath.startswith('..' + os.path.sep):
-                raise ValueError('path %r is not in repo' % relpath)
+            relpath = str(Path(p).resolve().relative_to(repo_path))
             # FIXME: Support patterns, directories.
             if ignore_manager.is_ignored(relpath):
                 ignored.add(relpath)
@@ -482,7 +477,7 @@ def remove(repo=".", paths=None, cached=False):
     with open_repo_closing(repo) as r:
         index = r.open_index()
         for p in paths:
-            full_path = os.path.abspath(p).encode(sys.getfilesystemencoding())
+            full_path = os.fsencode(os.path.abspath(p))
             tree_path = path_to_tree_path(r.path, p)
             try:
                 index_sha = index[tree_path].sha
@@ -703,7 +698,7 @@ def log(repo=".", paths=None, outstream=sys.stdout, max_entries=None,
             print_commit(entry.commit, decode, outstream)
             if name_status:
                 outstream.writelines(
-                    [l+'\n' for l in print_name_status(entry.changes())])
+                    [line+'\n' for line in print_name_status(entry.changes())])
 
 
 # TODO(jelmer): better default for encoding?
@@ -912,14 +907,17 @@ def push(repo, remote_location, refspecs,
         try:
             client.send_pack(
                 path, update_refs,
-                generate_pack_data=r.object_store.generate_pack_data,
+                generate_pack_data=r.generate_pack_data,
                 progress=errstream.write)
             errstream.write(
                 b"Push to " + remote_location_bytes + b" successful.\n")
-        except (UpdateRefsError, SendPackError) as e:
+        except UpdateRefsError as e:
             errstream.write(b"Push to " + remote_location_bytes +
                             b" failed -> " + e.message.encode(err_encoding) +
                             b"\n")
+        except SendPackError as e:
+            errstream.write(b"Push to " + remote_location_bytes +
+                            b" failed -> " + e.args[0] + b"\n")
 
 
 def pull(repo, remote_location=None, refspecs=None,
@@ -937,9 +935,14 @@ def pull(repo, remote_location=None, refspecs=None,
     # Open the repo
     with open_repo_closing(repo) as r:
         if remote_location is None:
-            # TODO(jelmer): Lookup 'remote' for current branch in config
-            raise NotImplementedError(
-                "looking up remote from branch config not supported yet")
+            config = r.get_config()
+            remote_name = get_branch_remote(r.path)
+            section = (b'remote', remote_name)
+
+            if config.has_section(section):
+                url = config.get(section, 'url')
+                remote_location = url.decode()
+
         if refspecs is None:
             refspecs = [b"HEAD"]
         selected_refs = []
@@ -1234,6 +1237,26 @@ def active_branch(repo):
         return active_ref[len(LOCAL_BRANCH_PREFIX):]
 
 
+def get_branch_remote(repo):
+    """Return the active branch's remote name, if any.
+
+    Args:
+      repo: Repository to open
+    Returns:
+      remote name
+    Raises:
+      KeyError: if the repository does not have a working tree
+    """
+    with open_repo_closing(repo) as r:
+        branch_name = active_branch(r.path)
+        config = r.get_config()
+        try:
+            remote_name = config.get((b'branch', branch_name), 'remote')
+        except KeyError:
+            remote_name = b'origin'
+    return remote_name
+
+
 def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout,
           errstream=default_bytes_err_stream, message=None, depth=None,
           prune=False, prune_tags=False, **kwargs):
@@ -1428,12 +1451,9 @@ def check_mailmap(repo, contact):
     """
     with open_repo_closing(repo) as r:
         from dulwich.mailmap import Mailmap
-        import errno
         try:
             mailmap = Mailmap.from_path(os.path.join(r.path, '.mailmap'))
-        except IOError as e:
-            if e.errno != errno.ENOENT:
-                raise
+        except FileNotFoundError:
             mailmap = Mailmap()
         return mailmap.lookup(contact)
 

dulwich/protocol.py

@@ -48,6 +48,7 @@ SIDE_BAND_CHANNEL_PROGRESS = 2
 # fatal error message just before stream aborts
 SIDE_BAND_CHANNEL_FATAL = 3
 
+CAPABILITY_ATOMIC = b'atomic'
 CAPABILITY_DEEPEN_SINCE = b'deepen-since'
 CAPABILITY_DEEPEN_NOT = b'deepen-not'
 CAPABILITY_DEEPEN_RELATIVE = b'deepen-relative'
@@ -89,7 +90,11 @@ KNOWN_UPLOAD_CAPABILITIES = set(COMMON_CAPABILITIES + [
     CAPABILITY_DEEPEN_RELATIVE,
     ])
 KNOWN_RECEIVE_CAPABILITIES = set(COMMON_CAPABILITIES + [
-    CAPABILITY_REPORT_STATUS])
+    CAPABILITY_REPORT_STATUS,
+    CAPABILITY_DELETE_REFS,
+    CAPABILITY_QUIET,
+    CAPABILITY_ATOMIC,
+    ])
 
 
 def agent_string():

dulwich/reflog.py

@@ -75,5 +75,5 @@ def read_reflog(f):
       f: File-like object
     Returns: Iterator over Entry objects
     """
-    for l in f:
-        yield parse_reflog_line(l)
+    for line in f:
+        yield parse_reflog_line(line)

dulwich/refs.py

@@ -22,9 +22,7 @@
 """Ref handling.
 
 """
-import errno
 import os
-import sys
 
 from dulwich.errors import (
     PackedRefsException,
@@ -472,8 +470,8 @@ class InfoRefsContainer(RefsContainer):
     def __init__(self, f):
         self._refs = {}
         self._peeled = {}
-        for l in f.readlines():
-            sha, name = l.rstrip(b'\n').split(b'\t')
+        for line in f.readlines():
+            sha, name = line.rstrip(b'\n').split(b'\t')
             if name.endswith(ANNOTATED_TAG_SUFFIX):
                 name = name[:-3]
                 if not check_ref_format(name):
@@ -506,12 +504,12 @@ class DiskRefsContainer(RefsContainer):
     def __init__(self, path, worktree_path=None, logger=None):
         super(DiskRefsContainer, self).__init__(logger=logger)
         if getattr(path, 'encode', None) is not None:
-            path = path.encode(sys.getfilesystemencoding())
+            path = os.fsencode(path)
         self.path = path
         if worktree_path is None:
             worktree_path = path
         if getattr(worktree_path, 'encode', None) is not None:
-            worktree_path = worktree_path.encode(sys.getfilesystemencoding())
+            worktree_path = os.fsencode(worktree_path)
         self.worktree_path = worktree_path
         self._packed_refs = None
         self._peeled_refs = None
@@ -525,8 +523,7 @@ class DiskRefsContainer(RefsContainer):
         for root, unused_dirs, files in os.walk(path):
             dir = root[len(path):]
             if os.path.sep != '/':
-                dir = dir.replace(os.path.sep.encode(
-                    sys.getfilesystemencoding()), b"/")
+                dir = dir.replace(os.fsencode(os.path.sep), b"/")
             dir = dir.strip(b'/')
             for filename in files:
                 refname = b"/".join(([dir] if dir else []) + [filename])
@@ -548,8 +545,7 @@ class DiskRefsContainer(RefsContainer):
         for root, unused_dirs, files in os.walk(refspath):
             dir = root[len(path):]
             if os.path.sep != '/':
-                dir = dir.replace(
-                    os.path.sep.encode(sys.getfilesystemencoding()), b"/")
+                dir = dir.replace(os.fsencode(os.path.sep), b"/")
             for filename in files:
                 refname = b"/".join([dir, filename])
                 if check_ref_format(refname):
@@ -562,9 +558,7 @@ class DiskRefsContainer(RefsContainer):
 
         """
         if os.path.sep != "/":
-            name = name.replace(
-                    b"/",
-                    os.path.sep.encode(sys.getfilesystemencoding()))
+            name = name.replace(b"/", os.fsencode(os.path.sep))
         # TODO: as the 'HEAD' reference is working tree specific, it
         # should actually not be a part of RefsContainer
         if name == b'HEAD':
@@ -589,10 +583,8 @@ class DiskRefsContainer(RefsContainer):
             path = os.path.join(self.path, b'packed-refs')
             try:
                 f = GitFile(path, 'rb')
-            except IOError as e:
-                if e.errno == errno.ENOENT:
-                    return {}
-                raise
+            except FileNotFoundError:
+                return {}
             with f:
                 first_line = next(iter(f)).rstrip()
                 if (first_line.startswith(b'# pack-refs') and b' peeled' in
@@ -649,10 +641,8 @@ class DiskRefsContainer(RefsContainer):
                 else:
                     # Read only the first 40 bytes
                     return header + f.read(40 - len(SYMREF))
-        except IOError as e:
-            if e.errno in (errno.ENOENT, errno.EISDIR, errno.ENOTDIR):
-                return None
-            raise
+        except (FileNotFoundError, IsADirectoryError, NotADirectoryError):
+            return None
 
     def _remove_packed_ref(self, name):
         if self._packed_refs is None:
@@ -728,8 +718,7 @@ class DiskRefsContainer(RefsContainer):
         packed_refs = self.get_packed_refs()
         while probe_ref:
             if packed_refs.get(probe_ref, None) is not None:
-                raise OSError(errno.ENOTDIR,
-                              'Not a directory: {}'.format(filename))
+                raise NotADirectoryError(filename)
             probe_ref = os.path.dirname(probe_ref)
 
         ensure_dir_exists(os.path.dirname(filename))
@@ -823,9 +812,8 @@ class DiskRefsContainer(RefsContainer):
             # remove the reference file itself
             try:
                 os.remove(filename)
-            except OSError as e:
-                if e.errno != errno.ENOENT:  # may only be packed
-                    raise
+            except FileNotFoundError:
+                pass  # may only be packed
 
             self._remove_packed_ref(name)
             self._log(name, old_ref, None, committer=committer,
@@ -877,14 +865,14 @@ def read_packed_refs(f):
       f: file-like object to read from
     Returns: Iterator over tuples with SHA1s and ref names.
     """
-    for l in f:
-        if l.startswith(b'#'):
+    for line in f:
+        if line.startswith(b'#'):
             # Comment
             continue
-        if l.startswith(b'^'):
+        if line.startswith(b'^'):
             raise PackedRefsException(
               "found peeled ref in packed-refs without peeled")
-        yield _split_ref_line(l)
+        yield _split_ref_line(line)
 
 
 def read_packed_refs_with_peeled(f):
@@ -939,8 +927,8 @@ def write_packed_refs(f, packed_refs, peeled_refs=None):
 
 def read_info_refs(f):
     ret = {}
-    for l in f.readlines():
-        (sha, name) = l.rstrip(b"\r\n").split(b"\t", 1)
+    for line in f.readlines():
+        (sha, name) = line.rstrip(b"\r\n").split(b"\t", 1)
         ret[name] = sha
     return ret
 

dulwich/repo.py

@@ -29,7 +29,6 @@ local disk (Repo).
 """
 
 from io import BytesIO
-import errno
 import os
 import sys
 import stat
@@ -70,6 +69,7 @@ from dulwich.hooks import (
     PreCommitShellHook,
     PostCommitShellHook,
     CommitMsgShellHook,
+    PostReceiveShellHook,
     )
 
 from dulwich.line_ending import BlobNormalizer
@@ -211,8 +211,8 @@ def parse_graftpoints(graftpoints):
     https://git.wiki.kernel.org/index.php/GraftPoint
     """
     grafts = {}
-    for l in graftpoints:
-        raw_graft = l.split(None, 1)
+    for line in graftpoints:
+        raw_graft = line.split(None, 1)
 
         commit = raw_graft[0]
         if len(raw_graft) == 2:
@@ -263,7 +263,7 @@ def _set_filesystem_hidden(path):
             ("SetFileAttributesW", ctypes.windll.kernel32))
 
         if isinstance(path, bytes):
-            path = path.decode(sys.getfilesystemencoding())
+            path = os.fsdecode(path)
         if not SetFileAttributesW(path, FILE_ATTRIBUTE_HIDDEN):
             pass  # Could raise or log `ctypes.WinError()` here
 
@@ -467,10 +467,23 @@ class BaseRepo(object):
 
         return self.object_store.iter_shas(
           self.object_store.find_missing_objects(
-              haves, wants, progress,
-              get_tagged,
+              haves, wants, self.get_shallow(),
+              progress, get_tagged,
               get_parents=get_parents))
 
+    def generate_pack_data(self, have, want, progress=None, ofs_delta=None):
+        """Generate pack data objects for a set of wants/haves.
+
+        Args:
+          have: List of SHA1s of objects that should not be sent
+          want: List of SHA1s of objects that should be sent
+          ofs_delta: Whether OFS deltas can be included
+          progress: Optional progress reporting method
+        """
+        return self.object_store.generate_pack_data(
+            have, want, shallow=self.get_shallow(),
+            progress=progress, ofs_delta=ofs_delta)
+
     def get_graph_walker(self, heads=None):
         """Retrieve a graph walker.
 
@@ -591,7 +604,7 @@ class BaseRepo(object):
         if f is None:
             return set()
         with f:
-            return set(l.strip() for l in f)
+            return set(line.strip() for line in f)
 
     def update_shallow(self, new_shallow, new_unshallow):
         """Update the list of shallow objects.
@@ -755,7 +768,7 @@ class BaseRepo(object):
         if f is None:
             return []
         with f:
-            return [l.strip() for l in f.readlines() if l.strip()]
+            return [line.strip() for line in f.readlines() if line.strip()]
 
     def do_commit(self, message=None, committer=None,
                   author=None, commit_timestamp=None,
@@ -929,13 +942,14 @@ class Repo(BaseRepo):
             with commondir:
                 self._commondir = os.path.join(
                     self.controldir(),
-                    commondir.read().rstrip(b"\r\n").decode(
-                        sys.getfilesystemencoding()))
+                    os.fsdecode(commondir.read().rstrip(b"\r\n")))
         else:
             self._commondir = self._controldir
         self.path = root
-        object_store = DiskObjectStore(
-            os.path.join(self.commondir(), OBJECTDIR))
+        config = self.get_config()
+        object_store = DiskObjectStore.from_config(
+            os.path.join(self.commondir(), OBJECTDIR),
+            config)
         refs = DiskRefsContainer(self.commondir(), self._controldir,
                                  logger=self._write_reflog)
         BaseRepo.__init__(self, object_store, refs)
@@ -955,18 +969,16 @@ class Repo(BaseRepo):
         self.hooks['pre-commit'] = PreCommitShellHook(self.controldir())
         self.hooks['commit-msg'] = CommitMsgShellHook(self.controldir())
         self.hooks['post-commit'] = PostCommitShellHook(self.controldir())
+        self.hooks['post-receive'] = PostReceiveShellHook(self.controldir())
 
     def _write_reflog(self, ref, old_sha, new_sha, committer, timestamp,
                       timezone, message):
         from .reflog import format_reflog_line
-        path = os.path.join(
-                self.controldir(), 'logs',
-                ref.decode(sys.getfilesystemencoding()))
+        path = os.path.join(self.controldir(), 'logs', os.fsdecode(ref))
         try:
             os.makedirs(os.path.dirname(path))
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
+        except FileExistsError:
+            pass
         if committer is None:
             config = self.get_config_stack()
             committer = self._get_user_identity(config)
@@ -1026,10 +1038,8 @@ class Repo(BaseRepo):
         st1 = os.lstat(fname)
         try:
             os.chmod(fname, st1.st_mode ^ stat.S_IXUSR)
-        except EnvironmentError as e:
-            if e.errno == errno.EPERM:
-                return False
-            raise
+        except PermissionError:
+            return False
         st2 = os.lstat(fname)
 
         os.unlink(fname)
@@ -1053,10 +1063,8 @@ class Repo(BaseRepo):
     def _del_named_file(self, path):
         try:
             os.unlink(os.path.join(self.controldir(), path))
-        except (IOError, OSError) as e:
-            if e.errno == errno.ENOENT:
-                return
-            raise
+        except FileNotFoundError:
+            return
 
     def get_named_file(self, path, basedir=None):
         """Get a file from the control dir with a specific name.
@@ -1078,10 +1086,8 @@ class Repo(BaseRepo):
         path = path.lstrip(os.path.sep)
         try:
             return open(os.path.join(basedir, path), 'rb')
-        except (IOError, OSError) as e:
-            if e.errno == errno.ENOENT:
-                return None
-            raise
+        except FileNotFoundError:
+            return None
 
     def index_path(self):
         """Return path to the index file."""
@@ -1112,7 +1118,7 @@ class Repo(BaseRepo):
           fs_paths: List of paths, relative to the repository path
         """
 
-        root_path_bytes = self.path.encode(sys.getfilesystemencoding())
+        root_path_bytes = os.fsencode(self.path)
 
         if not isinstance(fs_paths, list):
             fs_paths = [fs_paths]
@@ -1125,7 +1131,7 @@ class Repo(BaseRepo):
         blob_normalizer = self.get_blob_normalizer()
         for fs_path in fs_paths:
             if not isinstance(fs_path, bytes):
-                fs_path = fs_path.encode(sys.getfilesystemencoding())
+                fs_path = os.fsencode(fs_path)
             if os.path.isabs(fs_path):
                 raise ValueError(
                     "path %r should be relative to "
@@ -1141,16 +1147,17 @@ class Repo(BaseRepo):
                 except KeyError:
                     pass  # already removed
             else:
-                if not stat.S_ISDIR(st.st_mode):
-                    blob = blob_from_path_and_stat(full_path, st)
-                    blob = blob_normalizer.checkin_normalize(blob, fs_path)
-                    self.object_store.add_object(blob)
-                    index[tree_path] = index_entry_from_stat(st, blob.id, 0)
-                else:
+                if (not stat.S_ISREG(st.st_mode) and
+                        not stat.S_ISLNK(st.st_mode)):
                     try:
                         del index[tree_path]
                     except KeyError:
                         pass
+                else:
+                    blob = blob_from_path_and_stat(full_path, st)
+                    blob = blob_normalizer.checkin_normalize(blob, fs_path)
+                    self.object_store.add_object(blob)
+                    index[tree_path] = index_entry_from_stat(st, blob.id, 0)
         index.write()
 
     def clone(self, target_path, mkdir=True, bare=False,
@@ -1174,7 +1181,7 @@ class Repo(BaseRepo):
         self.fetch(target)
         encoded_path = self.path
         if not isinstance(encoded_path, bytes):
-            encoded_path = encoded_path.encode(sys.getfilesystemencoding())
+            encoded_path = os.fsencode(encoded_path)
         ref_message = b"clone: from " + encoded_path
         target.refs.import_refs(
             b'refs/remotes/' + origin, self.refs.as_dict(b'refs/heads'),
@@ -1243,9 +1250,7 @@ class Repo(BaseRepo):
         path = os.path.join(self._controldir, 'config')
         try:
             return ConfigFile.from_path(path)
-        except (IOError, OSError) as e:
-            if e.errno != errno.ENOENT:
-                raise
+        except FileNotFoundError:
             ret = ConfigFile()
             ret.path = path
             return ret
@@ -1259,9 +1264,7 @@ class Repo(BaseRepo):
         try:
             with GitFile(path, 'rb') as f:
                 return f.read()
-        except (IOError, OSError) as e:
-            if e.errno != errno.ENOENT:
-                raise
+        except FileNotFoundError:
             return None
 
     def __repr__(self):
@@ -1323,21 +1326,17 @@ class Repo(BaseRepo):
         worktree_controldir = os.path.join(main_worktreesdir, identifier)
         gitdirfile = os.path.join(path, CONTROLDIR)
         with open(gitdirfile, 'wb') as f:
-            f.write(b'gitdir: ' +
-                    worktree_controldir.encode(sys.getfilesystemencoding()) +
-                    b'\n')
+            f.write(b'gitdir: ' + os.fsencode(worktree_controldir) + b'\n')
         try:
             os.mkdir(main_worktreesdir)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
+        except FileExistsError:
+            pass
         try:
             os.mkdir(worktree_controldir)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
+        except FileExistsError:
+            pass
         with open(os.path.join(worktree_controldir, GITDIR), 'wb') as f:
-            f.write(gitdirfile.encode(sys.getfilesystemencoding()) + b'\n')
+            f.write(os.fsencode(gitdirfile) + b'\n')
         with open(os.path.join(worktree_controldir, COMMONDIR), 'wb') as f:
             f.write(b'../..\n')
         with open(os.path.join(worktree_controldir, 'HEAD'), 'wb') as f:

dulwich/server.py

@@ -47,18 +47,17 @@ import os
 import socket
 import sys
 import time
+from typing import List, Tuple, Dict, Optional, Iterable
 import zlib
 
-try:
-    import SocketServer
-except ImportError:
-    import socketserver as SocketServer
+import socketserver
 
 from dulwich.archive import tar_stream
 from dulwich.errors import (
     ApplyDeltaError,
     ChecksumMismatch,
     GitProtocolError,
+    HookError,
     NotGitRepository,
     UnexpectedCommandError,
     ObjectFormatException,
@@ -75,6 +74,7 @@ from dulwich.protocol import (  # noqa: F401
     BufferedPktLineWriter,
     capability_agent,
     CAPABILITIES_REF,
+    CAPABILITY_AGENT,
     CAPABILITY_DELETE_REFS,
     CAPABILITY_INCLUDE_TAG,
     CAPABILITY_MULTI_ACK_DETAILED,
@@ -114,6 +114,7 @@ from dulwich.refs import (
     write_info_refs,
     )
 from dulwich.repo import (
+    BaseRepo,
     Repo,
     )
 
@@ -146,7 +147,7 @@ class BackendRepo(object):
     object_store = None
     refs = None
 
-    def get_refs(self):
+    def get_refs(self) -> Dict[bytes, bytes]:
         """
         Get all the refs in the repository
 
@@ -154,7 +155,7 @@ class BackendRepo(object):
         """
         raise NotImplementedError
 
-    def get_peeled(self, name):
+    def get_peeled(self, name: bytes) -> Optional[bytes]:
         """Return the cached peeled value of a ref, if available.
 
         Args:
@@ -185,7 +186,7 @@ class DictBackend(Backend):
     def __init__(self, repos):
         self.repos = repos
 
-    def open_repository(self, path):
+    def open_repository(self, path: str) -> BaseRepo:
         logger.debug('Opening repository at %s', path)
         try:
             return self.repos[path]
@@ -242,41 +243,43 @@ class PackHandler(Handler):
         return b"".join([b" " + c for c in capabilities])
 
     @classmethod
-    def capabilities(cls):
+    def capabilities(cls) -> Iterable[bytes]:
         raise NotImplementedError(cls.capabilities)
 
     @classmethod
-    def innocuous_capabilities(cls):
+    def innocuous_capabilities(cls) -> Iterable[bytes]:
         return [CAPABILITY_INCLUDE_TAG, CAPABILITY_THIN_PACK,
                 CAPABILITY_NO_PROGRESS, CAPABILITY_OFS_DELTA,
                 capability_agent()]
 
     @classmethod
-    def required_capabilities(cls):
+    def required_capabilities(cls) -> Iterable[bytes]:
         """Return a list of capabilities that we require the client to have."""
         return []
 
-    def set_client_capabilities(self, caps):
+    def set_client_capabilities(self, caps: Iterable[bytes]) -> None:
         allowable_caps = set(self.innocuous_capabilities())
         allowable_caps.update(self.capabilities())
         for cap in caps:
+            if cap.startswith(CAPABILITY_AGENT + b'='):
+                continue
             if cap not in allowable_caps:
-                raise GitProtocolError('Client asked for capability %s that '
+                raise GitProtocolError('Client asked for capability %r that '
                                        'was not advertised.' % cap)
         for cap in self.required_capabilities():
             if cap not in caps:
                 raise GitProtocolError('Client does not support required '
-                                       'capability %s.' % cap)
+                                       'capability %r.' % cap)
         self._client_capabilities = set(caps)
         logger.info('Client capabilities: %s', caps)
 
-    def has_capability(self, cap):
+    def has_capability(self, cap: bytes) -> bool:
         if self._client_capabilities is None:
-            raise GitProtocolError('Server attempted to access capability %s '
+            raise GitProtocolError('Server attempted to access capability %r '
                                    'before asking client' % cap)
         return cap in self._client_capabilities
 
-    def notify_done(self):
+    def notify_done(self) -> None:
         self._done_received = True
 
 
@@ -901,12 +904,14 @@ class ReceivePackHandler(PackHandler):
         self.advertise_refs = advertise_refs
 
     @classmethod
-    def capabilities(cls):
+    def capabilities(cls) -> Iterable[bytes]:
         return [CAPABILITY_REPORT_STATUS, CAPABILITY_DELETE_REFS,
                 CAPABILITY_QUIET, CAPABILITY_OFS_DELTA,
                 CAPABILITY_SIDE_BAND_64K, CAPABILITY_NO_DONE]
 
-    def _apply_pack(self, refs):
+    def _apply_pack(
+            self, refs: List[Tuple[bytes, bytes, bytes]]
+            ) -> List[Tuple[bytes, bytes]]:
         all_exceptions = (IOError, OSError, ChecksumMismatch, ApplyDeltaError,
                           AssertionError, socket.error, zlib.error,
                           ObjectFormatException)
@@ -925,7 +930,8 @@ class ReceivePackHandler(PackHandler):
                 self.repo.object_store.add_thin_pack(self.proto.read, recv)
                 status.append((b'unpack', b'ok'))
             except all_exceptions as e:
-                status.append((b'unpack', str(e).replace('\n', '')))
+                status.append(
+                    (b'unpack', str(e).replace('\n', '').encode('utf-8')))
                 # The pack may still have been moved in, but it may contain
                 # broken objects. We trust a later GC to clean it up.
         else:
@@ -956,7 +962,7 @@ class ReceivePackHandler(PackHandler):
 
         return status
 
-    def _report_status(self, status):
+    def _report_status(self, status: List[Tuple[bytes, bytes]]) -> None:
         if self.has_capability(CAPABILITY_SIDE_BAND_64K):
             writer = BufferedPktLineWriter(
               lambda d: self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, d))
@@ -981,7 +987,18 @@ class ReceivePackHandler(PackHandler):
         write(None)
         flush()
 
-    def handle(self):
+    def _on_post_receive(self, client_refs):
+        hook = self.repo.hooks.get('post-receive', None)
+        if not hook:
+            return
+        try:
+            output = hook.execute(client_refs)
+            if output:
+                self.proto.write_sideband(SIDE_BAND_CHANNEL_PROGRESS, output)
+        except HookError as err:
+            self.proto.write_sideband(SIDE_BAND_CHANNEL_FATAL, repr(err))
+
+    def handle(self) -> None:
         if self.advertise_refs or not self.http_req:
             refs = sorted(self.repo.get_refs().items())
             symrefs = sorted(self.repo.refs.get_symrefs().items())
@@ -1018,6 +1035,8 @@ class ReceivePackHandler(PackHandler):
         # backend can now deal with this refs and read a pack using self.read
         status = self._apply_pack(client_refs)
 
+        self._on_post_receive(client_refs)
+
         # when we have read all the pack from the client, send a status report
         # if the client asked for it
         if self.has_capability(CAPABILITY_REPORT_STATUS):
@@ -1071,11 +1090,11 @@ DEFAULT_HANDLERS = {
 }
 
 
-class TCPGitRequestHandler(SocketServer.StreamRequestHandler):
+class TCPGitRequestHandler(socketserver.StreamRequestHandler):
 
     def __init__(self, handlers, *args, **kwargs):
         self.handlers = handlers
-        SocketServer.StreamRequestHandler.__init__(self, *args, **kwargs)
+        socketserver.StreamRequestHandler.__init__(self, *args, **kwargs)
 
     def handle(self):
         proto = ReceivableProtocol(self.connection.recv, self.wfile.write)
@@ -1089,10 +1108,10 @@ class TCPGitRequestHandler(SocketServer.StreamRequestHandler):
         h.handle()
 
 
-class TCPGitServer(SocketServer.TCPServer):
+class TCPGitServer(socketserver.TCPServer):
 
     allow_reuse_address = True
-    serve = SocketServer.TCPServer.serve_forever
+    serve = socketserver.TCPServer.serve_forever
 
     def _make_handler(self, *args, **kwargs):
         return TCPGitRequestHandler(self.handlers, *args, **kwargs)
@@ -1104,7 +1123,7 @@ class TCPGitServer(SocketServer.TCPServer):
         self.backend = backend
         logger.info('Listening for TCP connections on %s:%d',
                     listen_addr, port)
-        SocketServer.TCPServer.__init__(self, (listen_addr, port),
+        socketserver.TCPServer.__init__(self, (listen_addr, port),
                                         self._make_handler)
 
     def verify_request(self, request, client_address):
@@ -1177,7 +1196,7 @@ def generate_objects_info_packs(repo):
     """Generate an index for for packs."""
     for pack in repo.object_store.packs:
         yield (
-            b'P ' + pack.data.filename.encode(sys.getfilesystemencoding()) +
+            b'P ' + os.fsencode(pack.data.filename) +
             b'\n')
 
 

dulwich/stash.py

@@ -22,7 +22,6 @@
 
 from __future__ import absolute_import
 
-import errno
 import os
 
 from dulwich.file import GitFile
@@ -52,10 +51,8 @@ class Stash(object):
         try:
             with GitFile(reflog_path, 'rb') as f:
                 return reversed(list(read_reflog(f)))
-        except EnvironmentError as e:
-            if e.errno == errno.ENOENT:
-                return []
-            raise
+        except FileNotFoundError:
+            return []
 
     @classmethod
     def from_repo(cls, repo):

dulwich/tests/__init__.py

@@ -112,6 +112,7 @@ def self_test_suite():
         'hooks',
         'ignore',
         'index',
+        'lfs',
         'line_ending',
         'lru_cache',
         'mailmap',

dulwich/tests/compat/test_client.py

@@ -25,25 +25,16 @@ from io import BytesIO
 import os
 import select
 import signal
+import stat
 import subprocess
 import sys
 import tarfile
 import tempfile
 import threading
 
-try:
-    from urlparse import unquote
-except ImportError:
-    from urllib.parse import unquote
+from urllib.parse import unquote
 
-
-try:
-    import BaseHTTPServer
-    import SimpleHTTPServer
-except ImportError:
-    import http.server
-    BaseHTTPServer = http.server
-    SimpleHTTPServer = http.server
+import http.server
 
 from dulwich import (
     client,
@@ -105,7 +96,7 @@ class DulwichClientTestBase(object):
             sendrefs = dict(src.get_refs())
             del sendrefs[b'HEAD']
             c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
-                        src.object_store.generate_pack_data)
+                        src.generate_pack_data)
 
     def test_send_pack(self):
         self._do_send_pack()
@@ -117,6 +108,43 @@ class DulwichClientTestBase(object):
         # nothing to send, but shouldn't raise either.
         self._do_send_pack()
 
+    @staticmethod
+    def _add_file(repo, tree_id, filename, contents):
+        tree = repo[tree_id]
+        blob = objects.Blob()
+        blob.data = contents.encode('utf-8')
+        repo.object_store.add_object(blob)
+        tree.add(filename.encode('utf-8'), stat.S_IFREG | 0o644, blob.id)
+        repo.object_store.add_object(tree)
+        return tree.id
+
+    def test_send_pack_from_shallow_clone(self):
+        c = self._client()
+        server_new_path = os.path.join(self.gitroot, 'server_new.export')
+        run_git_or_fail(['config', 'http.uploadpack', 'true'],
+                        cwd=server_new_path)
+        run_git_or_fail(['config', 'http.receivepack', 'true'],
+                        cwd=server_new_path)
+        remote_path = self._build_path('/server_new.export')
+        with repo.Repo(self.dest) as local:
+            result = c.fetch(remote_path, local, depth=1)
+            for r in result.refs.items():
+                local.refs.set_if_equals(r[0], None, r[1])
+            tree_id = local[local.head()].tree
+            for filename, contents in [('bar', 'bar contents'),
+                                       ('zop', 'zop contents')]:
+                tree_id = self._add_file(local, tree_id, filename, contents)
+                commit_id = local.do_commit(
+                    message=b"add " + filename.encode('utf-8'),
+                    committer=b"Joe Example <joe@example.com>",
+                    tree=tree_id)
+            sendrefs = dict(local.get_refs())
+            del sendrefs[b'HEAD']
+            c.send_pack(remote_path, lambda _: sendrefs,
+                        local.generate_pack_data)
+        with repo.Repo(server_new_path) as remote:
+            self.assertEqual(remote.head(), commit_id)
+
     def test_send_without_report_status(self):
         c = self._client()
         c._send_capabilities.remove(b'report-status')
@@ -125,7 +153,7 @@ class DulwichClientTestBase(object):
             sendrefs = dict(src.get_refs())
             del sendrefs[b'HEAD']
             c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
-                        src.object_store.generate_pack_data)
+                        src.generate_pack_data)
             self.assertDestEqualsSrc()
 
     def make_dummy_commit(self, dest):
@@ -152,7 +180,7 @@ class DulwichClientTestBase(object):
     def compute_send(self, src):
         sendrefs = dict(src.get_refs())
         del sendrefs[b'HEAD']
-        return sendrefs, src.object_store.generate_pack_data
+        return sendrefs, src.generate_pack_data
 
     def test_send_pack_one_error(self):
         dest, dummy_commit = self.disable_ff_and_make_dummy_commit()
@@ -162,8 +190,8 @@ class DulwichClientTestBase(object):
             sendrefs, gen_pack = self.compute_send(src)
             c = self._client()
             try:
-                c.send_pack(self._build_path('/dest'),
-                            lambda _: sendrefs, gen_pack)
+                c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
+                            gen_pack)
             except errors.UpdateRefsError as e:
                 self.assertEqual('refs/heads/master failed to update',
                                  e.args[0])
@@ -398,7 +426,7 @@ class DulwichSubprocessClientTest(CompatTestCase, DulwichClientTestBase):
         return self.gitroot + path
 
 
-class GitHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+class GitHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
     """HTTP Request handler that calls out to 'git http-backend'."""
 
     # Make rfile unbuffered -- we need to read one line and then pass
@@ -525,12 +553,12 @@ class GitHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
         self.wfile.write(stdout)
 
 
-class HTTPGitServer(BaseHTTPServer.HTTPServer):
+class HTTPGitServer(http.server.HTTPServer):
 
     allow_reuse_address = True
 
     def __init__(self, server_address, root_path):
-        BaseHTTPServer.HTTPServer.__init__(
+        http.server.HTTPServer.__init__(
             self, server_address, GitHTTPRequestHandler)
         self.root_path = root_path
         self.server_name = "localhost"

dulwich/tests/compat/test_repository.py

@@ -182,16 +182,17 @@ class WorkingTreeTestCase(ObjectStoreTestCase):
         worktrees = self._parse_worktree_list(output)
         self.assertEqual(len(worktrees), self._number_of_working_tree)
         self.assertEqual(worktrees[0][1], '(bare)')
-        self.assertEqual(os.path.normcase(worktrees[0][0]),
-                         os.path.normcase(self._mainworktree_repo.path))
+        self.assertTrue(
+            os.path.samefile(worktrees[0][0], self._mainworktree_repo.path))
 
         output = run_git_or_fail(
             ['worktree', 'list'], cwd=self._mainworktree_repo.path)
         worktrees = self._parse_worktree_list(output)
         self.assertEqual(len(worktrees), self._number_of_working_tree)
         self.assertEqual(worktrees[0][1], '(bare)')
-        self.assertEqual(os.path.normcase(worktrees[0][0]),
-                         os.path.normcase(self._mainworktree_repo.path))
+        self.assertTrue(os.path.samefile(
+            worktrees[0][0],
+            self._mainworktree_repo.path))
 
 
 class InitNewWorkingDirectoryTestCase(WorkingTreeTestCase):

dulwich/tests/compat/test_web.py

@@ -28,6 +28,7 @@ warning: these tests should be fairly stable, but when writing/debugging new
 import threading
 from wsgiref import simple_server
 import sys
+from typing import Tuple
 
 from dulwich.server import (
     DictBackend,
@@ -87,7 +88,7 @@ class SmartWebTestCase(WebTests, CompatTestCase):
     This server test case does not use side-band-64k in git-receive-pack.
     """
 
-    min_git_version = (1, 6, 6)
+    min_git_version = (1, 6, 6)  # type: Tuple[int, ...]
 
     def _handlers(self):
         return {b'git-receive-pack': NoSideBand64kReceivePackHandler}

dulwich/tests/compat/utils.py

@@ -30,6 +30,7 @@ import subprocess
 import sys
 import tempfile
 import time
+from typing import Tuple
 
 from dulwich.repo import Repo
 from dulwich.protocol import TCP_GIT_PORT
@@ -215,7 +216,7 @@ class CompatTestCase(TestCase):
     min_git_version.
     """
 
-    min_git_version = (1, 5, 0)
+    min_git_version = (1, 5, 0)  # type: Tuple[int, ...]
 
     def setUp(self):
         super(CompatTestCase, self).setUp()

dulwich/tests/test_archive.py

@@ -43,7 +43,7 @@ from dulwich.tests.utils import (
 try:
     from mock import patch
 except ImportError:
-    patch = None
+    patch = None   # type: ignore
 
 
 class ArchiveTests(TestCase):

dulwich/tests/test_client.py

@@ -20,22 +20,16 @@
 
 from io import BytesIO
 import base64
+import os
 import sys
 import shutil
 import tempfile
 import warnings
 
-try:
-    from urllib import quote as urlquote
-except ImportError:
-    from urllib.parse import quote as urlquote
-
-try:
-    import urlparse
-except ImportError:
-    import urllib.parse as urlparse
-
-import urllib3
+from urllib.parse import (
+    quote as urlquote,
+    urlparse,
+    )
 
 import dulwich
 from dulwich import (
@@ -57,6 +51,7 @@ from dulwich.client import (
     UpdateRefsError,
     check_wants,
     default_urllib3_manager,
+    get_credentials_from_store,
     get_transport_and_path,
     get_transport_and_path_from_url,
     parse_rsync_url,
@@ -138,9 +133,10 @@ class GitClientTests(TestCase):
                               b'thin-pack', b'multi_ack_detailed', b'shallow',
                               agent_cap]),
                          set(self.client._fetch_capabilities))
-        self.assertEqual(set([b'ofs-delta', b'report-status', b'side-band-64k',
-                              agent_cap]),
-                         set(self.client._send_capabilities))
+        self.assertEqual(
+            set([b'delete-refs', b'ofs-delta', b'report-status',
+                 b'side-band-64k', agent_cap]),
+            set(self.client._send_capabilities))
 
     def test_archive_ack(self):
         self.rin.write(
@@ -246,7 +242,7 @@ class GitClientTests(TestCase):
         def generate_pack_data(have, want, ofs_delta=False):
             return 0, []
 
-        self.client.send_pack(b'/', update_refs, generate_pack_data)
+        self.client.send_pack(b'/', update_refs, set(), generate_pack_data)
         self.assertEqual(self.rout.getvalue(), b'0000')
 
     def test_send_pack_keep_and_delete(self):
@@ -266,14 +262,11 @@ class GitClientTests(TestCase):
             return 0, []
 
         self.client.send_pack(b'/', update_refs, generate_pack_data)
-        self.assertIn(
+        self.assertEqual(
             self.rout.getvalue(),
-            [b'007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
-             b'0000000000000000000000000000000000000000 '
-             b'refs/heads/master\x00report-status ofs-delta0000',
-             b'007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
-             b'0000000000000000000000000000000000000000 '
-             b'refs/heads/master\x00ofs-delta report-status0000'])
+            b'008b310ca9477129b8586fa2afc779c1f57cf64bba6c '
+            b'0000000000000000000000000000000000000000 '
+            b'refs/heads/master\x00delete-refs ofs-delta report-status0000')
 
     def test_send_pack_delete_only(self):
         self.rin.write(
@@ -291,14 +284,11 @@ class GitClientTests(TestCase):
             return 0, []
 
         self.client.send_pack(b'/', update_refs, generate_pack_data)
-        self.assertIn(
+        self.assertEqual(
             self.rout.getvalue(),
-            [b'007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
-             b'0000000000000000000000000000000000000000 '
-             b'refs/heads/master\x00report-status ofs-delta0000',
-             b'007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
-             b'0000000000000000000000000000000000000000 '
-             b'refs/heads/master\x00ofs-delta report-status0000'])
+            b'008b310ca9477129b8586fa2afc779c1f57cf64bba6c '
+            b'0000000000000000000000000000000000000000 '
+            b'refs/heads/master\x00delete-refs ofs-delta report-status0000')
 
     def test_send_pack_new_ref_only(self):
         self.rin.write(
@@ -323,16 +313,12 @@ class GitClientTests(TestCase):
         f = BytesIO()
         write_pack_objects(f, {})
         self.client.send_pack('/', update_refs, generate_pack_data)
-        self.assertIn(
+        self.assertEqual(
             self.rout.getvalue(),
-            [b'007f0000000000000000000000000000000000000000 '
-             b'310ca9477129b8586fa2afc779c1f57cf64bba6c '
-             b'refs/heads/blah12\x00report-status ofs-delta0000' +
-             f.getvalue(),
-             b'007f0000000000000000000000000000000000000000 '
-             b'310ca9477129b8586fa2afc779c1f57cf64bba6c '
-             b'refs/heads/blah12\x00ofs-delta report-status0000' +
-             f.getvalue()])
+            b'008b0000000000000000000000000000000000000000 '
+            b'310ca9477129b8586fa2afc779c1f57cf64bba6c '
+            b'refs/heads/blah12\x00delete-refs ofs-delta report-status0000' +
+            f.getvalue())
 
     def test_send_pack_new_ref(self):
         self.rin.write(
@@ -366,14 +352,11 @@ class GitClientTests(TestCase):
         f = BytesIO()
         write_pack_data(f, *generate_pack_data(None, None))
         self.client.send_pack(b'/', update_refs, generate_pack_data)
-        self.assertIn(
+        self.assertEqual(
             self.rout.getvalue(),
-            [b'007f0000000000000000000000000000000000000000 ' + commit.id +
-             b' refs/heads/blah12\x00report-status ofs-delta0000' +
-             f.getvalue(),
-             b'007f0000000000000000000000000000000000000000 ' + commit.id +
-             b' refs/heads/blah12\x00ofs-delta report-status0000' +
-             f.getvalue()])
+            b'008b0000000000000000000000000000000000000000 ' + commit.id +
+            b' refs/heads/blah12\x00delete-refs ofs-delta report-status0000' +
+            f.getvalue())
 
     def test_send_pack_no_deleteref_delete_only(self):
         pkts = [b'310ca9477129b8586fa2afc779c1f57cf64bba6c refs/heads/master'
@@ -885,7 +868,7 @@ class LocalGitClientTests(TestCase):
         ref_name = b"refs/heads/" + branch
         new_refs = client.send_pack(target.path,
                                     lambda _: {ref_name: local.refs[ref_name]},
-                                    local.object_store.generate_pack_data)
+                                    local.generate_pack_data)
 
         self.assertEqual(local.refs[ref_name], new_refs[ref_name])
 
@@ -896,14 +879,6 @@ class LocalGitClientTests(TestCase):
 
 class HttpGitClientTests(TestCase):
 
-    @staticmethod
-    def b64encode(s):
-        """Python 2/3 compatible Base64 encoder. Returns string."""
-        try:
-            return base64.b64encode(s)
-        except TypeError:
-            return base64.b64encode(s.encode('latin1')).decode('ascii')
-
     def test_get_url(self):
         base_url = 'https://github.com/jelmer/dulwich'
         path = '/jelmer/dulwich'
@@ -937,8 +912,8 @@ class HttpGitClientTests(TestCase):
 
         basic_auth = c.pool_manager.headers['authorization']
         auth_string = '%s:%s' % ('user', 'passwd')
-        b64_credentials = self.b64encode(auth_string)
-        expected_basic_auth = 'Basic %s' % b64_credentials
+        b64_credentials = base64.b64encode(auth_string.encode('latin1'))
+        expected_basic_auth = 'Basic %s' % b64_credentials.decode('latin1')
         self.assertEqual(basic_auth, expected_basic_auth)
 
     def test_init_no_username_passwd(self):
@@ -961,18 +936,17 @@ class HttpGitClientTests(TestCase):
             password=quoted_password
         )
 
-        c = HttpGitClient.from_parsedurl(urlparse.urlparse(url))
+        c = HttpGitClient.from_parsedurl(urlparse(url))
         self.assertEqual(original_username, c._username)
         self.assertEqual(original_password, c._password)
 
         basic_auth = c.pool_manager.headers['authorization']
         auth_string = '%s:%s' % (original_username, original_password)
-        b64_credentials = self.b64encode(auth_string)
-        expected_basic_auth = 'Basic %s' % str(b64_credentials)
+        b64_credentials = base64.b64encode(auth_string.encode('latin1'))
+        expected_basic_auth = 'Basic %s' % b64_credentials.decode('latin1')
         self.assertEqual(basic_auth, expected_basic_auth)
 
     def test_url_redirect_location(self):
-
         from urllib3.response import HTTPResponse
 
         test_data = {
@@ -1080,8 +1054,20 @@ class DefaultUrllib3ManagerTest(TestCase):
                          'CERT_REQUIRED')
 
     def test_config_no_proxy(self):
+        import urllib3
         manager = default_urllib3_manager(config=ConfigDict())
         self.assertNotIsInstance(manager, urllib3.ProxyManager)
+        self.assertIsInstance(manager, urllib3.PoolManager)
+
+    def test_config_no_proxy_custom_cls(self):
+        import urllib3
+
+        class CustomPoolManager(urllib3.PoolManager):
+            pass
+
+        manager = default_urllib3_manager(config=ConfigDict(),
+                                          pool_manager_cls=CustomPoolManager)
+        self.assertIsInstance(manager, CustomPoolManager)
 
     def test_config_ssl(self):
         config = ConfigDict()
@@ -1098,6 +1084,7 @@ class DefaultUrllib3ManagerTest(TestCase):
                          'CERT_NONE')
 
     def test_config_proxy(self):
+        import urllib3
         config = ConfigDict()
         config.set(b'http', b'proxy', b'http://localhost:3128/')
         manager = default_urllib3_manager(config=config)
@@ -1108,6 +1095,18 @@ class DefaultUrllib3ManagerTest(TestCase):
         self.assertEqual(manager.proxy.host, 'localhost')
         self.assertEqual(manager.proxy.port, 3128)
 
+    def test_config_proxy_custom_cls(self):
+        import urllib3
+
+        class CustomProxyManager(urllib3.ProxyManager):
+            pass
+
+        config = ConfigDict()
+        config.set(b'http', b'proxy', b'http://localhost:3128/')
+        manager = default_urllib3_manager(config=config,
+                                          proxy_manager_cls=CustomProxyManager)
+        self.assertIsInstance(manager, CustomProxyManager)
+
     def test_config_no_verify_ssl(self):
         manager = default_urllib3_manager(config=None, cert_reqs="CERT_NONE")
         self.assertEqual(manager.connection_pool_kw['cert_reqs'], 'CERT_NONE')
@@ -1305,3 +1304,46 @@ class FetchPackResultTests(TestCase):
                 {b'refs/heads/master':
                  b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'}, {},
                 b'user/agent'))
+
+
+class GitCredentialStoreTests(TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        with tempfile.NamedTemporaryFile(delete=False) as f:
+            f.write(b'https://user:pass@example.org')
+        cls.fname = f.name
+
+    @classmethod
+    def tearDownClass(cls):
+        os.unlink(cls.fname)
+
+    def test_nonmatching_scheme(self):
+        self.assertEqual(
+            get_credentials_from_store(
+                b'http', b'example.org', fnames=[self.fname]),
+            None)
+
+    def test_nonmatching_hostname(self):
+        self.assertEqual(
+            get_credentials_from_store(
+                b'https', b'noentry.org', fnames=[self.fname]),
+            None)
+
+    def test_match_without_username(self):
+        self.assertEqual(
+            get_credentials_from_store(
+                b'https', b'example.org', fnames=[self.fname]),
+            (b'user', b'pass'))
+
+    def test_match_with_matching_username(self):
+        self.assertEqual(
+            get_credentials_from_store(
+                b'https', b'example.org', b'user', fnames=[self.fname]),
+            (b'user', b'pass'))
+
+    def test_no_match_with_nonmatching_username(self):
+        self.assertEqual(
+            get_credentials_from_store(
+                b'https', b'example.org', b'otheruser', fnames=[self.fname]),
+            None)

dulwich/tests/test_hooks.py

@@ -22,6 +22,7 @@
 import os
 import stat
 import shutil
+import sys
 import tempfile
 
 from dulwich import errors
@@ -41,6 +42,7 @@ class ShellHookTests(TestCase):
         super(ShellHookTests, self).setUp()
         if os.name != 'posix':
             self.skipTest('shell hook tests requires POSIX shell')
+        self.assertTrue(os.path.exists('/bin/sh'))
 
     def test_hook_pre_commit(self):
         repo_dir = os.path.join(tempfile.mkdtemp())
@@ -55,7 +57,13 @@ exit 1
 exit 0
 """
         pre_commit_cwd = """#!/bin/sh
-if [ "$(pwd)" = '""" + repo_dir + "' ]; then exit 0; else exit 1; fi\n"
+if [ "$(pwd)" != '""" + repo_dir + """' ]; then
+    echo "Expected path '""" + repo_dir + """', got '$(pwd)'"
+    exit 1
+fi
+
+exit 0
+"""
 
         pre_commit = os.path.join(repo_dir, 'hooks', 'pre-commit')
         hook = PreCommitShellHook(repo_dir)
@@ -66,11 +74,14 @@ if [ "$(pwd)" = '""" + repo_dir + "' ]; then exit 0; else exit 1; fi\n"
 
         self.assertRaises(errors.HookError, hook.execute)
 
-        with open(pre_commit, 'w') as f:
-            f.write(pre_commit_cwd)
-        os.chmod(pre_commit, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+        if sys.platform != 'darwin':
+            # Don't bother running this test on darwin since path
+            # canonicalization messages with our simple string comparison.
+            with open(pre_commit, 'w') as f:
+                f.write(pre_commit_cwd)
+            os.chmod(pre_commit, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
 
-        hook.execute()
+            hook.execute()
 
         with open(pre_commit, 'w') as f:
             f.write(pre_commit_success)
@@ -104,11 +115,14 @@ if [ "$(pwd)" = '""" + repo_dir + "' ]; then exit 0; else exit 1; fi\n"
 
         self.assertRaises(errors.HookError, hook.execute, b'failed commit')
 
-        with open(commit_msg, 'w') as f:
-            f.write(commit_msg_cwd)
-        os.chmod(commit_msg, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+        if sys.platform != 'darwin':
+            # Don't bother running this test on darwin since path
+            # canonicalization messages with our simple string comparison.
+            with open(commit_msg, 'w') as f:
+                f.write(commit_msg_cwd)
+            os.chmod(commit_msg, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
 
-        hook.execute(b'cwd test commit')
+            hook.execute(b'cwd test commit')
 
         with open(commit_msg, 'w') as f:
             f.write(commit_msg_success)
@@ -144,11 +158,14 @@ if [ "$(pwd)" = '""" + repo_dir + "' ]; then exit 0; else exit 1; fi\n"
 
         self.assertRaises(errors.HookError, hook.execute)
 
-        with open(post_commit, 'w') as f:
-            f.write(post_commit_cwd)
-        os.chmod(post_commit, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+        if sys.platform != 'darwin':
+            # Don't bother running this test on darwin since path
+            # canonicalization messages with our simple string comparison.
+            with open(post_commit, 'w') as f:
+                f.write(post_commit_cwd)
+            os.chmod(post_commit, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
 
-        hook.execute()
+            hook.execute()
 
         with open(post_commit, 'w') as f:
             f.write(post_commit_success)

dulwich/tests/test_index.py

@@ -82,7 +82,7 @@ def can_symlink():
     test_target = test_source + 'can_symlink'
     try:
         os.symlink(test_source, test_target)
-    except OSError:
+    except (NotImplementedError, OSError):
         return False
     return True
 
@@ -501,7 +501,7 @@ class BuildIndexTests(TestCase):
 
     def test_no_decode_encode(self):
         repo_dir = tempfile.mkdtemp()
-        repo_dir_bytes = repo_dir.encode(sys.getfilesystemencoding())
+        repo_dir_bytes = os.fsencode(repo_dir)
         self.addCleanup(shutil.rmtree, repo_dir)
         with Repo.init(repo_dir) as repo:
 
@@ -520,16 +520,20 @@ class BuildIndexTests(TestCase):
                 [(o, None) for o in [file, tree]])
 
             try:
-                os.path.exists(latin1_path)
+                build_index_from_tree(
+                    repo.path, repo.index_path(),
+                    repo.object_store, tree.id)
+            except OSError as e:
+                if e.errno == 92 and sys.platform == 'darwin':
+                    # Our filename isn't supported by the platform :(
+                    self.skipTest('can not write filename %r' % e.filename)
+                else:
+                    raise
             except UnicodeDecodeError:
                 # This happens e.g. with python3.6 on Windows.
                 # It implicitly decodes using utf8, which doesn't work.
                 self.skipTest('can not implicitly convert as utf8')
 
-            build_index_from_tree(
-                repo.path, repo.index_path(),
-                repo.object_store, tree.id)
-
             # Verify index entries
             index = repo.open_index()
             self.assertIn(latin1_name, index)
@@ -749,14 +753,14 @@ class TestTreeFSPathConversion(TestCase):
         fs_path = _tree_to_fs_path(b'/prefix/path', tree_path)
         self.assertEqual(
             fs_path,
-            os.path.join(u'/prefix/path', u'délwíçh', u'foo').encode('utf8'))
+            os.fsencode(os.path.join(u'/prefix/path', u'délwíçh', u'foo')))
 
     def test_fs_to_tree_path_str(self):
         fs_path = os.path.join(os.path.join(u'délwíçh', u'foo'))
-        tree_path = _fs_to_tree_path(fs_path, "utf-8")
-        self.assertEqual(tree_path, u'délwíçh/foo'.encode("utf-8"))
+        tree_path = _fs_to_tree_path(fs_path)
+        self.assertEqual(tree_path, u'délwíçh/foo'.encode('utf-8'))
 
     def test_fs_to_tree_path_bytes(self):
-        fs_path = os.path.join(os.path.join(u'délwíçh', u'foo').encode('utf8'))
-        tree_path = _fs_to_tree_path(fs_path, "utf-8")
-        self.assertEqual(tree_path, u'délwíçh/foo'.encode('utf8'))
+        fs_path = os.path.join(os.fsencode(os.path.join(u'délwíçh', u'foo')))
+        tree_path = _fs_to_tree_path(fs_path)
+        self.assertEqual(tree_path, u'délwíçh/foo'.encode('utf-8'))

dulwich/tests/test_lfs.py

@@ -0,0 +1,44 @@
+# test_lfs.py -- tests for LFS
+# Copyright (C) 2020 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+# General Public License as public by the Free Software Foundation; version 2.0
+# or (at your option) any later version. You can redistribute it and/or
+# modify it under the terms of either of these two licenses.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# You should have received a copy of the licenses; if not, see
+# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+# License, Version 2.0.
+#
+
+"""Tests for LFS support."""
+
+from . import TestCase
+from ..lfs import LFSStore
+import shutil
+import tempfile
+
+
+class LFSTests(TestCase):
+
+    def setUp(self):
+        super(LFSTests, self).setUp()
+        self.test_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.test_dir)
+        self.lfs = LFSStore.create(self.test_dir)
+
+    def test_create(self):
+        sha = self.lfs.write_object([b'a', b'b'])
+        with self.lfs.open_object(sha) as f:
+            self.assertEqual(b'ab', f.read())
+
+    def test_missing(self):
+        self.assertRaises(
+            KeyError, self.lfs.open_object, 'abcdeabcdeabcdeabcde')

dulwich/tests/test_missing_obj_finder.py

@@ -43,7 +43,7 @@ class MissingObjectFinderTest(TestCase):
         return self.commits[n-1]
 
     def assertMissingMatch(self, haves, wants, expected):
-        for sha, path in self.store.find_missing_objects(haves, wants):
+        for sha, path in self.store.find_missing_objects(haves, wants, set()):
             self.assertTrue(
                     sha in expected,
                     "(%s,%s) erroneously reported as missing" % (sha, path))
@@ -112,7 +112,7 @@ class MOFLinearRepoTest(MissingObjectFinderTest):
         haves = [self.cmt(1).id]
         wants = [self.cmt(3).id, bogus_sha]
         self.assertRaises(
-                KeyError, self.store.find_missing_objects, haves, wants)
+                KeyError, self.store.find_missing_objects, haves, wants, set())
 
     def test_no_changes(self):
         self.assertMissingMatch([self.cmt(3).id], [self.cmt(3).id], [])

dulwich/tests/test_object_store.py

@@ -33,13 +33,13 @@ from dulwich.index import (
     )
 from dulwich.errors import (
     NotTreeError,
-    EmptyFileException,
     )
 from dulwich.objects import (
     sha_to_hex,
     Blob,
     Tree,
     TreeEntry,
+    EmptyFileException,
     )
 from dulwich.object_store import (
     DiskObjectStore,
@@ -342,6 +342,14 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
         TestCase.tearDown(self)
         PackBasedObjectStoreTests.tearDown(self)
 
+    def test_loose_compression_level(self):
+        alternate_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, alternate_dir)
+        alternate_store = DiskObjectStore(
+            alternate_dir, loose_compression_level=6)
+        b2 = make_object(Blob, data=b"yummy data")
+        alternate_store.add_object(b2)
+
     def test_alternates(self):
         alternate_dir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, alternate_dir)
@@ -381,6 +389,22 @@ class DiskObjectStoreTests(PackBasedObjectStoreTests, TestCase):
         self.assertEqual([testobject.id],
                          list(self.store._iter_loose_objects()))
 
+    def test_tempfile_in_loose_store(self):
+        self.store.add_object(testobject)
+        self.assertEqual([testobject.id],
+                         list(self.store._iter_loose_objects()))
+
+        # add temporary files to the loose store
+        for i in range(256):
+            dirname = os.path.join(self.store_dir, "%02x" % i)
+            if not os.path.isdir(dirname):
+                os.makedirs(dirname)
+            fd, n = tempfile.mkstemp(prefix="tmp_obj_", dir=dirname)
+            os.close(fd)
+
+        self.assertEqual([testobject.id],
+                         list(self.store._iter_loose_objects()))
+
     def test_add_alternate_path(self):
         store = DiskObjectStore(self.store_dir)
         self.assertEqual([], list(store._read_alternate_paths()))

dulwich/tests/test_objects.py

@@ -132,6 +132,12 @@ class BlobReadTests(TestCase):
         b2 = b1.from_file(BytesIO(b_raw))
         self.assertEqual(b1, b2)
 
+    def test_legacy_from_file_compression_level(self):
+        b1 = Blob.from_string(b'foo')
+        b_raw = b1.as_legacy_object(compression_level=6)
+        b2 = b1.from_file(BytesIO(b_raw))
+        self.assertEqual(b1, b2)
+
     def test_chunks(self):
         string = b'test 5\n'
         b = Blob.from_string(string)

dulwich/tests/test_pack.py

@@ -591,6 +591,18 @@ class WritePackTests(TestCase):
         sha_b.update(f.getvalue()[offset:])
         self.assertEqual(sha_a.digest(), sha_b.digest())
 
+    def test_write_pack_object_compression_level(self):
+        f = BytesIO()
+        f.write(b'header')
+        offset = f.tell()
+        sha_a = sha1(b'foo')
+        sha_b = sha_a.copy()
+        write_pack_object(
+            f, Blob.type_num, b'blob', sha=sha_a, compression_level=6)
+        self.assertNotEqual(sha_a.digest(), sha_b.digest())
+        sha_b.update(f.getvalue()[offset:])
+        self.assertEqual(sha_a.digest(), sha_b.digest())
+
 
 pack_checksum = hex_to_sha('721980e866af9a5f93ad674144e1459b8ba3e7b7')
 

dulwich/tests/test_porcelain.py

@@ -20,12 +20,7 @@
 
 """Tests for dulwich.porcelain."""
 
-from io import BytesIO
-try:
-    from StringIO import StringIO
-except ImportError:
-    from io import StringIO
-import errno
+from io import BytesIO, StringIO
 import os
 import shutil
 import tarfile
@@ -143,9 +138,8 @@ class CleanTests(PorcelainTestCase):
             parent_dir = os.path.dirname(abs_path)
             try:
                 os.makedirs(parent_dir)
-            except OSError as err:
-                if not err.errno == errno.EEXIST:
-                    raise err
+            except FileExistsError:
+                pass
             with open(abs_path, 'w') as f:
                 f.write('')
 
@@ -391,7 +385,12 @@ class AddTests(PorcelainTestCase):
         cwd = os.getcwd()
         try:
             os.chdir(self.repo.path)
-            porcelain.add(self.repo.path)
+            self.assertEqual(
+                set(['foo', 'blah', 'adir', '.git']),
+                set(os.listdir('.')))
+            self.assertEqual(
+                (['foo', os.path.join('adir', 'afile')], set()),
+                porcelain.add(self.repo.path))
         finally:
             os.chdir(cwd)
 
@@ -455,7 +454,7 @@ class AddTests(PorcelainTestCase):
             porcelain.add, self.repo,
             paths=[os.path.join(self.test_dir, "foo")])
         self.assertRaises(
-            ValueError,
+            (ValueError, FileNotFoundError),
             porcelain.add, self.repo,
             paths=["../foo"])
         self.assertEqual([], list(self.repo.open_index()))
@@ -1007,6 +1006,18 @@ class PullTests(PorcelainTestCase):
         with Repo(self.target_path) as r:
             self.assertEqual(r[b'HEAD'].id, self.repo[b'HEAD'].id)
 
+    def test_no_remote_location(self):
+        outstream = BytesIO()
+        errstream = BytesIO()
+
+        # Pull changes into the cloned repo
+        porcelain.pull(self.target_path, refspecs=b'refs/heads/master',
+                       outstream=outstream, errstream=errstream)
+
+        # Check the target repo for pushed changes
+        with Repo(self.target_path) as r:
+            self.assertEqual(r[b'HEAD'].id, self.repo[b'HEAD'].id)
+
 
 class StatusTests(PorcelainTestCase):
 
@@ -1279,7 +1290,7 @@ class ReceivePackTests(PorcelainTestCase):
             b'0091319b56ce3aee2d489f759736a79cc552c9bb86d9 HEAD\x00 report-status '  # noqa: E501
             b'delete-refs quiet ofs-delta side-band-64k '
             b'no-done symref=HEAD:refs/heads/master',
-           b'003f319b56ce3aee2d489f759736a79cc552c9bb86d9 refs/heads/master',
+            b'003f319b56ce3aee2d489f759736a79cc552c9bb86d9 refs/heads/master',
             b'0000'], outlines)
         self.assertEqual(0, exitcode)
 
@@ -1730,11 +1741,20 @@ class DescribeTests(PorcelainTestCase):
                 porcelain.describe(self.repo.path))
 
 
-class HelperTests(PorcelainTestCase):
+class PathToTreeTests(PorcelainTestCase):
+
+    def setUp(self):
+        super(PathToTreeTests, self).setUp()
+        self.fp = os.path.join(self.test_dir, 'bar')
+        with open(self.fp, 'w') as f:
+            f.write('something')
+        oldcwd = os.getcwd()
+        self.addCleanup(os.chdir, oldcwd)
+        os.chdir(self.test_dir)
 
     def test_path_to_tree_path_base(self):
         self.assertEqual(
-            b'bar', porcelain.path_to_tree_path('/home/foo', '/home/foo/bar'))
+            b'bar', porcelain.path_to_tree_path(self.test_dir, self.fp))
         self.assertEqual(b'bar', porcelain.path_to_tree_path('.', './bar'))
         self.assertEqual(b'bar', porcelain.path_to_tree_path('.', 'bar'))
         cwd = os.getcwd()
@@ -1743,13 +1763,12 @@ class HelperTests(PorcelainTestCase):
         self.assertEqual(b'bar', porcelain.path_to_tree_path(cwd, 'bar'))
 
     def test_path_to_tree_path_syntax(self):
-        self.assertEqual(b'bar', porcelain.path_to_tree_path(b'.', './bar'))
-        self.assertEqual(b'bar', porcelain.path_to_tree_path('.', b'./bar'))
-        self.assertEqual(b'bar', porcelain.path_to_tree_path(b'.', b'./bar'))
+        self.assertEqual(b'bar', porcelain.path_to_tree_path('.', './bar'))
 
     def test_path_to_tree_path_error(self):
         with self.assertRaises(ValueError):
-            porcelain.path_to_tree_path('/home/foo/', '/home/bar/baz')
+            with tempfile.TemporaryDirectory() as od:
+                porcelain.path_to_tree_path(od, self.fp)
 
     def test_path_to_tree_path_rel(self):
         cwd = os.getcwd()
@@ -1757,6 +1776,8 @@ class HelperTests(PorcelainTestCase):
         os.mkdir(os.path.join(self.repo.path, 'foo/bar'))
         try:
             os.chdir(os.path.join(self.repo.path, 'foo/bar'))
+            with open('baz', 'w') as f:
+                f.write('contents')
             self.assertEqual(b'bar/baz', porcelain.path_to_tree_path(
                 '..', 'baz'))
             self.assertEqual(b'bar/baz', porcelain.path_to_tree_path(

dulwich/tests/test_refs.py

@@ -506,8 +506,8 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
         refs_data = f.read()
         f.close()
         f = GitFile(refs_file, 'wb')
-        f.write(b'\n'.join(l for l in refs_data.split(b'\n')
-                           if not l or l[0] not in b'#^'))
+        f.write(b'\n'.join(line for line in refs_data.split(b'\n')
+                           if not line or line[0] not in b'#^'))
         f.close()
         self._repo = Repo(self._repo.path)
         refs = self._repo.refs
@@ -557,14 +557,11 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
 
     def test_non_ascii(self):
         try:
-            encoded_ref = u'refs/tags/schön'.encode(
-                    sys.getfilesystemencoding())
+            encoded_ref = os.fsencode(u'refs/tags/schön')
         except UnicodeEncodeError:
             raise SkipTest(
                     "filesystem encoding doesn't support special character")
-        p = os.path.join(
-                self._repo.path.encode(sys.getfilesystemencoding()),
-                encoded_ref)
+        p = os.path.join(os.fsencode(self._repo.path), encoded_ref)
         with open(p, 'w') as f:
             f.write('00' * 20)
 
@@ -575,15 +572,14 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
         self.assertEqual(expected_refs, self._repo.get_refs())
 
     def test_cyrillic(self):
-        if sys.platform == 'win32':
+        if sys.platform in ('darwin', 'win32'):
             raise SkipTest(
                     "filesystem encoding doesn't support arbitrary bytes")
         # reported in https://github.com/dulwich/dulwich/issues/608
         name = b'\xcd\xee\xe2\xe0\xff\xe2\xe5\xf2\xea\xe01'
         encoded_ref = b'refs/heads/' + name
         with open(os.path.join(
-            self._repo.path.encode(
-                sys.getfilesystemencoding()), encoded_ref), 'w') as f:
+                os.fsencode(self._repo.path), encoded_ref), 'w') as f:
             f.write('00' * 20)
 
         expected_refs = set(_TEST_REFS.keys())

dulwich/tests/test_repository.py

@@ -296,7 +296,7 @@ class RepositoryRootTests(TestCase):
     def test_init_mkdir_unicode(self):
         repo_name = u'\xa7'
         try:
-            repo_name.encode(sys.getfilesystemencoding())
+            os.fsencode(repo_name)
         except UnicodeEncodeError:
             self.skipTest('filesystem lacks unicode support')
         tmp_dir = self.mkdtemp()
@@ -361,9 +361,9 @@ class RepositoryRootTests(TestCase):
             c = t.get_config()
             encoded_path = r.path
             if not isinstance(encoded_path, bytes):
-                encoded_path = encoded_path.encode(sys.getfilesystemencoding())
-            self.assertEqual(encoded_path,
-                             c.get((b'remote', b'origin'), b'url'))
+                encoded_path = os.fsencode(encoded_path)
+            self.assertEqual(
+                encoded_path, c.get((b'remote', b'origin'), b'url'))
             self.assertEqual(
                 b'+refs/heads/*:refs/remotes/origin/*',
                 c.get((b'remote', b'origin'), b'fetch'))
@@ -442,19 +442,6 @@ class RepositoryRootTests(TestCase):
         r = self.open_repo('ooo_merge.git')
         self.assertIsInstance(r.get_config_stack(), Config)
 
-    @skipIf(not getattr(os, 'symlink', None), 'Requires symlink support')
-    def test_submodule(self):
-        temp_dir = self.mkdtemp()
-        self.addCleanup(shutil.rmtree, temp_dir)
-        repo_dir = os.path.join(os.path.dirname(__file__), 'data', 'repos')
-        shutil.copytree(os.path.join(repo_dir, 'a.git'),
-                        os.path.join(temp_dir, 'a.git'), symlinks=True)
-        rel = os.path.relpath(os.path.join(repo_dir, 'submodule'), temp_dir)
-        os.symlink(os.path.join(rel, 'dotgit'), os.path.join(temp_dir, '.git'))
-        with Repo(temp_dir) as r:
-            self.assertEqual(r.head(),
-                             b'a90fa2d900a17e99b433217e988c4eb4a2e9a097')
-
     def test_common_revisions(self):
         """
         This test demonstrates that ``find_common_revisions()`` actually
@@ -644,7 +631,7 @@ exit 1
             author_timestamp=12345, author_timezone=0)
         expected_warning = UserWarning(
             'post-commit hook failed: Hook post-commit exited with '
-            'non-zero status',)
+            'non-zero status 1',)
         for w in warnings_list:
             if (type(w) == type(expected_warning) and
                     w.args == expected_warning.args):
@@ -864,6 +851,15 @@ class BuildRepoRootTests(TestCase):
             encoding=b"iso8859-1")
         self.assertEqual(b"iso8859-1", r[commit_sha].encoding)
 
+    def test_compression_level(self):
+        r = self._repo
+        c = r.get_config()
+        c.set(('core',), 'compression', '3')
+        c.set(('core',), 'looseCompression', '4')
+        c.write_to_path()
+        r = Repo(self._repo_dir)
+        self.assertEqual(r.object_store.loose_compression_level, 4)
+
     def test_commit_encoding_from_config(self):
         r = self._repo
         c = r.get_config()
@@ -1081,11 +1077,11 @@ class BuildRepoRootTests(TestCase):
         r.stage(['c'])
         self.assertEqual([b'a'], list(r.open_index()))
 
-    @skipIf(sys.platform == 'win32' and sys.version_info[:2] >= (3, 6),
+    @skipIf(sys.platform in ('win32', 'darwin'),
             'tries to implicitly decode as utf8')
     def test_commit_no_encode_decode(self):
         r = self._repo
-        repo_path_bytes = r.path.encode(sys.getfilesystemencoding())
+        repo_path_bytes = os.fsencode(r.path)
         encodings = ('utf8', 'latin1')
         names = [u'À'.encode(encoding) for encoding in encodings]
         for name, encoding in zip(names, encodings):

dulwich/tests/test_server.py

@@ -1020,9 +1020,9 @@ class FileSystemBackendTests(TestCase):
 
     def test_absolute(self):
         repo = self.backend.open_repository(self.path)
-        self.assertEqual(
-            os.path.normcase(os.path.abspath(repo.path)),
-            os.path.normcase(os.path.abspath(self.repo.path)))
+        self.assertTrue(os.path.samefile(
+            os.path.abspath(repo.path),
+            os.path.abspath(self.repo.path)))
 
     def test_child(self):
         self.assertRaises(

dulwich/tests/test_web.py

@@ -24,6 +24,7 @@ from io import BytesIO
 import gzip
 import re
 import os
+from typing import Type
 
 from dulwich.object_store import (
     MemoryObjectStore,
@@ -107,7 +108,7 @@ class TestHTTPGitRequest(HTTPGitRequest):
 class WebTestCase(TestCase):
     """Base TestCase with useful instance vars and utility functions."""
 
-    _req_class = TestHTTPGitRequest
+    _req_class = TestHTTPGitRequest  # type: Type[HTTPGitRequest]
 
     def setUp(self):
         super(WebTestCase, self).setUp()

dulwich/web.py

@@ -29,6 +29,7 @@ import os
 import re
 import sys
 import time
+from typing import List, Tuple, Optional
 from wsgiref.simple_server import (
     WSGIRequestHandler,
     ServerHandler,
@@ -36,10 +37,7 @@ from wsgiref.simple_server import (
     make_server,
     )
 
-try:
-    from urlparse import parse_qs
-except ImportError:
-    from urllib.parse import parse_qs
+from urllib.parse import parse_qs
 
 
 from dulwich import log_utils
@@ -47,6 +45,7 @@ from dulwich.protocol import (
     ReceivableProtocol,
     )
 from dulwich.repo import (
+    BaseRepo,
     NotGitRepository,
     Repo,
     )
@@ -68,7 +67,7 @@ HTTP_FORBIDDEN = '403 Forbidden'
 HTTP_ERROR = '500 Internal Server Error'
 
 
-def date_time_string(timestamp=None):
+def date_time_string(timestamp: Optional[float] = None) -> str:
     # From BaseHTTPRequestHandler.date_time_string in BaseHTTPServer.py in the
     # Python 2.6.5 standard library, following modifications:
     #  - Made a global rather than an instance method.
@@ -81,12 +80,12 @@ def date_time_string(timestamp=None):
               'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
     if timestamp is None:
         timestamp = time.time()
-    year, month, day, hh, mm, ss, wd, y, z = time.gmtime(timestamp)
+    year, month, day, hh, mm, ss, wd = time.gmtime(timestamp)[:7]
     return '%s, %02d %3s %4d %02d:%02d:%02d GMD' % (
             weekdays[wd], day, months[month], year, hh, mm, ss)
 
 
-def url_prefix(mat):
+def url_prefix(mat) -> str:
     """Extract the URL prefix from a regex match.
 
     Args:
@@ -98,7 +97,7 @@ def url_prefix(mat):
     return '/' + mat.string[:mat.start()].strip('/')
 
 
-def get_repo(backend, mat):
+def get_repo(backend, mat) -> BaseRepo:
     """Get a Repo instance for the given backend and URL regex match."""
     return backend.open_repository(url_prefix(mat))
 
@@ -263,19 +262,23 @@ class HTTPGitRequest(object):
     :ivar environ: the WSGI environment for the request.
     """
 
-    def __init__(self, environ, start_response, dumb=False, handlers=None):
+    def __init__(
+            self, environ, start_response, dumb: bool = False, handlers=None):
         self.environ = environ
         self.dumb = dumb
         self.handlers = handlers
         self._start_response = start_response
-        self._cache_headers = []
-        self._headers = []
+        self._cache_headers = []  # type: List[Tuple[str, str]]
+        self._headers = []  # type: List[Tuple[str, str]]
 
     def add_header(self, name, value):
         """Add a header to the response."""
         self._headers.append((name, value))
 
-    def respond(self, status=HTTP_OK, content_type=None, headers=None):
+    def respond(
+            self, status: str = HTTP_OK,
+            content_type: Optional[str] = None,
+            headers: Optional[List[Tuple[str, str]]] = None):
         """Begin a response with the given status and other headers."""
         if headers:
             self._headers.extend(headers)
@@ -285,28 +288,28 @@ class HTTPGitRequest(object):
 
         return self._start_response(status, self._headers)
 
-    def not_found(self, message):
+    def not_found(self, message: str) -> bytes:
         """Begin a HTTP 404 response and return the text of a message."""
         self._cache_headers = []
         logger.info('Not found: %s', message)
         self.respond(HTTP_NOT_FOUND, 'text/plain')
         return message.encode('ascii')
 
-    def forbidden(self, message):
+    def forbidden(self, message: str) -> bytes:
         """Begin a HTTP 403 response and return the text of a message."""
         self._cache_headers = []
         logger.info('Forbidden: %s', message)
         self.respond(HTTP_FORBIDDEN, 'text/plain')
         return message.encode('ascii')
 
-    def error(self, message):
+    def error(self, message: str) -> bytes:
         """Begin a HTTP 500 response and return the text of a message."""
         self._cache_headers = []
         logger.error('Error: %s', message)
         self.respond(HTTP_ERROR, 'text/plain')
         return message.encode('ascii')
 
-    def nocache(self):
+    def nocache(self) -> None:
         """Set the response to never be cached by the client."""
         self._cache_headers = [
           ('Expires', 'Fri, 01 Jan 1980 00:00:00 GMT'),
@@ -314,7 +317,7 @@ class HTTPGitRequest(object):
           ('Cache-Control', 'no-cache, max-age=0, must-revalidate'),
           ]
 
-    def cache_forever(self):
+    def cache_forever(self) -> None:
         """Set the response to be cached forever by the client."""
         now = time.time()
         self._cache_headers = [
@@ -347,7 +350,9 @@ class HTTPGitApplication(object):
       ('POST', re.compile('/git-receive-pack$')): handle_service_request,
     }
 
-    def __init__(self, backend, dumb=False, handlers=None, fallback_app=None):
+    def __init__(
+            self, backend, dumb: bool = False, handlers=None,
+            fallback_app=None):
         self.backend = backend
         self.dumb = dumb
         self.handlers = dict(DEFAULT_HANDLERS)
@@ -443,11 +448,8 @@ class ServerHandlerLogger(ServerHandler):
     """ServerHandler that uses dulwich's logger for logging exceptions."""
 
     def log_exception(self, exc_info):
-        if sys.version_info < (2, 7):
-            logger.exception('Exception happened during processing of request')
-        else:
-            logger.exception('Exception happened during processing of request',
-                             exc_info=exc_info)
+        logger.exception('Exception happened during processing of request',
+                         exc_info=exc_info)
 
     def log_message(self, format, *args):
         logger.info(format, *args)

setup.cfg

@@ -1,6 +1,9 @@
 [flake8]
 exclude = build,.git,build-pypy,.tox
 
+[mypy]
+ignore_missing_imports = True
+
 [egg_info]
 tag_build = 
 tag_date = 0

setup.py

@@ -15,13 +15,7 @@ import io
 import os
 import sys
 
-dulwich_version_string = '0.19.15'
-
-include_dirs = []
-# Windows MSVC support
-if sys.platform == 'win32' and sys.version_info[:2] < (3, 6):
-    # Include dulwich/ for fallback stdint.h
-    include_dirs.append('dulwich')
+dulwich_version_string = '0.20.2'
 
 
 class DulwichDistribution(Distribution):
@@ -64,12 +58,9 @@ if '__pypy__' not in sys.modules and not sys.platform == 'win32':
 
 
 ext_modules = [
-    Extension('dulwich._objects', ['dulwich/_objects.c'],
-              include_dirs=include_dirs),
-    Extension('dulwich._pack', ['dulwich/_pack.c'],
-              include_dirs=include_dirs),
-    Extension('dulwich._diff_tree', ['dulwich/_diff_tree.c'],
-              include_dirs=include_dirs),
+    Extension('dulwich._objects', ['dulwich/_objects.c']),
+    Extension('dulwich._pack', ['dulwich/_pack.c']),
+    Extension('dulwich._diff_tree', ['dulwich/_diff_tree.c']),
 ]
 
 setup_kwargs = {}
@@ -112,10 +103,10 @@ setup(name='dulwich',
       classifiers=[
           'Development Status :: 4 - Beta',
           'License :: OSI Approved :: Apache Software License',
-          'Programming Language :: Python :: 2.7',
-          'Programming Language :: Python :: 3.4',
           'Programming Language :: Python :: 3.5',
           'Programming Language :: Python :: 3.6',
+          'Programming Language :: Python :: 3.7',
+          'Programming Language :: Python :: 3.8',
           'Programming Language :: Python :: Implementation :: CPython',
           'Programming Language :: Python :: Implementation :: PyPy',
           'Operating System :: POSIX',

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 downloadcache = {toxworkdir}/cache/
-envlist = py27, pypy, py27-noext, pypy-noext, py33, py33-noext, py34, py34-noext, py35, py35-noext, py36, py36-noext
+envlist = pypy, pypy-noext, py35, py35-noext, py36, py36-noext, py37, py37-noext, py38, py38-noext
 
 [testenv]
 
@@ -8,17 +8,17 @@ commands = make check
 recreate = True
 whitelist_externals = make
 
-[testenv:py27-noext]
+[testenv:pypy-noext]
 commands = make check-noextensions
 
-[testenv:pypy-noext]
+[testenv:py35-noext]
 commands = make check-noextensions
 
-[testenv:py34-noext]
+[testenv:py36-noext]
 commands = make check-noextensions
 
-[testenv:py35-noext]
+[testenv:py37-noext]
 commands = make check-noextensions
 
-[testenv:py36-noext]
+[testenv:py38-noext]
 commands = make check-noextensions