Import upstream version 0.19.7, md5 b342e41d768bc85c4d2d6ae80842e4a3

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+branch = True
+source = dulwich
+
+[report]
+exclude_lines =
+    raise NotImplementedError

.gitignore

@@ -0,0 +1,23 @@
+_trial_temp
+build
+build-pypy
+MANIFEST
+dist
+apidocs
+*,cover
+.testrepository
+*.pyc
+*.pyd
+*.pyo
+*.so
+*~
+*.swp
+*.swh
+*.swn
+*.swo
+docs/tutorial/index.html
+dulwich.egg-info/
+.tox/
+.idea/
+.coverage
+htmlcov/

.mailmap

@@ -0,0 +1,23 @@
+Jelmer Vernooij <jelmer@jelmer.uk>
+Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@jelmer.uk>
+Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@samba.org>
+Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@debian.org>
+Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@canonical.com>
+Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@google.com>
+Martin <gzlist@googlemail.com> <martin.packman@canonical.com>
+Dave Borowitz <dborowitz@google.com> <ddborowitz@gmail.com>
+Dave Borowitz <dborowitz@google.com> <dborowitz@google.com>
+John Carr <john.carr@unrouted.co.uk>
+Mark Mikofski <bwanamarko@yahoo.com> <mark.mikofski@sunpowercorp.com>
+Mark Mikofski <bwanamarko@yahoo.com> <bwana.marko@yahoo.com>
+David Carr <david@carrclan.us>
+Jon Bain <jsbain@yahoo.com> <jsbain@yahoo.com>
+James Westby <jw+debian@jameswestby.net> <jw+debian@jameswestby.net>
+David Keijser <david.keijser@klarna.com> <keijser@gmail.com>
+Benoît HERVIER <khertan@khertan.net> <khertan@khertan.net>
+Ryan Faulkner <rfaulk@yahoo-inc.com> <rfaulkner@wikimedia.org>
+David Bennett <davbennett@google.com> <david@dbinit.com>
+Risto Kankkunen <risto.kankkunen@iki.fi> <risto.kankkunen@f-secure.com>
+Augie Fackler <durin42@gmail.com> <raf@durin42.com>
+Damien Tournoud <damien@commerceguys.com> <damien@platform.sh>
+Marcin Kuźmiński <marcin@python-blog.com> <marcin@python-works.com>

.travis.yml

@@ -6,10 +6,8 @@ python:
   - 2.7
   - 3.4
   - 3.5
-  - 3.5-dev
   - 3.6
   - 3.6-dev
-  - pypy3.3-5.2-alpha1
   - pypy3.5
 
 env:
@@ -20,17 +18,15 @@ matrix:
   include:
     - python: pypy
       env: TEST_REQUIRE=fastimport
-    - python: 3.3
-      env: TEST_REQUIRE=fastimport
     - python: 3.7
       env: TEST_REQUIRE=fastimport
       dist: xenial
       sudo: true
-    - python: 3.8-dev
-      env: TEST_REQUIRE=fastimport
-      dist: xenial
-      sudo: true
-      
+    # flakes checker fails on python 3.8-dev:
+    #- python: 3.8-dev
+    #  env: TEST_REQUIRE=fastimport
+    #  dist: xenial
+    #  sudo: true
 
 install:
   - travis_retry pip install -U pip coverage codecov flake8 $TEST_REQUIRE

NEWS

@@ -1,3 +1,42 @@
+0.19.7	2018-11-05
+
+ CHANGES
+
+  * Drop support for Python 3 < 3.4. This is because
+    pkg_resources (which get used by setuptools and mock)
+    no longer supports 3.3 and earlier. (Jelmer Vernooij)
+
+ IMPROVEMENTS
+
+  * Support ``depth`` argument to ``GitClient.fetch_pack`` and support
+    fetching and updating shallow metadata. (Jelmer Vernooij, #240)
+
+ BUG FIXES
+
+  * Don't write to stdout and stderr when they are not available
+    (such as is the case for pythonw). (Sylvia van Os, #652)
+
+  * Fix compatibility with newer versions of git, which expect CONTENT_LENGTH
+    to be set to 0 for empty body requests. (Jelmer Vernooij, #657)
+
+  * Raise an exception client-side when a caller tries to request
+    SHAs that are not directly referenced the servers' refs.
+    (Jelmer Vernooij)
+
+  * Raise more informative errors when unable to connect to repository
+    over SSH or subprocess. (Jelmer Vernooij)
+
+  * Handle commit identity fields with multiple ">" characters.
+    (Nicolas Dandrimont)
+
+ IMPROVEMENTS
+
+  * ``dulwich.porcelain.get_object_by_path`` method for easily
+    accessing a path in another tree. (Jelmer Vernooij)
+
+  * Support the ``i18n.commitEncoding`` setting in config.
+    (Jelmer Vernooij)
+
 0.19.6	2018-08-11
 
  BUG FIXES
@@ -12,6 +51,9 @@
   * Support paths as bytestrings in various places in ``dulwich.index``
     (Jelmer Vernooij)
 
+  * Avoid setup.cfg for now, since it seems to break pypi metadata.
+    (Jelmer Vernooij, #658)
+
 0.19.5	2018-07-08
 
  IMPROVEMENTS

PKG-INFO

@@ -1,112 +1,114 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.19.6
-Summary: [![Build Status](https://travis-ci.org/dulwich/dulwich.png?branch=master)](https://travis-ci.org/dulwich/dulwich)
-[![Windows Build status](https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true)](https://ci.appveyor.com/project/jelmer/dulwich/branch/master)
-
-This is the Dulwich project.
-
-It aims to provide an interface to git repos (both local and remote) that
-doesn't call out to git directly but instead uses pure Python.
-
-**Main website**: [www.dulwich.io](https://www.dulwich.io/)
-
-**License**: Apache License, version 2 or GNU General Public License, version 2 or later.
-
-The project is named after the part of London that Mr. and Mrs. Git live in
-in the particular Monty Python sketch.
-
-Installation
-------------
-
-By default, Dulwich' setup.py will attempt to build and install the optional C
-extensions. The reason for this is that they significantly improve the performance
-since some low-level operations that are executed often are much slower in CPython.
-
-If you don't want to install the C bindings, specify the --pure argument to setup.py::
-
-    $ python setup.py --pure install
-
-or if you are installing from pip::
-
-    $ pip install dulwich --global-option="--pure"
-
-Note that you can also specify --global-option in a
-[requirements.txt](https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers)
-file, e.g. like this::
-
-    dulwich --global-option=--pure
-
-Getting started
----------------
-
-Dulwich comes with both a lower-level API and higher-level plumbing ("porcelain").
-
-For example, to use the lower level API to access the commit message of the
-last commit:
-
-    >>> from dulwich.repo import Repo
-    >>> r = Repo('.')
-    >>> r.head()
-    '57fbe010446356833a6ad1600059d80b1e731e15'
-    >>> c = r[r.head()]
-    >>> c
-    <Commit 015fc1267258458901a94d228e39f0a378370466>
-    >>> c.message
-    'Add note about encoding.\n'
-
-And to print it using porcelain:
-
-    >>> from dulwich import porcelain
-    >>> porcelain.log('.', max_entries=1)
-    --------------------------------------------------
-    commit: 57fbe010446356833a6ad1600059d80b1e731e15
-    Author: Jelmer Vernooij <jelmer@jelmer.uk>
-    Date:   Sat Apr 29 2017 23:57:34 +0000
-
-    Add note about encoding.
-
-Further documentation
----------------------
-
-The dulwich documentation can be found in docs/ and
-[on the web](https://www.dulwich.io/docs/).
-
-The API reference can be generated using pydoctor, by running "make pydoctor",
-or [on the web](https://www.dulwich.io/apidocs).
-
-Help
-----
-
-There is a *#dulwich* IRC channel on the [Freenode](https://www.freenode.net/), and
-[dulwich-announce](https://groups.google.com/forum/#!forum/dulwich-announce)
-and [dulwich-discuss](https://groups.google.com/forum/#!forum/dulwich-discuss)
-mailing lists.
-
-Contributing
-------------
-
-For a full list of contributors, see the git logs or [AUTHORS](AUTHORS).
-
-If you'd like to contribute to Dulwich, see the [CONTRIBUTING](CONTRIBUTING.md)
-file and [list of open issues](https://github.com/dulwich/dulwich/issues).
-
-Supported versions of Python
-----------------------------
-
-At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.3, 3.4, 3.5, 3.6 and Pypy.
-
+Version: 0.19.7
+Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij
 Author-email: jelmer@jelmer.uk
 License: Apachev2 or later or GPLv2
 Project-URL: Bug Tracker, https://github.com/dulwich/dulwich/issues
-Description: UNKNOWN
+Project-URL: GitHub, https://github.com/dulwich/dulwich
+Project-URL: Repository, https://www.dulwich.io/code/
+Description: [![Build Status](https://travis-ci.org/dulwich/dulwich.png?branch=master)](https://travis-ci.org/dulwich/dulwich)
+        [![Windows Build status](https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true)](https://ci.appveyor.com/project/jelmer/dulwich/branch/master)
+        
+        This is the Dulwich project.
+        
+        It aims to provide an interface to git repos (both local and remote) that
+        doesn't call out to git directly but instead uses pure Python.
+        
+        **Main website**: [www.dulwich.io](https://www.dulwich.io/)
+        
+        **License**: Apache License, version 2 or GNU General Public License, version 2 or later.
+        
+        The project is named after the part of London that Mr. and Mrs. Git live in
+        in the particular Monty Python sketch.
+        
+        Installation
+        ------------
+        
+        By default, Dulwich' setup.py will attempt to build and install the optional C
+        extensions. The reason for this is that they significantly improve the performance
+        since some low-level operations that are executed often are much slower in CPython.
+        
+        If you don't want to install the C bindings, specify the --pure argument to setup.py::
+        
+            $ python setup.py --pure install
+        
+        or if you are installing from pip::
+        
+            $ pip install dulwich --global-option="--pure"
+        
+        Note that you can also specify --global-option in a
+        [requirements.txt](https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers)
+        file, e.g. like this::
+        
+            dulwich --global-option=--pure
+        
+        Getting started
+        ---------------
+        
+        Dulwich comes with both a lower-level API and higher-level plumbing ("porcelain").
+        
+        For example, to use the lower level API to access the commit message of the
+        last commit:
+        
+            >>> from dulwich.repo import Repo
+            >>> r = Repo('.')
+            >>> r.head()
+            '57fbe010446356833a6ad1600059d80b1e731e15'
+            >>> c = r[r.head()]
+            >>> c
+            <Commit 015fc1267258458901a94d228e39f0a378370466>
+            >>> c.message
+            'Add note about encoding.\n'
+        
+        And to print it using porcelain:
+        
+            >>> from dulwich import porcelain
+            >>> porcelain.log('.', max_entries=1)
+            --------------------------------------------------
+            commit: 57fbe010446356833a6ad1600059d80b1e731e15
+            Author: Jelmer Vernooij <jelmer@jelmer.uk>
+            Date:   Sat Apr 29 2017 23:57:34 +0000
+        
+            Add note about encoding.
+        
+        Further documentation
+        ---------------------
+        
+        The dulwich documentation can be found in docs/ and
+        [on the web](https://www.dulwich.io/docs/).
+        
+        The API reference can be generated using pydoctor, by running "make pydoctor",
+        or [on the web](https://www.dulwich.io/apidocs).
+        
+        Help
+        ----
+        
+        There is a *#dulwich* IRC channel on the [Freenode](https://www.freenode.net/), and
+        [dulwich-announce](https://groups.google.com/forum/#!forum/dulwich-announce)
+        and [dulwich-discuss](https://groups.google.com/forum/#!forum/dulwich-discuss)
+        mailing lists.
+        
+        Contributing
+        ------------
+        
+        For a full list of contributors, see the git logs or [AUTHORS](AUTHORS).
+        
+        If you'd like to contribute to Dulwich, see the [CONTRIBUTING](CONTRIBUTING.md)
+        file and [list of open issues](https://github.com/dulwich/dulwich/issues).
+        
+        Supported versions of Python
+        ----------------------------
+        
+        At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.3, 3.4, 3.5, 3.6 and Pypy.
+        
+Keywords: git vcs
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.3
 Classifier: Programming Language :: Python :: 3.4
 Classifier: Programming Language :: Python :: 3.5
 Classifier: Programming Language :: Python :: 3.6

appveyor.yml

@@ -10,15 +10,6 @@ environment:
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "64"
 
-    - PYTHON: "C:\\Python33"
-      PYTHON_VERSION: "3.3.x"
-      PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python33-x64"
-      PYTHON_VERSION: "3.3.x"
-      PYTHON_ARCH: "64"
-      DISTUTILS_USE_SDK: "1"
-
     - PYTHON: "C:\\Python34"
       PYTHON_VERSION: "3.4.x"
       PYTHON_ARCH: "32"
@@ -84,6 +75,7 @@ build_script:
 
 test_script:
   - "build.cmd %PYTHON%\\python.exe setup.py test"
+  - "build.cmd %PYTHON%\\pythonw.exe setup.py test"
 
 after_test:
   - "build.cmd %PYTHON%\\python.exe setup.py bdist_wheel"

bin/dulwich

@@ -223,8 +223,13 @@ class cmd_init(Command):
 class cmd_clone(Command):
 
     def run(self, args):
-        opts, args = getopt(args, "", ["bare"])
-        opts = dict(opts)
+        parser = optparse.OptionParser()
+        parser.add_option("--bare", dest="bare",
+                          help="Whether to create a bare repository.",
+                          action="store_true")
+        parser.add_option("--depth", dest="depth",
+                          type=int, help="Depth at which to fetch")
+        options, args = parser.parse_args(args)
 
         if args == []:
             print("usage: dulwich clone host:path [PATH]")
@@ -236,7 +241,7 @@ class cmd_clone(Command):
         else:
             target = None
 
-        porcelain.clone(source, target, bare=("--bare" in opts))
+        porcelain.clone(source, target, bare=options.bare, depth=options.depth)
 
 
 class cmd_commit(Command):

build.cmd

@@ -0,0 +1,21 @@
+@echo off
+:: To build extensions for 64 bit Python 3, we need to configure environment
+:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 4
+::
+:: More details at:
+:: https://github.com/cython/cython/wiki/CythonExtensionsOnWindows
+
+IF "%DISTUTILS_USE_SDK%"=="1" (
+    ECHO Configuring environment to build with MSVC on a 64bit architecture
+    ECHO Using Windows SDK 7.1
+    "C:\Program Files\Microsoft SDKs\Windows\v7.1\Setup\WindowsSdkVer.exe" -q -version:v7.1
+    CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release
+    SET MSSdk=1
+    REM Need the following to allow tox to see the SDK compiler
+    SET TOX_TESTENV_PASSENV=DISTUTILS_USE_SDK MSSdk INCLUDE LIB
+) ELSE (
+    ECHO Using default MSVC build environment
+)
+
+CALL %*

devscripts/PREAMBLE.c

@@ -0,0 +1,15 @@
+ * Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+ * General Public License as public by the Free Software Foundation; version 2.0
+ * or (at your option) any later version. You can redistribute it and/or
+ * modify it under the terms of either of these two licenses.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * You should have received a copy of the licenses; if not, see
+ * <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+ * and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+ * License, Version 2.0.

devscripts/PREAMBLE.py

@@ -0,0 +1,16 @@
+# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+# General Public License as public by the Free Software Foundation; version 2.0
+# or (at your option) any later version. You can redistribute it and/or
+# modify it under the terms of either of these two licenses.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# You should have received a copy of the licenses; if not, see
+# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+# License, Version 2.0.
+#

devscripts/replace-preamble.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/zsh
+perl -i -p0e "s{\Q$(cat PREAMBLE.py.old)\E}{$(cat devscripts/PREAMBLE.py)}g" dulwich/**/*.py bin/dul*
+perl -i -p0e "s{\Q$(cat PREAMBLE.c.old)\E}{$(cat devscripts/PREAMBLE.c)}g" dulwich/*.c

docs/tutorial/.gitignore

@@ -0,0 +1,2 @@
+*.html
+myrepo

dulwich.egg-info/PKG-INFO

@@ -1,112 +1,114 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.19.6
-Summary: [![Build Status](https://travis-ci.org/dulwich/dulwich.png?branch=master)](https://travis-ci.org/dulwich/dulwich)
-[![Windows Build status](https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true)](https://ci.appveyor.com/project/jelmer/dulwich/branch/master)
-
-This is the Dulwich project.
-
-It aims to provide an interface to git repos (both local and remote) that
-doesn't call out to git directly but instead uses pure Python.
-
-**Main website**: [www.dulwich.io](https://www.dulwich.io/)
-
-**License**: Apache License, version 2 or GNU General Public License, version 2 or later.
-
-The project is named after the part of London that Mr. and Mrs. Git live in
-in the particular Monty Python sketch.
-
-Installation
-------------
-
-By default, Dulwich' setup.py will attempt to build and install the optional C
-extensions. The reason for this is that they significantly improve the performance
-since some low-level operations that are executed often are much slower in CPython.
-
-If you don't want to install the C bindings, specify the --pure argument to setup.py::
-
-    $ python setup.py --pure install
-
-or if you are installing from pip::
-
-    $ pip install dulwich --global-option="--pure"
-
-Note that you can also specify --global-option in a
-[requirements.txt](https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers)
-file, e.g. like this::
-
-    dulwich --global-option=--pure
-
-Getting started
----------------
-
-Dulwich comes with both a lower-level API and higher-level plumbing ("porcelain").
-
-For example, to use the lower level API to access the commit message of the
-last commit:
-
-    >>> from dulwich.repo import Repo
-    >>> r = Repo('.')
-    >>> r.head()
-    '57fbe010446356833a6ad1600059d80b1e731e15'
-    >>> c = r[r.head()]
-    >>> c
-    <Commit 015fc1267258458901a94d228e39f0a378370466>
-    >>> c.message
-    'Add note about encoding.\n'
-
-And to print it using porcelain:
-
-    >>> from dulwich import porcelain
-    >>> porcelain.log('.', max_entries=1)
-    --------------------------------------------------
-    commit: 57fbe010446356833a6ad1600059d80b1e731e15
-    Author: Jelmer Vernooij <jelmer@jelmer.uk>
-    Date:   Sat Apr 29 2017 23:57:34 +0000
-
-    Add note about encoding.
-
-Further documentation
----------------------
-
-The dulwich documentation can be found in docs/ and
-[on the web](https://www.dulwich.io/docs/).
-
-The API reference can be generated using pydoctor, by running "make pydoctor",
-or [on the web](https://www.dulwich.io/apidocs).
-
-Help
-----
-
-There is a *#dulwich* IRC channel on the [Freenode](https://www.freenode.net/), and
-[dulwich-announce](https://groups.google.com/forum/#!forum/dulwich-announce)
-and [dulwich-discuss](https://groups.google.com/forum/#!forum/dulwich-discuss)
-mailing lists.
-
-Contributing
-------------
-
-For a full list of contributors, see the git logs or [AUTHORS](AUTHORS).
-
-If you'd like to contribute to Dulwich, see the [CONTRIBUTING](CONTRIBUTING.md)
-file and [list of open issues](https://github.com/dulwich/dulwich/issues).
-
-Supported versions of Python
-----------------------------
-
-At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.3, 3.4, 3.5, 3.6 and Pypy.
-
+Version: 0.19.7
+Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij
 Author-email: jelmer@jelmer.uk
 License: Apachev2 or later or GPLv2
 Project-URL: Bug Tracker, https://github.com/dulwich/dulwich/issues
-Description: UNKNOWN
+Project-URL: GitHub, https://github.com/dulwich/dulwich
+Project-URL: Repository, https://www.dulwich.io/code/
+Description: [![Build Status](https://travis-ci.org/dulwich/dulwich.png?branch=master)](https://travis-ci.org/dulwich/dulwich)
+        [![Windows Build status](https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true)](https://ci.appveyor.com/project/jelmer/dulwich/branch/master)
+        
+        This is the Dulwich project.
+        
+        It aims to provide an interface to git repos (both local and remote) that
+        doesn't call out to git directly but instead uses pure Python.
+        
+        **Main website**: [www.dulwich.io](https://www.dulwich.io/)
+        
+        **License**: Apache License, version 2 or GNU General Public License, version 2 or later.
+        
+        The project is named after the part of London that Mr. and Mrs. Git live in
+        in the particular Monty Python sketch.
+        
+        Installation
+        ------------
+        
+        By default, Dulwich' setup.py will attempt to build and install the optional C
+        extensions. The reason for this is that they significantly improve the performance
+        since some low-level operations that are executed often are much slower in CPython.
+        
+        If you don't want to install the C bindings, specify the --pure argument to setup.py::
+        
+            $ python setup.py --pure install
+        
+        or if you are installing from pip::
+        
+            $ pip install dulwich --global-option="--pure"
+        
+        Note that you can also specify --global-option in a
+        [requirements.txt](https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers)
+        file, e.g. like this::
+        
+            dulwich --global-option=--pure
+        
+        Getting started
+        ---------------
+        
+        Dulwich comes with both a lower-level API and higher-level plumbing ("porcelain").
+        
+        For example, to use the lower level API to access the commit message of the
+        last commit:
+        
+            >>> from dulwich.repo import Repo
+            >>> r = Repo('.')
+            >>> r.head()
+            '57fbe010446356833a6ad1600059d80b1e731e15'
+            >>> c = r[r.head()]
+            >>> c
+            <Commit 015fc1267258458901a94d228e39f0a378370466>
+            >>> c.message
+            'Add note about encoding.\n'
+        
+        And to print it using porcelain:
+        
+            >>> from dulwich import porcelain
+            >>> porcelain.log('.', max_entries=1)
+            --------------------------------------------------
+            commit: 57fbe010446356833a6ad1600059d80b1e731e15
+            Author: Jelmer Vernooij <jelmer@jelmer.uk>
+            Date:   Sat Apr 29 2017 23:57:34 +0000
+        
+            Add note about encoding.
+        
+        Further documentation
+        ---------------------
+        
+        The dulwich documentation can be found in docs/ and
+        [on the web](https://www.dulwich.io/docs/).
+        
+        The API reference can be generated using pydoctor, by running "make pydoctor",
+        or [on the web](https://www.dulwich.io/apidocs).
+        
+        Help
+        ----
+        
+        There is a *#dulwich* IRC channel on the [Freenode](https://www.freenode.net/), and
+        [dulwich-announce](https://groups.google.com/forum/#!forum/dulwich-announce)
+        and [dulwich-discuss](https://groups.google.com/forum/#!forum/dulwich-discuss)
+        mailing lists.
+        
+        Contributing
+        ------------
+        
+        For a full list of contributors, see the git logs or [AUTHORS](AUTHORS).
+        
+        If you'd like to contribute to Dulwich, see the [CONTRIBUTING](CONTRIBUTING.md)
+        file and [list of open issues](https://github.com/dulwich/dulwich/issues).
+        
+        Supported versions of Python
+        ----------------------------
+        
+        At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.3, 3.4, 3.5, 3.6 and Pypy.
+        
+Keywords: git vcs
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.3
 Classifier: Programming Language :: Python :: 3.4
 Classifier: Programming Language :: Python :: 3.5
 Classifier: Programming Language :: Python :: 3.6

dulwich.egg-info/SOURCES.txt

@@ -1,3 +1,6 @@
+.coveragerc
+.gitignore
+.mailmap
 .testr.conf
 .travis.yml
 AUTHORS
@@ -10,19 +13,24 @@ README.md
 README.swift.md
 TODO
 appveyor.yml
+build.cmd
 dulwich.cfg
-setup.cfg
+requirements.txt
 setup.py
 tox.ini
 bin/dul-receive-pack
 bin/dul-upload-pack
 bin/dulwich
+devscripts/PREAMBLE.c
+devscripts/PREAMBLE.py
+devscripts/replace-preamble.sh
 docs/Makefile
 docs/conf.py
 docs/index.txt
 docs/make.bat
 docs/performance.txt
 docs/protocol.txt
+docs/tutorial/.gitignore
 docs/tutorial/Makefile
 docs/tutorial/conclusion.txt
 docs/tutorial/encoding.txt
@@ -72,6 +80,7 @@ dulwich.egg-info/SOURCES.txt
 dulwich.egg-info/dependency_links.txt
 dulwich.egg-info/requires.txt
 dulwich.egg-info/top_level.txt
+dulwich/contrib/README.md
 dulwich/contrib/__init__.py
 dulwich/contrib/paramiko_vendor.py
 dulwich/contrib/release_robot.py

dulwich/__init__.py

@@ -22,4 +22,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 19, 6)
+__version__ = (0, 19, 7)

dulwich/client.py

@@ -40,7 +40,6 @@ Known capabilities that are not supported:
 
 from contextlib import closing
 from io import BytesIO, BufferedReader
-import gzip
 import select
 import socket
 import subprocess
@@ -66,6 +65,7 @@ from dulwich.errors import (
     UpdateRefsError,
     )
 from dulwich.protocol import (
+    HangupException,
     _RBUFSIZE,
     agent_string,
     capability_agent,
@@ -77,12 +77,16 @@ from dulwich.protocol import (
     CAPABILITY_OFS_DELTA,
     CAPABILITY_QUIET,
     CAPABILITY_REPORT_STATUS,
+    CAPABILITY_SHALLOW,
     CAPABILITY_SYMREF,
     CAPABILITY_SIDE_BAND_64K,
     CAPABILITY_THIN_PACK,
     CAPABILITIES_REF,
     KNOWN_RECEIVE_CAPABILITIES,
     KNOWN_UPLOAD_CAPABILITIES,
+    COMMAND_DEEPEN,
+    COMMAND_SHALLOW,
+    COMMAND_UNSHALLOW,
     COMMAND_DONE,
     COMMAND_HAVE,
     COMMAND_WANT,
@@ -103,9 +107,19 @@ from dulwich.pack import (
     )
 from dulwich.refs import (
     read_info_refs,
+    ANNOTATED_TAG_SUFFIX,
     )
 
 
+class InvalidWants(Exception):
+    """Invalid wants."""
+
+    def __init__(self, wants):
+        Exception.__init__(
+            self,
+            "requested wants not in server provided refs: %r" % wants)
+
+
 def _fileno_can_read(fileno):
     """Check if a file descriptor is readable."""
     return len(select.select([fileno], [], [], 0)[0]) > 0
@@ -126,7 +140,8 @@ def _win32_peek_avail(handle):
 
 COMMON_CAPABILITIES = [CAPABILITY_OFS_DELTA, CAPABILITY_SIDE_BAND_64K]
 UPLOAD_CAPABILITIES = ([CAPABILITY_THIN_PACK, CAPABILITY_MULTI_ACK,
-                        CAPABILITY_MULTI_ACK_DETAILED] + COMMON_CAPABILITIES)
+                        CAPABILITY_MULTI_ACK_DETAILED, CAPABILITY_SHALLOW]
+                       + COMMON_CAPABILITIES)
 RECEIVE_CAPABILITIES = [CAPABILITY_REPORT_STATUS] + COMMON_CAPABILITIES
 
 
@@ -196,7 +211,7 @@ def read_pkt_refs(proto):
     for pkt in proto.read_pkt_seq():
         (sha, ref) = pkt.rstrip(b'\n').split(None, 1)
         if sha == b'ERR':
-            raise GitProtocolError(ref)
+            raise GitProtocolError(ref.decode('utf-8', 'replace'))
         if server_capabilities is None:
             (ref, server_capabilities) = extract_capabilities(ref)
         refs[ref] = sha
@@ -222,10 +237,13 @@ class FetchPackResult(object):
             'setdefault', 'update', 'values', 'viewitems', 'viewkeys',
             'viewvalues']
 
-    def __init__(self, refs, symrefs, agent):
+    def __init__(self, refs, symrefs, agent, new_shallow=None,
+                 new_unshallow=None):
         self.refs = refs
         self.symrefs = symrefs
         self.agent = agent
+        self.new_shallow = new_shallow
+        self.new_unshallow = new_unshallow
 
     def _warn_deprecated(self):
         import warnings
@@ -268,6 +286,20 @@ class FetchPackResult(object):
                 self.__class__.__name__, self.refs, self.symrefs, self.agent)
 
 
+def _read_shallow_updates(proto):
+    new_shallow = set()
+    new_unshallow = set()
+    for pkt in proto.read_pkt_seq():
+        cmd, sha = pkt.split(b' ', 1)
+        if cmd == COMMAND_SHALLOW:
+            new_shallow.add(sha.strip())
+        elif cmd == COMMAND_UNSHALLOW:
+            new_unshallow.add(sha.strip())
+        else:
+            raise GitProtocolError('unknown command %s' % pkt)
+    return (new_shallow, new_unshallow)
+
+
 # TODO(durin42): this doesn't correctly degrade if the server doesn't
 # support some capabilities. This should work properly with servers
 # that don't support multi_ack.
@@ -331,7 +363,8 @@ class GitClient(object):
         """
         raise NotImplementedError(self.send_pack)
 
-    def fetch(self, path, target, determine_wants=None, progress=None):
+    def fetch(self, path, target, determine_wants=None, progress=None,
+              depth=None):
         """Fetch into a target repository.
 
         :param path: Path to fetch from (as bytestring)
@@ -340,6 +373,7 @@ class GitClient(object):
             to fetch. Receives dictionary of name->sha, should return
             list of shas to fetch. Defaults to all shas.
         :param progress: Optional progress function
+        :param depth: Depth to fetch at
         :return: Dictionary with all remote refs (not just those fetched)
         """
         if determine_wants is None:
@@ -361,16 +395,17 @@ class GitClient(object):
         try:
             result = self.fetch_pack(
                 path, determine_wants, target.get_graph_walker(), f.write,
-                progress)
+                progress=progress, depth=depth)
         except BaseException:
             abort()
             raise
         else:
             commit()
+        target.update_shallow(result.new_shallow, result.new_unshallow)
         return result
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-                   progress=None):
+                   progress=None, depth=None):
         """Retrieve a pack from a git smart server.
 
         :param path: Remote path to fetch from
@@ -380,6 +415,7 @@ class GitClient(object):
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
+        :param depth: Shallow fetch depth
         :return: FetchPackResult object
         """
         raise NotImplementedError(self.fetch_pack)
@@ -540,7 +576,7 @@ class GitClient(object):
         return (negotiated_capabilities, symrefs, agent)
 
     def _handle_upload_pack_head(self, proto, capabilities, graph_walker,
-                                 wants, can_read):
+                                 wants, can_read, depth):
         """Handle the head of a 'git-upload-pack' request.
 
         :param proto: Protocol object to read from
@@ -549,17 +585,34 @@ class GitClient(object):
         :param wants: List of commits to fetch
         :param can_read: function that returns a boolean that indicates
             whether there is extra graph data to read on proto
+        :param depth: Depth for request
         """
         assert isinstance(wants, list) and isinstance(wants[0], bytes)
         proto.write_pkt_line(COMMAND_WANT + b' ' + wants[0] + b' ' +
                              b' '.join(capabilities) + b'\n')
         for want in wants[1:]:
             proto.write_pkt_line(COMMAND_WANT + b' ' + want + b'\n')
-        proto.write_pkt_line(None)
+        if depth not in (0, None) or getattr(graph_walker, 'shallow', None):
+            if CAPABILITY_SHALLOW not in capabilities:
+                raise GitProtocolError(
+                    "server does not support shallow capability required for "
+                    "depth")
+            for sha in graph_walker.shallow:
+                proto.write_pkt_line(COMMAND_SHALLOW + b' ' + sha + b'\n')
+            proto.write_pkt_line(COMMAND_DEEPEN + b' ' +
+                                 str(depth).encode('ascii') + b'\n')
+            proto.write_pkt_line(None)
+            if can_read is not None:
+                (new_shallow, new_unshallow) = _read_shallow_updates(proto)
+            else:
+                new_shallow = new_unshallow = None
+        else:
+            new_shallow = new_unshallow = set()
+            proto.write_pkt_line(None)
         have = next(graph_walker)
         while have:
             proto.write_pkt_line(COMMAND_HAVE + b' ' + have + b'\n')
-            if can_read():
+            if can_read is not None and can_read():
                 pkt = proto.read_pkt_line()
                 parts = pkt.rstrip(b'\n').split(b' ')
                 if parts[0] == b'ACK':
@@ -574,6 +627,7 @@ class GitClient(object):
                             parts[2])
             have = next(graph_walker)
         proto.write_pkt_line(COMMAND_DONE + b'\n')
+        return (new_shallow, new_unshallow)
 
     def _handle_upload_pack_tail(self, proto, capabilities, graph_walker,
                                  pack_data, progress=None, rbufsize=_RBUFSIZE):
@@ -613,6 +667,31 @@ class GitClient(object):
                 pack_data(data)
 
 
+def check_wants(wants, refs):
+    """Check that a set of wants is valid.
+
+    :param wants: Set of object SHAs to fetch
+    :param refs: Refs dictionary to check against
+    """
+    missing = set(wants) - {
+            v for (k, v) in refs.items()
+            if not k.endswith(ANNOTATED_TAG_SUFFIX)}
+    if missing:
+        raise InvalidWants(missing)
+
+
+def remote_error_from_stderr(stderr):
+    """Return an appropriate exception based on stderr output. """
+    if stderr is None:
+        return HangupException()
+    for l in stderr.readlines():
+        if l.startswith(b'ERROR: '):
+            return GitProtocolError(
+                l[len(b'ERROR: '):].decode('utf-8', 'replace'))
+        return GitProtocolError(l.decode('utf-8', 'replace'))
+    return HangupException()
+
+
 class TraditionalGitClient(GitClient):
     """Traditional Git client."""
 
@@ -654,9 +733,12 @@ class TraditionalGitClient(GitClient):
         :return: new_refs dictionary containing the changes that were made
             {refname: new_ref}, including deleted refs.
         """
-        proto, unused_can_read = self._connect(b'receive-pack', path)
+        proto, unused_can_read, stderr = self._connect(b'receive-pack', path)
         with proto:
-            old_refs, server_capabilities = read_pkt_refs(proto)
+            try:
+                old_refs, server_capabilities = read_pkt_refs(proto)
+            except HangupException:
+                raise remote_error_from_stderr(stderr)
             negotiated_capabilities = \
                 self._negotiate_receive_pack_capabilities(server_capabilities)
             if CAPABILITY_REPORT_STATUS in negotiated_capabilities:
@@ -713,7 +795,7 @@ class TraditionalGitClient(GitClient):
             return new_refs
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-                   progress=None):
+                   progress=None, depth=None):
         """Retrieve a pack from a git smart server.
 
         :param path: Remote path to fetch from
@@ -723,11 +805,15 @@ class TraditionalGitClient(GitClient):
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
+        :param depth: Shallow fetch depth
         :return: FetchPackResult object
         """
-        proto, can_read = self._connect(b'upload-pack', path)
+        proto, can_read, stderr = self._connect(b'upload-pack', path)
         with proto:
-            refs, server_capabilities = read_pkt_refs(proto)
+            try:
+                refs, server_capabilities = read_pkt_refs(proto)
+            except HangupException:
+                raise remote_error_from_stderr(stderr)
             negotiated_capabilities, symrefs, agent = (
                     self._negotiate_upload_pack_capabilities(
                             server_capabilities))
@@ -746,25 +832,31 @@ class TraditionalGitClient(GitClient):
             if not wants:
                 proto.write_pkt_line(None)
                 return FetchPackResult(refs, symrefs, agent)
-            self._handle_upload_pack_head(
-                proto, negotiated_capabilities, graph_walker, wants, can_read)
+            check_wants(wants, refs)
+            (new_shallow, new_unshallow) = self._handle_upload_pack_head(
+                proto, negotiated_capabilities, graph_walker, wants, can_read,
+                depth=depth)
             self._handle_upload_pack_tail(
                 proto, negotiated_capabilities, graph_walker, pack_data,
                 progress)
-            return FetchPackResult(refs, symrefs, agent)
+            return FetchPackResult(
+                    refs, symrefs, agent, new_shallow, new_unshallow)
 
     def get_refs(self, path):
         """Retrieve the current refs from a git smart server."""
         # stock `git ls-remote` uses upload-pack
-        proto, _ = self._connect(b'upload-pack', path)
+        proto, _, stderr = self._connect(b'upload-pack', path)
         with proto:
-            refs, _ = read_pkt_refs(proto)
+            try:
+                refs, _ = read_pkt_refs(proto)
+            except HangupException:
+                raise remote_error_from_stderr(stderr)
             proto.write_pkt_line(None)
             return refs
 
     def archive(self, path, committish, write_data, progress=None,
                 write_error=None, format=None, subdirs=None, prefix=None):
-        proto, can_read = self._connect(b'upload-archive', path)
+        proto, can_read, stderr = self._connect(b'upload-archive', path)
         with proto:
             if format is not None:
                 proto.write_pkt_line(b"argument --format=" + format)
@@ -775,13 +867,17 @@ class TraditionalGitClient(GitClient):
             if prefix is not None:
                 proto.write_pkt_line(b"argument --prefix=" + prefix)
             proto.write_pkt_line(None)
-            pkt = proto.read_pkt_line()
+            try:
+                pkt = proto.read_pkt_line()
+            except HangupException:
+                raise remote_error_from_stderr(stderr)
             if pkt == b"NACK\n":
                 return
             elif pkt == b"ACK\n":
                 pass
             elif pkt.startswith(b"ERR "):
-                raise GitProtocolError(pkt[4:].rstrip(b"\n"))
+                raise GitProtocolError(
+                        pkt[4:].rstrip(b"\n").decode('utf-8', 'replace'))
             else:
                 raise AssertionError("invalid response %r" % pkt)
             ret = proto.read_pkt_line()
@@ -828,7 +924,8 @@ class TCPGitClient(TraditionalGitClient):
             try:
                 s.connect(sockaddr)
                 break
-            except socket.error as err:
+            except socket.error as e:
+                err = e
                 if s is not None:
                     s.close()
                 s = None
@@ -851,7 +948,7 @@ class TCPGitClient(TraditionalGitClient):
         # TODO(jelmer): Alternative to ascii?
         proto.send_cmd(
             b'git-' + cmd, path, b'host=' + self._host.encode('ascii'))
-        return proto, lambda: _fileno_can_read(s)
+        return proto, lambda: _fileno_can_read(s), None
 
 
 class SubprocessWrapper(object):
@@ -865,6 +962,10 @@ class SubprocessWrapper(object):
             self.read = BufferedReader(proc.stdout).read
         self.write = proc.stdin.write
 
+    @property
+    def stderr(self):
+        return self.proc.stderr
+
     def can_read(self):
         if sys.platform == 'win32':
             from msvcrt import get_osfhandle
@@ -899,14 +1000,6 @@ def find_git_command():
 class SubprocessGitClient(TraditionalGitClient):
     """Git client that talks to a server using a subprocess."""
 
-    def __init__(self, **kwargs):
-        self._connection = None
-        self._stderr = None
-        self._stderr = kwargs.get('stderr')
-        if 'stderr' in kwargs:
-            del kwargs['stderr']
-        super(SubprocessGitClient, self).__init__(**kwargs)
-
     @classmethod
     def from_parsedurl(cls, parsedurl, **kwargs):
         return cls(**kwargs)
@@ -921,12 +1014,13 @@ class SubprocessGitClient(TraditionalGitClient):
         if self.git_command is None:
             git_command = find_git_command()
         argv = git_command + [service.decode('ascii'), path]
-        p = SubprocessWrapper(
-            subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
+        p = subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE,
-                             stderr=self._stderr))
-        return Protocol(p.read, p.write, p.close,
-                        report_activity=self._report_activity), p.can_read
+                             stderr=subprocess.PIPE)
+        pw = SubprocessWrapper(p)
+        return (Protocol(pw.read, pw.write, pw.close,
+                         report_activity=self._report_activity),
+                pw.can_read, p.stderr)
 
 
 class LocalGitClient(GitClient):
@@ -1010,7 +1104,8 @@ class LocalGitClient(GitClient):
 
         return new_refs
 
-    def fetch(self, path, target, determine_wants=None, progress=None):
+    def fetch(self, path, target, determine_wants=None, progress=None,
+              depth=None):
         """Fetch into a target repository.
 
         :param path: Path to fetch from (as bytestring)
@@ -1019,16 +1114,17 @@ class LocalGitClient(GitClient):
             to fetch. Receives dictionary of name->sha, should return
             list of shas to fetch. Defaults to all shas.
         :param progress: Optional progress function
+        :param depth: Shallow fetch depth
         :return: FetchPackResult object
         """
         with self._open_repo(path) as r:
             refs = r.fetch(target, determine_wants=determine_wants,
-                           progress=progress)
+                           progress=progress, depth=depth)
             return FetchPackResult(refs, r.refs.get_symrefs(),
                                    agent_string())
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-                   progress=None):
+                   progress=None, depth=None):
         """Retrieve a pack from a git smart server.
 
         :param path: Remote path to fetch from
@@ -1038,11 +1134,12 @@ class LocalGitClient(GitClient):
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
+        :param depth: Shallow fetch depth
         :return: FetchPackResult object
         """
         with self._open_repo(path) as r:
             objects_iter = r.fetch_objects(
-                determine_wants, graph_walker, progress)
+                determine_wants, graph_walker, progress=progress, depth=depth)
             symrefs = r.refs.get_symrefs()
             agent = agent_string()
 
@@ -1128,7 +1225,8 @@ class SubprocessSSHVendor(SSHVendor):
 
         proc = subprocess.Popen(args + [command], bufsize=0,
                                 stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE)
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
         return SubprocessWrapper(proc)
 
 
@@ -1164,7 +1262,8 @@ class PLinkSSHVendor(SSHVendor):
 
         proc = subprocess.Popen(args + [command], bufsize=0,
                                 stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE)
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
         return SubprocessWrapper(proc)
 
 
@@ -1236,7 +1335,7 @@ class SSHGitClient(TraditionalGitClient):
             **kwargs)
         return (Protocol(con.read, con.write, con.close,
                          report_activity=self._report_activity),
-                con.can_read)
+                con.can_read, getattr(con, 'stderr', None))
 
 
 def default_user_agent_string():
@@ -1329,7 +1428,6 @@ class HttpGitClient(GitClient):
         self._username = username
         self._password = password
         self.dumb = dumb
-        self.headers = {}
 
         if pool_manager is None:
             self.pool_manager = default_urllib3_manager(config)
@@ -1353,14 +1451,13 @@ class HttpGitClient(GitClient):
     def from_parsedurl(cls, parsedurl, **kwargs):
         password = parsedurl.password
         if password is not None:
-            password = urlunquote(password)
+            kwargs['password'] = urlunquote(password)
         username = parsedurl.username
         if username is not None:
-            username = urlunquote(username)
+            kwargs['username'] = urlunquote(username)
         # TODO(jelmer): This also strips the username
         parsedurl = parsedurl._replace(netloc=parsedurl.hostname)
-        return cls(urlparse.urlunparse(parsedurl),
-                   password=password, username=username, **kwargs)
+        return cls(urlparse.urlunparse(parsedurl), **kwargs)
 
     def __repr__(self):
         return "%s(%r, dumb=%r)" % (
@@ -1413,10 +1510,7 @@ class HttpGitClient(GitClient):
         # `BytesIO`, if we can guarantee that the entire response is consumed
         # before issuing the next to still allow for connection reuse from the
         # pool.
-        if resp.getheader("Content-Encoding") == "gzip":
-            read = gzip.GzipFile(fileobj=BytesIO(resp.data)).read
-        else:
-            read = BytesIO(resp.data).read
+        read = BytesIO(resp.data).read
 
         resp.content_type = resp.getheader("Content-Type")
         resp.redirect_location = resp.get_redirect_location()
@@ -1530,13 +1624,14 @@ class HttpGitClient(GitClient):
             resp.close()
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-                   progress=None):
+                   progress=None, depth=None):
         """Retrieve a pack from a git smart server.
 
         :param determine_wants: Callback that returns list of commits to fetch
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
+        :param depth: Depth for request
         :return: FetchPackResult object
         """
         url = self._get_url(path)
@@ -1552,19 +1647,24 @@ class HttpGitClient(GitClient):
             return FetchPackResult(refs, symrefs, agent)
         if self.dumb:
             raise NotImplementedError(self.send_pack)
+        check_wants(wants, refs)
         req_data = BytesIO()
         req_proto = Protocol(None, req_data.write)
-        self._handle_upload_pack_head(
+        (new_shallow, new_unshallow) = self._handle_upload_pack_head(
                 req_proto, negotiated_capabilities, graph_walker, wants,
-                lambda: False)
+                can_read=None, depth=depth)
         resp, read = self._smart_request(
             "git-upload-pack", url, data=req_data.getvalue())
         try:
             resp_proto = Protocol(read, None)
+            if new_shallow is None and new_unshallow is None:
+                (new_shallow, new_unshallow) = _read_shallow_updates(
+                        resp_proto)
             self._handle_upload_pack_tail(
                 resp_proto, negotiated_capabilities, graph_walker, pack_data,
                 progress)
-            return FetchPackResult(refs, symrefs, agent)
+            return FetchPackResult(
+                    refs, symrefs, agent, new_shallow, new_unshallow)
         finally:
             resp.close()
 

dulwich/contrib/README.md

@@ -0,0 +1,3 @@
+This directory contains code that some may find useful. Code here is not an official
+part of Dulwich, and may no longer work. Unlike the rest of Dulwich, it is not regularly
+tested.

dulwich/contrib/paramiko_vendor.py

@@ -32,55 +32,20 @@ This implementation is experimental and does not have any tests.
 
 import paramiko
 import paramiko.client
-import threading
 
 
 class _ParamikoWrapper(object):
-    STDERR_READ_N = 2048  # 2k
 
-    def __init__(self, client, channel, progress_stderr=None):
+    def __init__(self, client, channel):
         self.client = client
         self.channel = channel
-        self.progress_stderr = progress_stderr
-        self.should_monitor = bool(progress_stderr) or True
-        self.monitor_thread = None
-        self.stderr = b''
 
         # Channel must block
         self.channel.setblocking(True)
 
-        # Start
-        if self.should_monitor:
-            self.monitor_thread = threading.Thread(
-                target=self.monitor_stderr)
-            self.monitor_thread.start()
-
-    def monitor_stderr(self):
-        while self.should_monitor:
-            # Block and read
-            data = self.read_stderr(self.STDERR_READ_N)
-
-            # Socket closed
-            if not data:
-                self.should_monitor = False
-                break
-
-            # Emit data
-            if self.progress_stderr:
-                self.progress_stderr(data)
-
-            # Append to buffer
-            self.stderr += data
-
-    def stop_monitoring(self):
-        # Stop StdErr thread
-        if self.should_monitor:
-            self.should_monitor = False
-            self.monitor_thread.join()
-
-            # Get left over data
-            data = self.channel.in_stderr_buffer.empty()
-            self.stderr += data
+    @property
+    def stderr(self):
+        return self.channel.makefile_stderr()
 
     def can_read(self):
         return self.channel.recv_ready()
@@ -88,9 +53,6 @@ class _ParamikoWrapper(object):
     def write(self, data):
         return self.channel.sendall(data)
 
-    def read_stderr(self, n):
-        return self.channel.recv_stderr(n)
-
     def read(self, n=None):
         data = self.channel.recv(n)
         data_len = len(data)
@@ -107,7 +69,6 @@ class _ParamikoWrapper(object):
 
     def close(self):
         self.channel.close()
-        self.stop_monitoring()
 
 
 class ParamikoSSHVendor(object):
@@ -118,7 +79,6 @@ class ParamikoSSHVendor(object):
 
     def run_command(self, host, command,
                     username=None, port=None,
-                    progress_stderr=None,
                     password=None, pkey=None,
                     key_filename=None, **kwargs):
 
@@ -148,5 +108,4 @@ class ParamikoSSHVendor(object):
         # Run commands
         channel.exec_command(command)
 
-        return _ParamikoWrapper(
-            client, channel, progress_stderr=progress_stderr)
+        return _ParamikoWrapper(client, channel)

dulwich/ignore.py

@@ -96,7 +96,7 @@ def translate(pat):
     if not pat.endswith(b'/'):
         res += b'/?'
 
-    return res + b'\Z'
+    return res + b'\\Z'
 
 
 def read_ignore_patterns(f):

dulwich/object_store.py

@@ -65,6 +65,7 @@ from dulwich.pack import (
     PackIndexer,
     PackStreamCopier,
     )
+from dulwich.refs import ANNOTATED_TAG_SUFFIX
 
 INFODIR = 'info'
 PACKDIR = 'pack'
@@ -75,7 +76,8 @@ class BaseObjectStore(object):
 
     def determine_wants_all(self, refs):
         return [sha for (ref, sha) in refs.items()
-                if sha not in self and not ref.endswith(b"^{}") and
+                if sha not in self and
+                not ref.endswith(ANNOTATED_TAG_SUFFIX) and
                 not sha == ZERO_SHA]
 
     def iter_shas(self, shas):
@@ -192,7 +194,8 @@ class BaseObjectStore(object):
 
     def find_missing_objects(self, haves, wants, progress=None,
                              get_tagged=None,
-                             get_parents=lambda commit: commit.parents):
+                             get_parents=lambda commit: commit.parents,
+                             depth=None):
         """Find the missing objects required for a set of revisions.
 
         :param haves: Iterable over SHAs already in common.
@@ -1163,7 +1166,7 @@ class ObjectStoreGraphWalker(object):
     :ivar get_parents: Function to retrieve parents in the local repo
     """
 
-    def __init__(self, local_heads, get_parents):
+    def __init__(self, local_heads, get_parents, shallow=None):
         """Create a new instance.
 
         :param local_heads: Heads to start search with
@@ -1172,6 +1175,9 @@ class ObjectStoreGraphWalker(object):
         self.heads = set(local_heads)
         self.get_parents = get_parents
         self.parents = {}
+        if shallow is None:
+            shallow = set()
+        self.shallow = shallow
 
     def ack(self, sha):
         """Ack that a revision and its ancestors are present in the source."""
@@ -1315,3 +1321,14 @@ class OverlayObjectStore(BaseObjectStore):
                 return True
         else:
             return False
+
+
+def read_packs_file(f):
+    """Yield the packs listed in a packs file."""
+    for line in f.read().splitlines():
+        if not line:
+            continue
+        (kind, name) = line.split(b" ", 1)
+        if kind != b"P":
+            continue
+        yield name.decode(sys.getfilesystemencoding())

dulwich/objects.py

@@ -1112,7 +1112,7 @@ def parse_time_entry(value):
     :return: Tuple of (author, time, (timezone, timezone_neg_utc))
     """
     try:
-        sep = value.index(b'> ')
+        sep = value.rindex(b'> ')
     except ValueError:
         return (value, None, (None, False))
     try:

dulwich/porcelain.py

@@ -61,7 +61,7 @@ from contextlib import (
     closing,
     contextmanager,
 )
-from io import BytesIO
+from io import BytesIO, RawIOBase
 import datetime
 import os
 import posixpath
@@ -139,8 +139,27 @@ from dulwich.server import (
 GitStatus = namedtuple('GitStatus', 'staged unstaged untracked')
 
 
-default_bytes_out_stream = getattr(sys.stdout, 'buffer', sys.stdout)
-default_bytes_err_stream = getattr(sys.stderr, 'buffer', sys.stderr)
+class NoneStream(RawIOBase):
+    """Fallback if stdout or stderr are unavailable, does nothing."""
+    def read(self, size=-1):
+        return None
+
+    def readall(self):
+        return None
+
+    def readinto(self, b):
+        return None
+
+    def write(self, b):
+        return None
+
+
+default_bytes_out_stream = getattr(
+        sys.stdout, 'buffer', sys.stdout
+    ) or NoneStream()
+default_bytes_err_stream = getattr(
+        sys.stderr, 'buffer', sys.stderr
+    ) or NoneStream()
 
 
 DEFAULT_ENCODING = 'utf-8'
@@ -188,6 +207,8 @@ def path_to_tree_path(repopath, path):
     treepath = os.path.relpath(path, repopath)
     if treepath.startswith(b'..'):
         raise ValueError('Path not in repo')
+    if os.path.sep != '/':
+        treepath = treepath.replace(os.path.sep.encode('ascii'), b'/')
     return treepath
 
 
@@ -288,7 +309,7 @@ def init(path=".", bare=False):
 
 def clone(source, target=None, bare=False, checkout=None,
           errstream=default_bytes_err_stream, outstream=None,
-          origin=b"origin", **kwargs):
+          origin=b"origin", depth=None, **kwargs):
     """Clone a local or remote git repository.
 
     :param source: Path or URL for source repository
@@ -298,6 +319,7 @@ def clone(source, target=None, bare=False, checkout=None,
     :param errstream: Optional stream to write progress to
     :param outstream: Optional stream to write progress to (deprecated)
     :param origin: Name of remote from the repository used to clone
+    :param depth: Depth to fetch at
     :return: The new repository
     """
     # TODO(jelmer): This code overlaps quite a bit with Repo.clone
@@ -328,7 +350,7 @@ def clone(source, target=None, bare=False, checkout=None,
     try:
         fetch_result = fetch(
             r, source, origin, errstream=errstream, message=reflog_message,
-            **kwargs)
+            depth=depth, **kwargs)
         target_config = r.get_config()
         if not isinstance(source, bytes):
             source = source.encode(DEFAULT_ENCODING)
@@ -1065,7 +1087,8 @@ def branch_list(repo):
 
 
 def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout,
-          errstream=default_bytes_err_stream, message=None, **kwargs):
+          errstream=default_bytes_err_stream, message=None, depth=None,
+          **kwargs):
     """Fetch objects from a remote server.
 
     :param repo: Path to the repository
@@ -1074,6 +1097,7 @@ def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout,
     :param outstream: Output stream (defaults to stdout)
     :param errstream: Error stream (defaults to stderr)
     :param message: Reflog message (defaults to b"fetch: from <remote_name>")
+    :param depth: Depth to fetch at
     :return: Dictionary with refs on the remote
     """
     if message is None:
@@ -1081,7 +1105,8 @@ def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout,
     with open_repo_closing(repo) as r:
         client, path = get_transport_and_path(
             remote_location, config=r.get_config_stack(), **kwargs)
-        fetch_result = client.fetch(path, r, progress=errstream.write)
+        fetch_result = client.fetch(path, r, progress=errstream.write,
+                                    depth=depth)
         stripped_refs = strip_peeled_refs(fetch_result.refs)
         branches = {
             n[len(b'refs/heads/'):]: v for (n, v) in stripped_refs.items()
@@ -1364,3 +1389,25 @@ def describe(repo):
 
         # Return plain commit if no parent tag can be found
         return 'g{}'.format(latest_commit.id.decode('ascii')[:7])
+
+
+def get_object_by_path(repo, path, committish=None):
+    """Get an object by path.
+
+    :param repo: A path to the repository
+    :param path: Path to look up
+    :param committish: Commit to look up path in
+    :return: A `ShaFile` object
+    """
+    if committish is None:
+        committish = "HEAD"
+    # Get the repository
+    with open_repo_closing(repo) as r:
+        commit = parse_commit(repo, committish)
+        base_tree = commit.tree
+        if not isinstance(path, bytes):
+            path = path.encode(commit.encoding or DEFAULT_ENCODING)
+        (mode, sha) = tree_lookup_path(
+            r.object_store.__getitem__,
+            base_tree, path)
+        return r[sha]

dulwich/refs.py

@@ -800,7 +800,7 @@ class DiskRefsContainer(RefsContainer):
             parent_filename = self.refpath(parent)
             try:
                 os.rmdir(parent_filename)
-            except OSError as e:
+            except OSError:
                 # this can be caused by the parent directory being
                 # removed by another process, being not empty, etc.
                 # in any case, this is non fatal because we already

dulwich/repo.py

@@ -259,24 +259,26 @@ class BaseRepo(object):
         """
         raise NotImplementedError(self.open_index)
 
-    def fetch(self, target, determine_wants=None, progress=None):
+    def fetch(self, target, determine_wants=None, progress=None, depth=None):
         """Fetch objects into another repository.
 
         :param target: The target repository
         :param determine_wants: Optional function to determine what refs to
             fetch.
         :param progress: Optional progress function
+        :param depth: Optional shallow fetch depth
         :return: The local refs
         """
         if determine_wants is None:
             determine_wants = target.object_store.determine_wants_all
         count, pack_data = self.fetch_pack_data(
-                determine_wants, target.get_graph_walker(), progress)
+                determine_wants, target.get_graph_walker(), progress=progress,
+                depth=depth)
         target.object_store.add_pack_data(count, pack_data, progress)
         return self.get_refs()
 
     def fetch_pack_data(self, determine_wants, graph_walker, progress,
-                        get_tagged=None):
+                        get_tagged=None, depth=None):
         """Fetch the pack data required for a set of revisions.
 
         :param determine_wants: Function that takes a dictionary with heads
@@ -288,15 +290,16 @@ class BaseRepo(object):
             updated progress strings.
         :param get_tagged: Function that returns a dict of pointed-to sha ->
             tag sha for including tags.
+        :param depth: Shallow fetch depth
         :return: count and iterator over pack data
         """
         # TODO(jelmer): Fetch pack data directly, don't create objects first.
         objects = self.fetch_objects(determine_wants, graph_walker, progress,
-                                     get_tagged)
+                                     get_tagged, depth=depth)
         return pack_objects_to_data(objects)
 
     def fetch_objects(self, determine_wants, graph_walker, progress,
-                      get_tagged=None):
+                      get_tagged=None, depth=None):
         """Fetch the missing objects required for a set of revisions.
 
         :param determine_wants: Function that takes a dictionary with heads
@@ -308,8 +311,12 @@ class BaseRepo(object):
             updated progress strings.
         :param get_tagged: Function that returns a dict of pointed-to sha ->
             tag sha for including tags.
+        :param depth: Shallow fetch depth
         :return: iterator over objects, with __len__ implemented
         """
+        if depth not in (None, 0):
+            raise NotImplementedError("depth not supported yet")
+
         wants = determine_wants(self.get_refs())
         if not isinstance(wants, list):
             raise TypeError("determine_wants() did not return a list")
@@ -361,7 +368,8 @@ class BaseRepo(object):
         """
         if heads is None:
             heads = self.refs.as_dict(b'refs/heads').values()
-        return ObjectStoreGraphWalker(heads, self.get_parents)
+        return ObjectStoreGraphWalker(
+            heads, self.get_parents, shallow=self.get_shallow())
 
     def get_refs(self):
         """Get dictionary with all refs.
@@ -458,7 +466,26 @@ class BaseRepo(object):
 
         :return: Set of shallow commits.
         """
-        return set()
+        f = self.get_named_file('shallow')
+        if f is None:
+            return set()
+        with f:
+            return set(l.strip() for l in f)
+
+    def update_shallow(self, new_shallow, new_unshallow):
+        """Update the list of shallow objects.
+
+        :param new_shallow: Newly shallow objects
+        :param new_unshallow: Newly no longer shallow objects
+        """
+        shallow = self.get_shallow()
+        if new_shallow:
+            shallow.update(new_shallow)
+        if new_unshallow:
+            shallow.difference_update(new_unshallow)
+        self._put_named_file(
+            'shallow',
+            b''.join([sha + b'\n' for sha in shallow]))
 
     def get_peeled(self, ref):
         """Get the peeled value of a ref.
@@ -565,12 +592,11 @@ class BaseRepo(object):
         else:
             raise ValueError(name)
 
-    def _get_user_identity(self):
+    def _get_user_identity(self, config):
         """Determine the identity to use for new commits.
         """
         user = os.environ.get("GIT_COMMITTER_NAME")
         email = os.environ.get("GIT_COMMITTER_EMAIL")
-        config = self.get_config_stack()
         if user is None:
             try:
                 user = config.get(("user", ), "name")
@@ -652,11 +678,12 @@ class BaseRepo(object):
         except KeyError:  # no hook defined, silent fallthrough
             pass
 
+        config = self.get_config_stack()
         if merge_heads is None:
             # FIXME: Read merge heads from .git/MERGE_HEADS
             merge_heads = []
         if committer is None:
-            committer = self._get_user_identity()
+            committer = self._get_user_identity(config)
         check_user_identity(committer)
         c.committer = committer
         if commit_timestamp is None:
@@ -680,6 +707,11 @@ class BaseRepo(object):
         if author_timezone is None:
             author_timezone = commit_timezone
         c.author_timezone = author_timezone
+        if encoding is None:
+            try:
+                encoding = config.get(('i18n', ), 'commitEncoding')
+            except KeyError:
+                pass  # No dice
         if encoding is not None:
             c.encoding = encoding
         if message is None:
@@ -816,7 +848,8 @@ class Repo(BaseRepo):
             if e.errno != errno.EEXIST:
                 raise
         if committer is None:
-            committer = self._get_user_identity()
+            config = self.get_config_stack()
+            committer = self._get_user_identity(config)
         check_user_identity(committer)
         if timestamp is None:
             timestamp = int(time.time())
@@ -914,13 +947,6 @@ class Repo(BaseRepo):
                 return None
             raise
 
-    def get_shallow(self):
-        f = self.get_named_file('shallow')
-        if f is None:
-            return set()
-        with f:
-            return set(l.strip() for l in f)
-
     def index_path(self):
         """Return path to the index file."""
         return os.path.join(self.controldir(), INDEX_FILENAME)

dulwich/server.py

@@ -370,12 +370,10 @@ class UploadPackHandler(PackHandler):
                 self._done_received):
             return
 
-        self.progress(b"dul-daemon says what\n")
         self.progress(
                 ("counting objects: %d, done.\n" % len(objects_iter)).encode(
                     'ascii'))
         write_pack_objects(ProtocolFile(None, write), objects_iter)
-        self.progress(b"how was that, then?\n")
         # we are done
         self.proto.write_pkt_line(None)
 
@@ -929,7 +927,7 @@ class ReceivePackHandler(PackHandler):
                         self.repo.refs.set_if_equals(ref, oldsha, sha)
                     except all_exceptions:
                         ref_status = b'failed to write'
-            except KeyError as e:
+            except KeyError:
                 ref_status = b'bad ref'
             status.append((ref, ref_status))
 

dulwich/tests/compat/test_client.py

@@ -210,6 +210,18 @@ class DulwichClientTestBase(object):
                 dest.refs.set_if_equals(r[0], None, r[1])
             self.assertDestEqualsSrc()
 
+    def test_fetch_pack_depth(self):
+        c = self._client()
+        with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
+            result = c.fetch(self._build_path('/server_new.export'), dest,
+                             depth=1)
+            for r in result.refs.items():
+                dest.refs.set_if_equals(r[0], None, r[1])
+            self.assertEqual(
+                    dest.get_shallow(),
+                    set([b'35e0b59e187dd72a0af294aedffc213eaa4d03ff',
+                         b'514dc6d3fbfe77361bcaef320c4d21b72bc10be9']))
+
     def test_repeat(self):
         c = self._client()
         with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
@@ -380,7 +392,7 @@ class DulwichSubprocessClientTest(CompatTestCase, DulwichClientTestBase):
         CompatTestCase.tearDown(self)
 
     def _client(self):
-        return client.SubprocessGitClient(stderr=subprocess.PIPE)
+        return client.SubprocessGitClient()
 
     def _build_path(self, path):
         return self.gitroot + path
@@ -500,6 +512,7 @@ class GitHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
             data = self.rfile.read(nbytes)
         else:
             data = None
+            env['CONTENT_LENGTH'] = '0'
         # throw away additional data [see bug #427345]
         while select.select([self.rfile._sock], [], [], 0)[0]:
             if not self.rfile._sock.recv(1):

dulwich/tests/compat/test_pack.py

@@ -46,7 +46,7 @@ from dulwich.tests.compat.utils import (
     run_git_or_fail,
     )
 
-_NON_DELTA_RE = re.compile(b'non delta: (?P<non_delta>\d+) objects')
+_NON_DELTA_RE = re.compile(b'non delta: (?P<non_delta>\\d+) objects')
 
 
 def _git_verify_pack_object_list(output):

dulwich/tests/test_client.py

@@ -42,6 +42,7 @@ from dulwich import (
     client,
     )
 from dulwich.client import (
+    InvalidWants,
     LocalGitClient,
     TraditionalGitClient,
     TCPGitClient,
@@ -53,6 +54,7 @@ from dulwich.client import (
     SubprocessSSHVendor,
     PLinkSSHVendor,
     UpdateRefsError,
+    check_wants,
     default_urllib3_manager,
     get_transport_and_path,
     get_transport_and_path_from_url,
@@ -98,7 +100,7 @@ class DummyClient(TraditionalGitClient):
         TraditionalGitClient.__init__(self)
 
     def _connect(self, service, path):
-        return Protocol(self.read, self.write), self.can_read
+        return Protocol(self.read, self.write), self.can_read, None
 
 
 class DummyPopen():
@@ -132,7 +134,7 @@ class GitClientTests(TestCase):
         agent_cap = (
             'agent=dulwich/%d.%d.%d' % dulwich.__version__).encode('ascii')
         self.assertEqual(set([b'multi_ack', b'side-band-64k', b'ofs-delta',
-                              b'thin-pack', b'multi_ack_detailed',
+                              b'thin-pack', b'multi_ack_detailed', b'shallow',
                               agent_cap]),
                          set(self.client._fetch_capabilities))
         self.assertEqual(set([b'ofs-delta', b'report-status', b'side-band-64k',
@@ -190,6 +192,20 @@ class GitClientTests(TestCase):
         self.assertEqual({}, ret.symrefs)
         self.assertEqual(self.rout.getvalue(), b'0000')
 
+    def test_fetch_pack_sha_not_in_ref(self):
+        self.rin.write(
+            b'008855dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7 HEAD\x00multi_ack '
+            b'thin-pack side-band side-band-64k ofs-delta shallow no-progress '
+            b'include-tag\n'
+            b'0000')
+        self.rin.seek(0)
+        self.assertRaises(
+                InvalidWants, self.client.fetch_pack,
+                b'bla',
+                lambda heads: ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'],
+                None, None,
+                None)
+
     def test_send_pack_no_sideband64k_with_update_ref_error(self):
         # No side-bank-64k reported by server shouldn't try to parse
         # side band data
@@ -216,7 +232,7 @@ class GitClientTests(TestCase):
         commit.encoding = b'UTF-8'
         commit.message = b'test message'
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {b'refs/foo/bar': commit.id, }
 
         def generate_pack_data(have, want, ofs_delta=False):
@@ -224,7 +240,7 @@ class GitClientTests(TestCase):
 
         self.assertRaises(UpdateRefsError,
                           self.client.send_pack, "blah",
-                          determine_wants, generate_pack_data)
+                          update_refs, generate_pack_data)
 
     def test_send_pack_none(self):
         self.rin.write(
@@ -234,7 +250,7 @@ class GitClientTests(TestCase):
             b'0000')
         self.rin.seek(0)
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {
                 b'refs/heads/master':
                     b'310ca9477129b8586fa2afc779c1f57cf64bba6c'
@@ -243,7 +259,7 @@ class GitClientTests(TestCase):
         def generate_pack_data(have, want, ofs_delta=False):
             return 0, []
 
-        self.client.send_pack(b'/', determine_wants, generate_pack_data)
+        self.client.send_pack(b'/', update_refs, generate_pack_data)
         self.assertEqual(self.rout.getvalue(), b'0000')
 
     def test_send_pack_keep_and_delete(self):
@@ -256,13 +272,13 @@ class GitClientTests(TestCase):
             b'0000')
         self.rin.seek(0)
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {b'refs/heads/master': b'0' * 40}
 
         def generate_pack_data(have, want, ofs_delta=False):
             return 0, []
 
-        self.client.send_pack(b'/', determine_wants, generate_pack_data)
+        self.client.send_pack(b'/', update_refs, generate_pack_data)
         self.assertIn(
             self.rout.getvalue(),
             [b'007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
@@ -281,13 +297,13 @@ class GitClientTests(TestCase):
             b'0000')
         self.rin.seek(0)
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {b'refs/heads/master': b'0' * 40}
 
         def generate_pack_data(have, want, ofs_delta=False):
             return 0, []
 
-        self.client.send_pack(b'/', determine_wants, generate_pack_data)
+        self.client.send_pack(b'/', update_refs, generate_pack_data)
         self.assertIn(
             self.rout.getvalue(),
             [b'007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
@@ -306,7 +322,7 @@ class GitClientTests(TestCase):
             b'0000')
         self.rin.seek(0)
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {
                 b'refs/heads/blah12':
                 b'310ca9477129b8586fa2afc779c1f57cf64bba6c',
@@ -319,7 +335,7 @@ class GitClientTests(TestCase):
 
         f = BytesIO()
         write_pack_objects(f, {})
-        self.client.send_pack('/', determine_wants, generate_pack_data)
+        self.client.send_pack('/', update_refs, generate_pack_data)
         self.assertIn(
             self.rout.getvalue(),
             [b'007f0000000000000000000000000000000000000000 '
@@ -350,7 +366,7 @@ class GitClientTests(TestCase):
         commit.encoding = b'UTF-8'
         commit.message = b'test message'
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {
                 b'refs/heads/blah12': commit.id,
                 b'refs/heads/master':
@@ -362,7 +378,7 @@ class GitClientTests(TestCase):
 
         f = BytesIO()
         write_pack_data(f, *generate_pack_data(None, None))
-        self.client.send_pack(b'/', determine_wants, generate_pack_data)
+        self.client.send_pack(b'/', update_refs, generate_pack_data)
         self.assertIn(
             self.rout.getvalue(),
             [b'007f0000000000000000000000000000000000000000 ' + commit.id +
@@ -384,7 +400,7 @@ class GitClientTests(TestCase):
                 self.rin.write(("%04x" % (len(pkt)+4)).encode('ascii') + pkt)
         self.rin.seek(0)
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {b'refs/heads/master': b'0' * 40}
 
         def generate_pack_data(have, want, ofs_delta=False):
@@ -392,7 +408,7 @@ class GitClientTests(TestCase):
 
         self.assertRaises(UpdateRefsError,
                           self.client.send_pack, b"/",
-                          determine_wants, generate_pack_data)
+                          update_refs, generate_pack_data)
         self.assertEqual(self.rout.getvalue(), b'0000')
 
 
@@ -549,6 +565,28 @@ class TestGetTransportAndPath(TestCase):
         self.assertEqual('user', c._username)
         self.assertEqual('passwd', c._password)
 
+    def test_http_auth_with_username(self):
+        url = 'https://github.com/jelmer/dulwich'
+
+        c, path = get_transport_and_path(
+                url, username='user2', password='blah')
+
+        self.assertTrue(isinstance(c, HttpGitClient))
+        self.assertEqual('/jelmer/dulwich', path)
+        self.assertEqual('user2', c._username)
+        self.assertEqual('blah', c._password)
+
+    def test_http_auth_with_username_and_in_url(self):
+        url = 'https://user:passwd@github.com/jelmer/dulwich'
+
+        c, path = get_transport_and_path(
+                url, username='user2', password='blah')
+
+        self.assertTrue(isinstance(c, HttpGitClient))
+        self.assertEqual('/jelmer/dulwich', path)
+        self.assertEqual('user', c._username)
+        self.assertEqual('passwd', c._password)
+
     def test_http_no_auth(self):
         url = 'https://github.com/jelmer/dulwich'
 
@@ -1156,3 +1194,25 @@ class RsyncUrlTests(TestCase):
 
     def test_path(self):
         self.assertRaises(ValueError, parse_rsync_url, '/path')
+
+
+class CheckWantsTests(TestCase):
+
+    def test_fine(self):
+        check_wants(
+            [b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'],
+            {b'refs/heads/blah': b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'})
+
+    def test_missing(self):
+        self.assertRaises(
+            InvalidWants, check_wants,
+            [b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'],
+            {b'refs/heads/blah': b'3f3dc7a53fb752a6961d3a56683df46d4d3bf262'})
+
+    def test_annotated(self):
+        self.assertRaises(
+            InvalidWants, check_wants,
+            [b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'],
+            {b'refs/heads/blah': b'3f3dc7a53fb752a6961d3a56683df46d4d3bf262',
+             b'refs/heads/blah^{}':
+                b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'})

dulwich/tests/test_diff_tree.py

@@ -725,8 +725,8 @@ class RenameDetectionTest(DiffTestCase):
     def test_content_rename_one_to_one(self):
         b11 = make_object(Blob, data=b'a\nb\nc\nd\n')
         b12 = make_object(Blob, data=b'a\nb\nc\ne\n')
-        b21 = make_object(Blob, data=b'e\nf\ng\n\h')
-        b22 = make_object(Blob, data=b'e\nf\ng\n\i')
+        b21 = make_object(Blob, data=b'e\nf\ng\n\nh')
+        b22 = make_object(Blob, data=b'e\nf\ng\n\ni')
         tree1 = self.commit_tree([(b'a', b11), (b'b', b21)])
         tree2 = self.commit_tree([(b'c', b12), (b'd', b22)])
         self.assertEqual(

dulwich/tests/test_ignore.py

@@ -106,10 +106,10 @@ class ReadIgnorePatterns(TestCase):
 
 # and an empty line:
 
-\#not a comment
+\\#not a comment
 !negative
 with trailing whitespace 
-with escaped trailing whitespace\ 
+with escaped trailing whitespace\\ 
 """)  # noqa: W291
         self.assertEqual(list(read_ignore_patterns(f)), [
             b'\\#not a comment',

dulwich/tests/test_object_store.py

@@ -47,6 +47,7 @@ from dulwich.object_store import (
     OverlayObjectStore,
     ObjectStoreGraphWalker,
     commit_tree_changes,
+    read_packs_file,
     tree_lookup_path,
     )
 from dulwich.pack import (
@@ -645,3 +646,10 @@ class CommitTreeChangesTests(TestCase):
         self.assertEqual(set(new_tree), {b'a', b'ad', b'c'})
         ad_tree = self.store[new_tree[b'ad'][1]]
         self.assertEqual(set(ad_tree), {b'b', b'c'})
+
+
+class TestReadPacksFile(TestCase):
+
+    def test_read_packs(self):
+        self.assertEqual(["pack-1.pack"], list(read_packs_file(BytesIO(b"""P pack-1.pack
+"""))))

dulwich/tests/test_objects.py

@@ -673,6 +673,27 @@ class CommitParseTests(ShaFileCheckTests):
             with self.assertRaises(ObjectFormatException):
                 commit.check()
 
+    def test_mangled_author_line(self):
+        """Mangled author line should successfully parse"""
+        author_line = (
+            b'Karl MacMillan <kmacmill@redhat.com> <"Karl MacMillan '
+            b'<kmacmill@redhat.com>"> 1197475547 -0500'
+        )
+        expected_identity = (
+            b'Karl MacMillan <kmacmill@redhat.com> <"Karl MacMillan '
+            b'<kmacmill@redhat.com>">'
+        )
+        commit = Commit.from_string(
+            self.make_commit_text(author=author_line)
+        )
+
+        # The commit parses properly
+        self.assertEqual(commit.author, expected_identity)
+
+        # But the check fails because the author identity is bogus
+        with self.assertRaises(ObjectFormatException):
+            commit.check()
+
     def test_parse_gpgsig(self):
         c = Commit.from_string(b"""tree aaff74984cccd156a469afa7d9ab10e4777beb24
 author Jelmer Vernooij <jelmer@samba.org> 1412179807 +0200

dulwich/tests/test_porcelain.py

@@ -224,6 +224,20 @@ class CloneTests(PorcelainTestCase):
             self.repo.path, target_path, checkout=True, errstream=errstream)
         r.close()
 
+    def test_no_head_no_checkout_outstream_errstream_autofallback(self):
+        f1_1 = make_object(Blob, data=b'f1')
+        commit_spec = [[1]]
+        trees = {1: [(b'f1', f1_1), (b'f2', f1_1)]}
+
+        (c1, ) = build_commit_graph(self.repo.object_store, commit_spec, trees)
+        self.repo.refs[b"refs/heads/master"] = c1.id
+        target_path = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, target_path)
+        errstream = porcelain.NoneStream()
+        r = porcelain.clone(
+            self.repo.path, target_path, checkout=True, errstream=errstream)
+        r.close()
+
 
 class InitTests(TestCase):
 
@@ -1490,6 +1504,7 @@ class DescribeTests(PorcelainTestCase):
 
 
 class HelperTests(PorcelainTestCase):
+
     def test_path_to_tree_path_base(self):
         self.assertEqual(
             b'bar', porcelain.path_to_tree_path('/home/foo', '/home/foo/bar'))
@@ -1526,3 +1541,24 @@ class HelperTests(PorcelainTestCase):
                 os.path.join(os.getcwd(), '..'), 'baz'))
         finally:
             os.chdir(cwd)
+
+
+class GetObjectBypathTests(PorcelainTestCase):
+
+    def test_simple(self):
+        fullpath = os.path.join(self.repo.path, 'foo')
+        with open(fullpath, 'w') as f:
+            f.write("BAR")
+        porcelain.add(repo=self.repo.path, paths=[fullpath])
+        porcelain.commit(
+                self.repo.path, message=b"Some message",
+                author=b"Joe <joe@example.com>",
+                committer=b"Bob <bob@example.com>")
+        self.assertEqual(
+            b"BAR",
+            porcelain.get_object_by_path(self.repo, 'foo').data)
+
+    def test_missing(self):
+        self.assertRaises(
+            KeyError,
+            porcelain.get_object_by_path, self.repo, 'foo')

dulwich/tests/test_repository.py

@@ -662,13 +662,28 @@ class BuildRepoRootTests(TestCase):
         self.assertEqual([], r[commit_sha].parents)
         self._root_commit = commit_sha
 
-    def test_shallow(self):
+    def test_get_shallow(self):
         self.assertEqual(set(), self._repo.get_shallow())
         with open(os.path.join(self._repo.path, '.git', 'shallow'), 'wb') as f:
             f.write(b'a90fa2d900a17e99b433217e988c4eb4a2e9a097\n')
         self.assertEqual({b'a90fa2d900a17e99b433217e988c4eb4a2e9a097'},
                          self._repo.get_shallow())
 
+    def test_update_shallow(self):
+        self._repo.update_shallow(None, None)  # no op
+        self.assertEquals(set(), self._repo.get_shallow())
+        self._repo.update_shallow(
+                [b'a90fa2d900a17e99b433217e988c4eb4a2e9a097'],
+                None)
+        self.assertEqual(
+                {b'a90fa2d900a17e99b433217e988c4eb4a2e9a097'},
+                self._repo.get_shallow())
+        self._repo.update_shallow(
+                [b'a90fa2d900a17e99b433217e988c4eb4a2e9a097'],
+                [b'f9e39b120c68182a4ba35349f832d0e4e61f485c'])
+        self.assertEqual({b'a90fa2d900a17e99b433217e988c4eb4a2e9a097'},
+                         self._repo.get_shallow())
+
     def test_build_repo(self):
         r = self._repo
         self.assertEqual(b'ref: refs/heads/master', r.refs.read_ref(b'HEAD'))
@@ -748,6 +763,19 @@ class BuildRepoRootTests(TestCase):
             encoding=b"iso8859-1")
         self.assertEqual(b"iso8859-1", r[commit_sha].encoding)
 
+    def test_commit_encoding_from_config(self):
+        r = self._repo
+        c = r.get_config()
+        c.set(('i18n',), 'commitEncoding', 'iso8859-1')
+        c.write_to_path()
+        commit_sha = r.do_commit(
+            b'commit with strange character \xee',
+            committer=b'Test Committer <test@nodomain.com>',
+            author=b'Test Author <test@nodomain.com>',
+            commit_timestamp=12395, commit_timezone=0,
+            author_timestamp=12395, author_timezone=0)
+        self.assertEqual(b"iso8859-1", r[commit_sha].encoding)
+
     def test_commit_config_identity(self):
         # commit falls back to the users' identity if it wasn't specified
         r = self._repo

requirements.txt

@@ -0,0 +1 @@
+urllib3[secure]==1.22

setup.cfg

@@ -1,29 +1,3 @@
-[metadata]
-name = dulwich
-author = Jelmer Vernooij
-author_email = jelmer@jelmer.uk
-home-page = https://www.dulwich.io/
-description = file:README.md
-summary = Python Git Library
-classifiers = 
-	Development Status :: 4 - Beta
-	License :: OSI Approved :: Apache Software License
-	Programming Language :: Python :: 2.7
-	Programming Language :: Python :: 3.3
-	Programming Language :: Python :: 3.4
-	Programming Language :: Python :: 3.5
-	Programming Language :: Python :: 3.6
-	Programming Language :: Python :: Implementation :: CPython
-	Programming Language :: Python :: Implementation :: PyPy
-	Operating System :: POSIX
-	Operating System :: Microsoft :: Windows
-	Topic :: Software Development :: Version Control
-keyword = git, vcs
-project_urls = 
-	Bug Tracker = https://github.com/dulwich/dulwich/issues
-
-[build_ext]
-
 [egg_info]
 tag_build = 
 tag_date = 0

setup.py

@@ -14,7 +14,7 @@ from distutils.core import Distribution
 import os
 import sys
 
-dulwich_version_string = '0.19.6'
+dulwich_version_string = '0.19.7'
 
 include_dirs = []
 # Windows MSVC support
@@ -83,15 +83,41 @@ if has_setuptools:
     setup_kwargs['test_suite'] = 'dulwich.tests.test_suite'
     setup_kwargs['tests_require'] = tests_require
 
+with open('README.md', 'r') as f:
+    description = f.read()
 
 setup(name='dulwich',
+      author="Jelmer Vernooij",
+      author_email="jelmer@jelmer.uk",
+      url="https://www.dulwich.io/",
+      long_description=description,
+      description="Python Git Library",
       version=dulwich_version_string,
       license='Apachev2 or later or GPLv2',
+      project_urls={
+          "Bug Tracker": "https://github.com/dulwich/dulwich/issues",
+          "Repository": "https://www.dulwich.io/code/",
+          "GitHub": "https://github.com/dulwich/dulwich",
+      },
+      keywords="git vcs",
       packages=['dulwich', 'dulwich.tests', 'dulwich.tests.compat',
                 'dulwich.contrib'],
       package_data={'': ['../docs/tutorial/*.txt']},
       scripts=['bin/dulwich', 'bin/dul-receive-pack', 'bin/dul-upload-pack'],
       ext_modules=ext_modules,
       distclass=DulwichDistribution,
+      classifiers=[
+          'Development Status :: 4 - Beta',
+          'License :: OSI Approved :: Apache Software License',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3.4',
+          'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
+          'Programming Language :: Python :: Implementation :: CPython',
+          'Programming Language :: Python :: Implementation :: PyPy',
+          'Operating System :: POSIX',
+          'Operating System :: Microsoft :: Windows',
+          'Topic :: Software Development :: Version Control',
+      ],
       **setup_kwargs
       )