New upstream version 0.19.10

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+branch = True
+source = dulwich
+
+[report]
+exclude_lines =
+    raise NotImplementedError

.gitignore

@@ -0,0 +1,24 @@
+_trial_temp
+build
+build-pypy
+MANIFEST
+dist
+apidocs
+*,cover
+.testrepository
+*.pyc
+*.pyd
+*.pyo
+*.so
+*~
+*.swp
+*.swh
+*.swn
+*.swo
+docs/tutorial/index.html
+dulwich.egg-info/
+.tox/
+.idea/
+.coverage
+htmlcov/
+docs/api/*.txt

.mailmap

@@ -0,0 +1,23 @@
+Jelmer Vernooij <jelmer@jelmer.uk>
+Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@jelmer.uk>
+Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@samba.org>
+Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@debian.org>
+Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@canonical.com>
+Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@google.com>
+Martin <gzlist@googlemail.com> <martin.packman@canonical.com>
+Dave Borowitz <dborowitz@google.com> <ddborowitz@gmail.com>
+Dave Borowitz <dborowitz@google.com> <dborowitz@google.com>
+John Carr <john.carr@unrouted.co.uk>
+Mark Mikofski <bwanamarko@yahoo.com> <mark.mikofski@sunpowercorp.com>
+Mark Mikofski <bwanamarko@yahoo.com> <bwana.marko@yahoo.com>
+David Carr <david@carrclan.us>
+Jon Bain <jsbain@yahoo.com> <jsbain@yahoo.com>
+James Westby <jw+debian@jameswestby.net> <jw+debian@jameswestby.net>
+David Keijser <david.keijser@klarna.com> <keijser@gmail.com>
+Benoît HERVIER <khertan@khertan.net> <khertan@khertan.net>
+Ryan Faulkner <rfaulk@yahoo-inc.com> <rfaulkner@wikimedia.org>
+David Bennett <davbennett@google.com> <david@dbinit.com>
+Risto Kankkunen <risto.kankkunen@iki.fi> <risto.kankkunen@f-secure.com>
+Augie Fackler <durin42@gmail.com> <raf@durin42.com>
+Damien Tournoud <damien@commerceguys.com> <damien@platform.sh>
+Marcin Kuźmiński <marcin@python-blog.com> <marcin@python-works.com>

.travis.yml

@@ -6,10 +6,8 @@ python:
   - 2.7
   - 3.4
   - 3.5
-  - 3.5-dev
   - 3.6
   - 3.6-dev
-  - pypy3.3-5.2-alpha1
   - pypy3.5
 
 env:
@@ -20,17 +18,15 @@ matrix:
   include:
     - python: pypy
       env: TEST_REQUIRE=fastimport
-    - python: 3.3
-      env: TEST_REQUIRE=fastimport
     - python: 3.7
       env: TEST_REQUIRE=fastimport
       dist: xenial
       sudo: true
-    - python: 3.8-dev
-      env: TEST_REQUIRE=fastimport
-      dist: xenial
-      sudo: true
-      
+    # flakes checker fails on python 3.8-dev:
+    #- python: 3.8-dev
+    #  env: TEST_REQUIRE=fastimport
+    #  dist: xenial
+    #  sudo: true
 
 install:
   - travis_retry pip install -U pip coverage codecov flake8 $TEST_REQUIRE

AUTHORS

@@ -144,5 +144,7 @@ Daniel M. Capella <polyzen@users.noreply.github.com>
 grun <grunseid@gmail.com>
 Sylvia van Os <sylvia@hackerchick.me>
 Boris Feld <lothiraldan@gmail.com>
+KS Chan <mrkschan@gmail.com>
+egor <egor@sourced.tech>
 
 If you contributed but are missing from this list, please send me an e-mail.

CONTRIBUTING.md → CONTRIBUTING.rst

@@ -23,7 +23,8 @@ Running the tests
 To run the testsuite, you should be able to simply run "make check". This
 will run the tests using unittest.
 
- $ make check
+::
+   $ make check
 
 Tox configuration is also present as well as a Travis configuration file.
 

MANIFEST.in

@@ -1,10 +1,10 @@
 include NEWS
 include AUTHORS
-include README.md
-include README.swift.md
+include README.rst
+include README.swift.rst
 include Makefile
 include COPYING
-include CONTRIBUTING.md
+include CONTRIBUTING.rst
 include TODO
 include setup.cfg
 include dulwich/stdint.h

Makefile

@@ -3,7 +3,6 @@ PYFLAKES = pyflakes
 PEP8 = pep8
 FLAKE8 ?= flake8
 SETUP = $(PYTHON) setup.py
-PYDOCTOR ?= pydoctor
 TESTRUNNER ?= unittest
 RUNTEST = PYTHONHASHSEED=random PYTHONPATH=$(shell pwd)$(if $(PYTHONPATH),:$(PYTHONPATH),) $(PYTHON) -m $(TESTRUNNER) $(TEST_OPTIONS)
 COVERAGE = python3-coverage
@@ -12,15 +11,11 @@ DESTDIR=/
 
 all: build
 
-doc:: pydoctor
 doc:: sphinx
 
 sphinx::
 	$(MAKE) -C docs html
 
-pydoctor::
-	$(PYDOCTOR) --make-html -c dulwich.cfg
-
 build::
 	$(SETUP) build
 	$(SETUP) build_ext -i
@@ -59,7 +54,7 @@ pep8:
 	$(PEP8) dulwich
 
 style:
-	$(FLAKE8) --exclude=build,.git,build-pypy,.tox
+	$(FLAKE8)
 
 before-push: check
 	git diff origin/master | $(PEP8) --diff

NEWS

@@ -1,3 +1,85 @@
+0.19.10	2018-01-15
+
+ IMPROVEMENTS
+
+ * Add `dulwich.porcelain.write_tree`.
+   (Jelmer Vernooij)
+
+ * Support reading ``MERGE_HEADS`` in ``Repo.do_commit``.
+   (Jelmer Vernooij)
+
+ * Import from ``collections.abc`` rather than ``collections`` where
+   applicable. Required for 3.8 compatibility.
+   (Jelmer Vernooij)
+
+ * Support plain strings as refspec arguments to
+   ``dulwich.porcelain.push``. (Jelmer Vernooij)
+
+ * Add support for creating signed tags.
+   (Jelmer Vernooij, #542)
+
+ BUG FIXES
+
+ *  Handle invalid ref that pretends to be a sub-folder under a valid ref.
+    (KS Chan)
+
+0.19.9	2018-11-17
+
+ BUG FIXES
+
+ * Avoid fetching ghosts in ``Repo.fetch``.
+   (Jelmer Vernooij)
+
+ * Preserve port and username in parsed HTTP URLs.
+   (Jelmer Vernooij)
+
+ * Add basic server side implementation of ``git-upload-archive``.
+   (Jelmer Vernooij)
+
+0.19.8	2018-11-06
+
+ * Fix encoding when reading README file in setup.py.
+   (egor <egor@sourced.tech>, #668)
+
+0.19.7	2018-11-05
+
+ CHANGES
+
+  * Drop support for Python 3 < 3.4. This is because
+    pkg_resources (which get used by setuptools and mock)
+    no longer supports 3.3 and earlier. (Jelmer Vernooij)
+
+ IMPROVEMENTS
+
+  * Support ``depth`` argument to ``GitClient.fetch_pack`` and support
+    fetching and updating shallow metadata. (Jelmer Vernooij, #240)
+
+ BUG FIXES
+
+  * Don't write to stdout and stderr when they are not available
+    (such as is the case for pythonw). (Sylvia van Os, #652)
+
+  * Fix compatibility with newer versions of git, which expect CONTENT_LENGTH
+    to be set to 0 for empty body requests. (Jelmer Vernooij, #657)
+
+  * Raise an exception client-side when a caller tries to request
+    SHAs that are not directly referenced the servers' refs.
+    (Jelmer Vernooij)
+
+  * Raise more informative errors when unable to connect to repository
+    over SSH or subprocess. (Jelmer Vernooij)
+
+  * Handle commit identity fields with multiple ">" characters.
+    (Nicolas Dandrimont)
+
+ IMPROVEMENTS
+
+  * ``dulwich.porcelain.get_object_by_path`` method for easily
+    accessing a path in another tree. (Jelmer Vernooij)
+
+  * Support the ``i18n.commitEncoding`` setting in config.
+    (Jelmer Vernooij)
+
 0.19.6	2018-08-11
 
  BUG FIXES
@@ -12,6 +94,9 @@
   * Support paths as bytestrings in various places in ``dulwich.index``
     (Jelmer Vernooij)
 
+  * Avoid setup.cfg for now, since it seems to break pypi metadata.
+    (Jelmer Vernooij, #658)
+
 0.19.5	2018-07-08
 
  IMPROVEMENTS

PKG-INFO

@@ -1,112 +1,119 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.19.6
-Summary: [![Build Status](https://travis-ci.org/dulwich/dulwich.png?branch=master)](https://travis-ci.org/dulwich/dulwich)
-[![Windows Build status](https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true)](https://ci.appveyor.com/project/jelmer/dulwich/branch/master)
-
-This is the Dulwich project.
-
-It aims to provide an interface to git repos (both local and remote) that
-doesn't call out to git directly but instead uses pure Python.
-
-**Main website**: [www.dulwich.io](https://www.dulwich.io/)
-
-**License**: Apache License, version 2 or GNU General Public License, version 2 or later.
-
-The project is named after the part of London that Mr. and Mrs. Git live in
-in the particular Monty Python sketch.
-
-Installation
-------------
-
-By default, Dulwich' setup.py will attempt to build and install the optional C
-extensions. The reason for this is that they significantly improve the performance
-since some low-level operations that are executed often are much slower in CPython.
-
-If you don't want to install the C bindings, specify the --pure argument to setup.py::
-
-    $ python setup.py --pure install
-
-or if you are installing from pip::
-
-    $ pip install dulwich --global-option="--pure"
-
-Note that you can also specify --global-option in a
-[requirements.txt](https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers)
-file, e.g. like this::
-
-    dulwich --global-option=--pure
-
-Getting started
----------------
-
-Dulwich comes with both a lower-level API and higher-level plumbing ("porcelain").
-
-For example, to use the lower level API to access the commit message of the
-last commit:
-
-    >>> from dulwich.repo import Repo
-    >>> r = Repo('.')
-    >>> r.head()
-    '57fbe010446356833a6ad1600059d80b1e731e15'
-    >>> c = r[r.head()]
-    >>> c
-    <Commit 015fc1267258458901a94d228e39f0a378370466>
-    >>> c.message
-    'Add note about encoding.\n'
-
-And to print it using porcelain:
-
-    >>> from dulwich import porcelain
-    >>> porcelain.log('.', max_entries=1)
-    --------------------------------------------------
-    commit: 57fbe010446356833a6ad1600059d80b1e731e15
-    Author: Jelmer Vernooij <jelmer@jelmer.uk>
-    Date:   Sat Apr 29 2017 23:57:34 +0000
-
-    Add note about encoding.
-
-Further documentation
----------------------
-
-The dulwich documentation can be found in docs/ and
-[on the web](https://www.dulwich.io/docs/).
-
-The API reference can be generated using pydoctor, by running "make pydoctor",
-or [on the web](https://www.dulwich.io/apidocs).
-
-Help
-----
-
-There is a *#dulwich* IRC channel on the [Freenode](https://www.freenode.net/), and
-[dulwich-announce](https://groups.google.com/forum/#!forum/dulwich-announce)
-and [dulwich-discuss](https://groups.google.com/forum/#!forum/dulwich-discuss)
-mailing lists.
-
-Contributing
-------------
-
-For a full list of contributors, see the git logs or [AUTHORS](AUTHORS).
-
-If you'd like to contribute to Dulwich, see the [CONTRIBUTING](CONTRIBUTING.md)
-file and [list of open issues](https://github.com/dulwich/dulwich/issues).
-
-Supported versions of Python
-----------------------------
-
-At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.3, 3.4, 3.5, 3.6 and Pypy.
-
+Version: 0.19.10
+Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij
 Author-email: jelmer@jelmer.uk
 License: Apachev2 or later or GPLv2
 Project-URL: Bug Tracker, https://github.com/dulwich/dulwich/issues
-Description: UNKNOWN
+Project-URL: GitHub, https://github.com/dulwich/dulwich
+Project-URL: Repository, https://www.dulwich.io/code/
+Description: .. image:: https://travis-ci.org/dulwich/dulwich.png?branch=master
+          :alt: Build Status
+          :target: https://travis-ci.org/dulwich/dulwich
+        
+        .. image:: https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true
+          :alt: Windows Build Status
+          :target: https://ci.appveyor.com/project/jelmer/dulwich/branch/master
+        
+        This is the Dulwich project.
+        
+        It aims to provide an interface to git repos (both local and remote) that
+        doesn't call out to git directly but instead uses pure Python.
+        
+        **Main website**: <https://www.dulwich.io/>
+        
+        **License**: Apache License, version 2 or GNU General Public License, version 2 or later.
+        
+        The project is named after the part of London that Mr. and Mrs. Git live in
+        in the particular Monty Python sketch.
+        
+        Installation
+        ------------
+        
+        By default, Dulwich' setup.py will attempt to build and install the optional C
+        extensions. The reason for this is that they significantly improve the performance
+        since some low-level operations that are executed often are much slower in CPython.
+        
+        If you don't want to install the C bindings, specify the --pure argument to setup.py::
+        
+            $ python setup.py --pure install
+        
+        or if you are installing from pip::
+        
+            $ pip install dulwich --global-option="--pure"
+        
+        Note that you can also specify --global-option in a
+        `requirements.txt <https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers>`_
+        file, e.g. like this::
+        
+            dulwich --global-option=--pure
+        
+        Getting started
+        ---------------
+        
+        Dulwich comes with both a lower-level API and higher-level plumbing ("porcelain").
+        
+        For example, to use the lower level API to access the commit message of the
+        last commit::
+        
+            >>> from dulwich.repo import Repo
+            >>> r = Repo('.')
+            >>> r.head()
+            '57fbe010446356833a6ad1600059d80b1e731e15'
+            >>> c = r[r.head()]
+            >>> c
+            <Commit 015fc1267258458901a94d228e39f0a378370466>
+            >>> c.message
+            'Add note about encoding.\n'
+        
+        And to print it using porcelain::
+        
+            >>> from dulwich import porcelain
+            >>> porcelain.log('.', max_entries=1)
+            --------------------------------------------------
+            commit: 57fbe010446356833a6ad1600059d80b1e731e15
+            Author: Jelmer Vernooij <jelmer@jelmer.uk>
+            Date:   Sat Apr 29 2017 23:57:34 +0000
+        
+            Add note about encoding.
+        
+        Further documentation
+        ---------------------
+        
+        The dulwich documentation can be found in docs/ and
+        `on the web <https://www.dulwich.io/docs/>`_.
+        
+        The API reference can be generated using pydoctor, by running "make pydoctor",
+        or `on the web <https://www.dulwich.io/apidocs>`_.
+        
+        Help
+        ----
+        
+        There is a *#dulwich* IRC channel on the `Freenode <https://www.freenode.net/>`_, and
+        `dulwich-announce <https://groups.google.com/forum/#!forum/dulwich-announce>`_
+        and `dulwich-discuss <https://groups.google.com/forum/#!forum/dulwich-discuss>`_
+        mailing lists.
+        
+        Contributing
+        ------------
+        
+        For a full list of contributors, see the git logs or `AUTHORS <AUTHORS>`_.
+        
+        If you'd like to contribute to Dulwich, see the `CONTRIBUTING <CONTRIBUTING.rst>`_
+        file and `list of open issues <https://github.com/dulwich/dulwich/issues>`_.
+        
+        Supported versions of Python
+        ----------------------------
+        
+        At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6 and Pypy.
+        
+Keywords: git vcs
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.3
 Classifier: Programming Language :: Python :: 3.4
 Classifier: Programming Language :: Python :: 3.5
 Classifier: Programming Language :: Python :: 3.6

README.md → README.rst

@@ -1,12 +1,17 @@
-[![Build Status](https://travis-ci.org/dulwich/dulwich.png?branch=master)](https://travis-ci.org/dulwich/dulwich)
-[![Windows Build status](https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true)](https://ci.appveyor.com/project/jelmer/dulwich/branch/master)
+.. image:: https://travis-ci.org/dulwich/dulwich.png?branch=master
+  :alt: Build Status
+  :target: https://travis-ci.org/dulwich/dulwich
+
+.. image:: https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true
+  :alt: Windows Build Status
+  :target: https://ci.appveyor.com/project/jelmer/dulwich/branch/master
 
 This is the Dulwich project.
 
 It aims to provide an interface to git repos (both local and remote) that
 doesn't call out to git directly but instead uses pure Python.
 
-**Main website**: [www.dulwich.io](https://www.dulwich.io/)
+**Main website**: <https://www.dulwich.io/>
 
 **License**: Apache License, version 2 or GNU General Public License, version 2 or later.
 
@@ -29,7 +34,7 @@ or if you are installing from pip::
     $ pip install dulwich --global-option="--pure"
 
 Note that you can also specify --global-option in a
-[requirements.txt](https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers)
+`requirements.txt <https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers>`_
 file, e.g. like this::
 
     dulwich --global-option=--pure
@@ -40,7 +45,7 @@ Getting started
 Dulwich comes with both a lower-level API and higher-level plumbing ("porcelain").
 
 For example, to use the lower level API to access the commit message of the
-last commit:
+last commit::
 
     >>> from dulwich.repo import Repo
     >>> r = Repo('.')
@@ -52,7 +57,7 @@ last commit:
     >>> c.message
     'Add note about encoding.\n'
 
-And to print it using porcelain:
+And to print it using porcelain::
 
     >>> from dulwich import porcelain
     >>> porcelain.log('.', max_entries=1)
@@ -67,28 +72,28 @@ Further documentation
 ---------------------
 
 The dulwich documentation can be found in docs/ and
-[on the web](https://www.dulwich.io/docs/).
+`on the web <https://www.dulwich.io/docs/>`_.
 
 The API reference can be generated using pydoctor, by running "make pydoctor",
-or [on the web](https://www.dulwich.io/apidocs).
+or `on the web <https://www.dulwich.io/apidocs>`_.
 
 Help
 ----
 
-There is a *#dulwich* IRC channel on the [Freenode](https://www.freenode.net/), and
-[dulwich-announce](https://groups.google.com/forum/#!forum/dulwich-announce)
-and [dulwich-discuss](https://groups.google.com/forum/#!forum/dulwich-discuss)
+There is a *#dulwich* IRC channel on the `Freenode <https://www.freenode.net/>`_, and
+`dulwich-announce <https://groups.google.com/forum/#!forum/dulwich-announce>`_
+and `dulwich-discuss <https://groups.google.com/forum/#!forum/dulwich-discuss>`_
 mailing lists.
 
 Contributing
 ------------
 
-For a full list of contributors, see the git logs or [AUTHORS](AUTHORS).
+For a full list of contributors, see the git logs or `AUTHORS <AUTHORS>`_.
 
-If you'd like to contribute to Dulwich, see the [CONTRIBUTING](CONTRIBUTING.md)
-file and [list of open issues](https://github.com/dulwich/dulwich/issues).
+If you'd like to contribute to Dulwich, see the `CONTRIBUTING <CONTRIBUTING.rst>`_
+file and `list of open issues <https://github.com/dulwich/dulwich/issues>`_.
 
 Supported versions of Python
 ----------------------------
 
-At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.3, 3.4, 3.5, 3.6 and Pypy.
+At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6 and Pypy.

README.swift.md → README.swift.rst

@@ -18,7 +18,7 @@ Configuration file
 
 We need to provide some configuration values in order to let Dulwich
 talk and authenticate against Swift. The following config file must
-be used as template:
+be used as template::
 
     [swift]
     # Authentication URL (Keystone or Swift)
@@ -54,7 +54,7 @@ How to start unittest
 ---------------------
 
 There is no need to have a Swift cluster running to run the unitests.
-Just run the following command in the Dulwich source directory:
+Just run the following command in the Dulwich source directory::
 
     $ PYTHONPATH=. python -m dulwich.contrib.test_swift
 
@@ -63,7 +63,7 @@ How to start functional tests
 
 We provide some basic tests to perform smoke tests against a real Swift
 cluster. To run those functional tests you need a properly configured
-configuration file. The tests can be run as follow:
+configuration file. The tests can be run as follow::
 
     $ DULWICH_SWIFT_CFG=/etc/swift-dul.conf PYTHONPATH=. python -m dulwich.contrib.test_swift_smoke
 
@@ -71,14 +71,14 @@ How to install
 --------------
 
 Install the Dulwich library via the setup.py. The dependencies will be
-automatically retrieved from pypi:
+automatically retrieved from pypi::
 
     $ python ./setup.py install
 
 How to run the server
 ---------------------
 
-Start the server using the following command:
+Start the server using the following command::
 
     $ python -m dulwich.contrib.swift daemon -c /etc/swift-dul.conf -l 127.0.0.1
 
@@ -92,7 +92,7 @@ How to use
 
 Once you have validated that the functional tests is working as expected and
 the server is running we can init a bare repository. Run this
-command with the name of the repository to create:
+command with the name of the repository to create::
 
     $ python -m dulwich.contrib.swift init -c /etc/swift-dul.conf edeploy
 
@@ -100,11 +100,11 @@ The repository name will be the container that will contain all the Git
 objects for the repository. Then standard c Git client can be used to
 perform operations against this repository.
 
-As an example we can clone the previously empty bare repository:
+As an example we can clone the previously empty bare repository::
 
     $ git clone git://localhost/edeploy
 
-Then push an existing project in it:
+Then push an existing project in it::
 
     $ git clone https://github.com/enovance/edeploy.git edeployclone
     $ cd edeployclone

appveyor.yml

@@ -10,15 +10,6 @@ environment:
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "64"
 
-    - PYTHON: "C:\\Python33"
-      PYTHON_VERSION: "3.3.x"
-      PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python33-x64"
-      PYTHON_VERSION: "3.3.x"
-      PYTHON_ARCH: "64"
-      DISTUTILS_USE_SDK: "1"
-
     - PYTHON: "C:\\Python34"
       PYTHON_VERSION: "3.4.x"
       PYTHON_ARCH: "32"
@@ -84,6 +75,7 @@ build_script:
 
 test_script:
   - "build.cmd %PYTHON%\\python.exe setup.py test"
+  - "build.cmd %PYTHON%\\pythonw.exe setup.py test"
 
 after_test:
   - "build.cmd %PYTHON%\\python.exe setup.py bdist_wheel"

bin/dulwich

@@ -223,8 +223,13 @@ class cmd_init(Command):
 class cmd_clone(Command):
 
     def run(self, args):
-        opts, args = getopt(args, "", ["bare"])
-        opts = dict(opts)
+        parser = optparse.OptionParser()
+        parser.add_option("--bare", dest="bare",
+                          help="Whether to create a bare repository.",
+                          action="store_true")
+        parser.add_option("--depth", dest="depth",
+                          type=int, help="Depth at which to fetch")
+        options, args = parser.parse_args(args)
 
         if args == []:
             print("usage: dulwich clone host:path [PATH]")
@@ -236,7 +241,7 @@ class cmd_clone(Command):
         else:
             target = None
 
-        porcelain.clone(source, target, bare=("--bare" in opts))
+        porcelain.clone(source, target, bare=options.bare, depth=options.depth)
 
 
 class cmd_commit(Command):
@@ -306,11 +311,13 @@ class cmd_rev_list(Command):
 class cmd_tag(Command):
 
     def run(self, args):
-        opts, args = getopt(args, '', [])
-        if len(args) < 2:
-            print('Usage: dulwich tag NAME')
-            sys.exit(1)
-        porcelain.tag('.', args[0])
+        parser = optparse.OptionParser()
+        parser.add_option("-a", "--annotated", help="Create an annotated tag.", action="store_true")
+        parser.add_option("-s", "--sign", help="Sign the annotated tag.", action="store_true")
+        options, args = parser.parse_args(args)
+        porcelain.tag_create(
+            '.', args[0], annotated=options.annotated,
+            sign=options.sign)
 
 
 class cmd_repack(Command):
@@ -383,6 +390,14 @@ class cmd_web_daemon(Command):
                              port=options.port)
 
 
+class cmd_write_tree(Command):
+
+    def run(self, args):
+        parser = optparse.OptionParser()
+        options, args = parser.parse_args(args)
+        sys.stdout.write('%s\n' % porcelain.write_tree('.'))
+
+
 class cmd_receive_pack(Command):
 
     def run(self, args):
@@ -684,6 +699,7 @@ commands = {
     "update-server-info": cmd_update_server_info,
     "upload-pack": cmd_upload_pack,
     "web-daemon": cmd_web_daemon,
+    "write-tree": cmd_write_tree,
     }
 
 if len(sys.argv) < 2:

build.cmd

@@ -0,0 +1,21 @@
+@echo off
+:: To build extensions for 64 bit Python 3, we need to configure environment
+:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 4
+::
+:: More details at:
+:: https://github.com/cython/cython/wiki/CythonExtensionsOnWindows
+
+IF "%DISTUTILS_USE_SDK%"=="1" (
+    ECHO Configuring environment to build with MSVC on a 64bit architecture
+    ECHO Using Windows SDK 7.1
+    "C:\Program Files\Microsoft SDKs\Windows\v7.1\Setup\WindowsSdkVer.exe" -q -version:v7.1
+    CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release
+    SET MSSdk=1
+    REM Need the following to allow tox to see the SDK compiler
+    SET TOX_TESTENV_PASSENV=DISTUTILS_USE_SDK MSSdk INCLUDE LIB
+) ELSE (
+    ECHO Using default MSVC build environment
+)
+
+CALL %*

devscripts/PREAMBLE.c

@@ -0,0 +1,15 @@
+ * Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+ * General Public License as public by the Free Software Foundation; version 2.0
+ * or (at your option) any later version. You can redistribute it and/or
+ * modify it under the terms of either of these two licenses.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * You should have received a copy of the licenses; if not, see
+ * <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+ * and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+ * License, Version 2.0.

devscripts/PREAMBLE.py

@@ -0,0 +1,16 @@
+# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+# General Public License as public by the Free Software Foundation; version 2.0
+# or (at your option) any later version. You can redistribute it and/or
+# modify it under the terms of either of these two licenses.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# You should have received a copy of the licenses; if not, see
+# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+# License, Version 2.0.
+#

devscripts/replace-preamble.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/zsh
+perl -i -p0e "s{\Q$(cat PREAMBLE.py.old)\E}{$(cat devscripts/PREAMBLE.py)}g" dulwich/**/*.py bin/dul*
+perl -i -p0e "s{\Q$(cat PREAMBLE.c.old)\E}{$(cat devscripts/PREAMBLE.c)}g" dulwich/*.c

docs/Makefile

@@ -31,33 +31,36 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 
-html:
+apidocs:
+	sphinx-apidoc -feM -s txt -o api ../dulwich
+
+html: apidocs
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
-dirhtml:
+dirhtml: apidocs
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 
-pickle:
+pickle: apidocs
 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 	@echo
 	@echo "Build finished; now you can process the pickle files."
 
-json:
+json: apidocs
 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 	@echo
 	@echo "Build finished; now you can process the JSON files."
 
-htmlhelp:
+htmlhelp: apidocs
 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 	@echo
 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 
-qthelp:
+qthelp: apidocs
 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 	@echo
 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
@@ -66,14 +69,14 @@ qthelp:
 	@echo "To view the help file:"
 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/dulwich.qhc"
 
-latex:
+latex: apidocs
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
 	@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
 	      "run these through (pdf)latex."
 
-changes:
+changes: apidocs
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 	@echo
 	@echo "The overview file is in $(BUILDDIR)/changes."

docs/api/index.txt

@@ -0,0 +1,14 @@
+This is the API documentation for Dulwich.
+
+Module reference
+----------------
+
+.. toctree::
+    :maxdepth: 3
+
+    modules
+
+Indices:
+
+* :ref:`modindex`
+* :ref:`search`

docs/conf.py

@@ -26,7 +26,12 @@ dulwich = __import__('dulwich')
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc']
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.ifconfig',
+    'sphinx.ext.intersphinx',
+    'sphinx_epytext',
+    ]
 try:
     import rst2pdf
     if rst2pdf.version >= '0.16':
@@ -51,7 +56,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'dulwich'
-copyright = u'2011, Jelmer Vernooij'
+copyright = u'2011-2018 Jelmer Vernooij'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -106,7 +111,7 @@ pygments_style = 'sphinx'
 # The theme to use for HTML and HTML Help pages.  Major themes that come with
 # Sphinx are currently 'default' and 'sphinxdoc'.
 # html_theme = 'default'
-html_theme = 'nature'
+html_theme = 'agogo'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

docs/index.txt

@@ -7,12 +7,11 @@ dulwich - Python implementation of Git
 Overview
 ========
 
-.. include:: ../README.md
+.. include:: ../README.rst
 
 Documentation
 =============
 
-
 .. toctree::
     :maxdepth: 2
 
@@ -21,6 +20,8 @@ Documentation
 
     tutorial/index
 
+    api/index
+
 
 Changelog
 =========

docs/tutorial/.gitignore

@@ -0,0 +1,2 @@
+*.html
+myrepo

dulwich.egg-info/PKG-INFO

@@ -1,112 +1,119 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.19.6
-Summary: [![Build Status](https://travis-ci.org/dulwich/dulwich.png?branch=master)](https://travis-ci.org/dulwich/dulwich)
-[![Windows Build status](https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true)](https://ci.appveyor.com/project/jelmer/dulwich/branch/master)
-
-This is the Dulwich project.
-
-It aims to provide an interface to git repos (both local and remote) that
-doesn't call out to git directly but instead uses pure Python.
-
-**Main website**: [www.dulwich.io](https://www.dulwich.io/)
-
-**License**: Apache License, version 2 or GNU General Public License, version 2 or later.
-
-The project is named after the part of London that Mr. and Mrs. Git live in
-in the particular Monty Python sketch.
-
-Installation
-------------
-
-By default, Dulwich' setup.py will attempt to build and install the optional C
-extensions. The reason for this is that they significantly improve the performance
-since some low-level operations that are executed often are much slower in CPython.
-
-If you don't want to install the C bindings, specify the --pure argument to setup.py::
-
-    $ python setup.py --pure install
-
-or if you are installing from pip::
-
-    $ pip install dulwich --global-option="--pure"
-
-Note that you can also specify --global-option in a
-[requirements.txt](https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers)
-file, e.g. like this::
-
-    dulwich --global-option=--pure
-
-Getting started
----------------
-
-Dulwich comes with both a lower-level API and higher-level plumbing ("porcelain").
-
-For example, to use the lower level API to access the commit message of the
-last commit:
-
-    >>> from dulwich.repo import Repo
-    >>> r = Repo('.')
-    >>> r.head()
-    '57fbe010446356833a6ad1600059d80b1e731e15'
-    >>> c = r[r.head()]
-    >>> c
-    <Commit 015fc1267258458901a94d228e39f0a378370466>
-    >>> c.message
-    'Add note about encoding.\n'
-
-And to print it using porcelain:
-
-    >>> from dulwich import porcelain
-    >>> porcelain.log('.', max_entries=1)
-    --------------------------------------------------
-    commit: 57fbe010446356833a6ad1600059d80b1e731e15
-    Author: Jelmer Vernooij <jelmer@jelmer.uk>
-    Date:   Sat Apr 29 2017 23:57:34 +0000
-
-    Add note about encoding.
-
-Further documentation
----------------------
-
-The dulwich documentation can be found in docs/ and
-[on the web](https://www.dulwich.io/docs/).
-
-The API reference can be generated using pydoctor, by running "make pydoctor",
-or [on the web](https://www.dulwich.io/apidocs).
-
-Help
-----
-
-There is a *#dulwich* IRC channel on the [Freenode](https://www.freenode.net/), and
-[dulwich-announce](https://groups.google.com/forum/#!forum/dulwich-announce)
-and [dulwich-discuss](https://groups.google.com/forum/#!forum/dulwich-discuss)
-mailing lists.
-
-Contributing
-------------
-
-For a full list of contributors, see the git logs or [AUTHORS](AUTHORS).
-
-If you'd like to contribute to Dulwich, see the [CONTRIBUTING](CONTRIBUTING.md)
-file and [list of open issues](https://github.com/dulwich/dulwich/issues).
-
-Supported versions of Python
-----------------------------
-
-At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.3, 3.4, 3.5, 3.6 and Pypy.
-
+Version: 0.19.10
+Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij
 Author-email: jelmer@jelmer.uk
 License: Apachev2 or later or GPLv2
 Project-URL: Bug Tracker, https://github.com/dulwich/dulwich/issues
-Description: UNKNOWN
+Project-URL: GitHub, https://github.com/dulwich/dulwich
+Project-URL: Repository, https://www.dulwich.io/code/
+Description: .. image:: https://travis-ci.org/dulwich/dulwich.png?branch=master
+          :alt: Build Status
+          :target: https://travis-ci.org/dulwich/dulwich
+        
+        .. image:: https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true
+          :alt: Windows Build Status
+          :target: https://ci.appveyor.com/project/jelmer/dulwich/branch/master
+        
+        This is the Dulwich project.
+        
+        It aims to provide an interface to git repos (both local and remote) that
+        doesn't call out to git directly but instead uses pure Python.
+        
+        **Main website**: <https://www.dulwich.io/>
+        
+        **License**: Apache License, version 2 or GNU General Public License, version 2 or later.
+        
+        The project is named after the part of London that Mr. and Mrs. Git live in
+        in the particular Monty Python sketch.
+        
+        Installation
+        ------------
+        
+        By default, Dulwich' setup.py will attempt to build and install the optional C
+        extensions. The reason for this is that they significantly improve the performance
+        since some low-level operations that are executed often are much slower in CPython.
+        
+        If you don't want to install the C bindings, specify the --pure argument to setup.py::
+        
+            $ python setup.py --pure install
+        
+        or if you are installing from pip::
+        
+            $ pip install dulwich --global-option="--pure"
+        
+        Note that you can also specify --global-option in a
+        `requirements.txt <https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers>`_
+        file, e.g. like this::
+        
+            dulwich --global-option=--pure
+        
+        Getting started
+        ---------------
+        
+        Dulwich comes with both a lower-level API and higher-level plumbing ("porcelain").
+        
+        For example, to use the lower level API to access the commit message of the
+        last commit::
+        
+            >>> from dulwich.repo import Repo
+            >>> r = Repo('.')
+            >>> r.head()
+            '57fbe010446356833a6ad1600059d80b1e731e15'
+            >>> c = r[r.head()]
+            >>> c
+            <Commit 015fc1267258458901a94d228e39f0a378370466>
+            >>> c.message
+            'Add note about encoding.\n'
+        
+        And to print it using porcelain::
+        
+            >>> from dulwich import porcelain
+            >>> porcelain.log('.', max_entries=1)
+            --------------------------------------------------
+            commit: 57fbe010446356833a6ad1600059d80b1e731e15
+            Author: Jelmer Vernooij <jelmer@jelmer.uk>
+            Date:   Sat Apr 29 2017 23:57:34 +0000
+        
+            Add note about encoding.
+        
+        Further documentation
+        ---------------------
+        
+        The dulwich documentation can be found in docs/ and
+        `on the web <https://www.dulwich.io/docs/>`_.
+        
+        The API reference can be generated using pydoctor, by running "make pydoctor",
+        or `on the web <https://www.dulwich.io/apidocs>`_.
+        
+        Help
+        ----
+        
+        There is a *#dulwich* IRC channel on the `Freenode <https://www.freenode.net/>`_, and
+        `dulwich-announce <https://groups.google.com/forum/#!forum/dulwich-announce>`_
+        and `dulwich-discuss <https://groups.google.com/forum/#!forum/dulwich-discuss>`_
+        mailing lists.
+        
+        Contributing
+        ------------
+        
+        For a full list of contributors, see the git logs or `AUTHORS <AUTHORS>`_.
+        
+        If you'd like to contribute to Dulwich, see the `CONTRIBUTING <CONTRIBUTING.rst>`_
+        file and `list of open issues <https://github.com/dulwich/dulwich/issues>`_.
+        
+        Supported versions of Python
+        ----------------------------
+        
+        At the moment, Dulwich supports (and is tested on) CPython 2.7, 3.4, 3.5, 3.6 and Pypy.
+        
+Keywords: git vcs
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.3
 Classifier: Programming Language :: Python :: 3.4
 Classifier: Programming Language :: Python :: 3.5
 Classifier: Programming Language :: Python :: 3.6

dulwich.egg-info/SOURCES.txt

@@ -1,28 +1,38 @@
+.coveragerc
+.gitignore
+.mailmap
 .testr.conf
 .travis.yml
 AUTHORS
-CONTRIBUTING.md
+CONTRIBUTING.rst
 COPYING
 MANIFEST.in
 Makefile
 NEWS
-README.md
-README.swift.md
+README.rst
+README.swift.rst
 TODO
 appveyor.yml
+build.cmd
 dulwich.cfg
+requirements.txt
 setup.cfg
 setup.py
 tox.ini
 bin/dul-receive-pack
 bin/dul-upload-pack
 bin/dulwich
+devscripts/PREAMBLE.c
+devscripts/PREAMBLE.py
+devscripts/replace-preamble.sh
 docs/Makefile
 docs/conf.py
 docs/index.txt
 docs/make.bat
 docs/performance.txt
 docs/protocol.txt
+docs/api/index.txt
+docs/tutorial/.gitignore
 docs/tutorial/Makefile
 docs/tutorial/conclusion.txt
 docs/tutorial/encoding.txt
@@ -49,6 +59,7 @@ dulwich/greenthreads.py
 dulwich/hooks.py
 dulwich/ignore.py
 dulwich/index.py
+dulwich/line_ending.py
 dulwich/log_utils.py
 dulwich/lru_cache.py
 dulwich/mailmap.py
@@ -72,6 +83,7 @@ dulwich.egg-info/SOURCES.txt
 dulwich.egg-info/dependency_links.txt
 dulwich.egg-info/requires.txt
 dulwich.egg-info/top_level.txt
+dulwich/contrib/README.md
 dulwich/contrib/__init__.py
 dulwich/contrib/paramiko_vendor.py
 dulwich/contrib/release_robot.py
@@ -92,6 +104,7 @@ dulwich/tests/test_greenthreads.py
 dulwich/tests/test_hooks.py
 dulwich/tests/test_ignore.py
 dulwich/tests/test_index.py
+dulwich/tests/test_line_ending.py
 dulwich/tests/test_lru_cache.py
 dulwich/tests/test_mailmap.py
 dulwich/tests/test_missing_obj_finder.py

dulwich.egg-info/requires.txt

@@ -1,8 +1,8 @@
 certifi
-urllib3>=1.21
+urllib3>=1.23
 
 [fastimport]
 fastimport
 
 [https]
-urllib3[secure]>=1.21
+urllib3[secure]>=1.23

dulwich/__init__.py

@@ -22,4 +22,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 19, 6)
+__version__ = (0, 19, 10)

dulwich/client.py

@@ -40,7 +40,6 @@ Known capabilities that are not supported:
 
 from contextlib import closing
 from io import BytesIO, BufferedReader
-import gzip
 import select
 import socket
 import subprocess
@@ -66,6 +65,7 @@ from dulwich.errors import (
     UpdateRefsError,
     )
 from dulwich.protocol import (
+    HangupException,
     _RBUFSIZE,
     agent_string,
     capability_agent,
@@ -77,12 +77,16 @@ from dulwich.protocol import (
     CAPABILITY_OFS_DELTA,
     CAPABILITY_QUIET,
     CAPABILITY_REPORT_STATUS,
+    CAPABILITY_SHALLOW,
     CAPABILITY_SYMREF,
     CAPABILITY_SIDE_BAND_64K,
     CAPABILITY_THIN_PACK,
     CAPABILITIES_REF,
     KNOWN_RECEIVE_CAPABILITIES,
     KNOWN_UPLOAD_CAPABILITIES,
+    COMMAND_DEEPEN,
+    COMMAND_SHALLOW,
+    COMMAND_UNSHALLOW,
     COMMAND_DONE,
     COMMAND_HAVE,
     COMMAND_WANT,
@@ -103,9 +107,19 @@ from dulwich.pack import (
     )
 from dulwich.refs import (
     read_info_refs,
+    ANNOTATED_TAG_SUFFIX,
     )
 
 
+class InvalidWants(Exception):
+    """Invalid wants."""
+
+    def __init__(self, wants):
+        Exception.__init__(
+            self,
+            "requested wants not in server provided refs: %r" % wants)
+
+
 def _fileno_can_read(fileno):
     """Check if a file descriptor is readable."""
     return len(select.select([fileno], [], [], 0)[0]) > 0
@@ -126,7 +140,8 @@ def _win32_peek_avail(handle):
 
 COMMON_CAPABILITIES = [CAPABILITY_OFS_DELTA, CAPABILITY_SIDE_BAND_64K]
 UPLOAD_CAPABILITIES = ([CAPABILITY_THIN_PACK, CAPABILITY_MULTI_ACK,
-                        CAPABILITY_MULTI_ACK_DETAILED] + COMMON_CAPABILITIES)
+                        CAPABILITY_MULTI_ACK_DETAILED, CAPABILITY_SHALLOW]
+                       + COMMON_CAPABILITIES)
 RECEIVE_CAPABILITIES = [CAPABILITY_REPORT_STATUS] + COMMON_CAPABILITIES
 
 
@@ -196,7 +211,7 @@ def read_pkt_refs(proto):
     for pkt in proto.read_pkt_seq():
         (sha, ref) = pkt.rstrip(b'\n').split(None, 1)
         if sha == b'ERR':
-            raise GitProtocolError(ref)
+            raise GitProtocolError(ref.decode('utf-8', 'replace'))
         if server_capabilities is None:
             (ref, server_capabilities) = extract_capabilities(ref)
         refs[ref] = sha
@@ -222,10 +237,13 @@ class FetchPackResult(object):
             'setdefault', 'update', 'values', 'viewitems', 'viewkeys',
             'viewvalues']
 
-    def __init__(self, refs, symrefs, agent):
+    def __init__(self, refs, symrefs, agent, new_shallow=None,
+                 new_unshallow=None):
         self.refs = refs
         self.symrefs = symrefs
         self.agent = agent
+        self.new_shallow = new_shallow
+        self.new_unshallow = new_unshallow
 
     def _warn_deprecated(self):
         import warnings
@@ -268,6 +286,20 @@ class FetchPackResult(object):
                 self.__class__.__name__, self.refs, self.symrefs, self.agent)
 
 
+def _read_shallow_updates(proto):
+    new_shallow = set()
+    new_unshallow = set()
+    for pkt in proto.read_pkt_seq():
+        cmd, sha = pkt.split(b' ', 1)
+        if cmd == COMMAND_SHALLOW:
+            new_shallow.add(sha.strip())
+        elif cmd == COMMAND_UNSHALLOW:
+            new_unshallow.add(sha.strip())
+        else:
+            raise GitProtocolError('unknown command %s' % pkt)
+    return (new_shallow, new_unshallow)
+
+
 # TODO(durin42): this doesn't correctly degrade if the server doesn't
 # support some capabilities. This should work properly with servers
 # that don't support multi_ack.
@@ -331,7 +363,8 @@ class GitClient(object):
         """
         raise NotImplementedError(self.send_pack)
 
-    def fetch(self, path, target, determine_wants=None, progress=None):
+    def fetch(self, path, target, determine_wants=None, progress=None,
+              depth=None):
         """Fetch into a target repository.
 
         :param path: Path to fetch from (as bytestring)
@@ -340,6 +373,7 @@ class GitClient(object):
             to fetch. Receives dictionary of name->sha, should return
             list of shas to fetch. Defaults to all shas.
         :param progress: Optional progress function
+        :param depth: Depth to fetch at
         :return: Dictionary with all remote refs (not just those fetched)
         """
         if determine_wants is None:
@@ -361,16 +395,17 @@ class GitClient(object):
         try:
             result = self.fetch_pack(
                 path, determine_wants, target.get_graph_walker(), f.write,
-                progress)
+                progress=progress, depth=depth)
         except BaseException:
             abort()
             raise
         else:
             commit()
+        target.update_shallow(result.new_shallow, result.new_unshallow)
         return result
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-                   progress=None):
+                   progress=None, depth=None):
         """Retrieve a pack from a git smart server.
 
         :param path: Remote path to fetch from
@@ -380,6 +415,7 @@ class GitClient(object):
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
+        :param depth: Shallow fetch depth
         :return: FetchPackResult object
         """
         raise NotImplementedError(self.fetch_pack)
@@ -540,7 +576,7 @@ class GitClient(object):
         return (negotiated_capabilities, symrefs, agent)
 
     def _handle_upload_pack_head(self, proto, capabilities, graph_walker,
-                                 wants, can_read):
+                                 wants, can_read, depth):
         """Handle the head of a 'git-upload-pack' request.
 
         :param proto: Protocol object to read from
@@ -549,17 +585,34 @@ class GitClient(object):
         :param wants: List of commits to fetch
         :param can_read: function that returns a boolean that indicates
             whether there is extra graph data to read on proto
+        :param depth: Depth for request
         """
         assert isinstance(wants, list) and isinstance(wants[0], bytes)
         proto.write_pkt_line(COMMAND_WANT + b' ' + wants[0] + b' ' +
                              b' '.join(capabilities) + b'\n')
         for want in wants[1:]:
             proto.write_pkt_line(COMMAND_WANT + b' ' + want + b'\n')
-        proto.write_pkt_line(None)
+        if depth not in (0, None) or getattr(graph_walker, 'shallow', None):
+            if CAPABILITY_SHALLOW not in capabilities:
+                raise GitProtocolError(
+                    "server does not support shallow capability required for "
+                    "depth")
+            for sha in graph_walker.shallow:
+                proto.write_pkt_line(COMMAND_SHALLOW + b' ' + sha + b'\n')
+            proto.write_pkt_line(COMMAND_DEEPEN + b' ' +
+                                 str(depth).encode('ascii') + b'\n')
+            proto.write_pkt_line(None)
+            if can_read is not None:
+                (new_shallow, new_unshallow) = _read_shallow_updates(proto)
+            else:
+                new_shallow = new_unshallow = None
+        else:
+            new_shallow = new_unshallow = set()
+            proto.write_pkt_line(None)
         have = next(graph_walker)
         while have:
             proto.write_pkt_line(COMMAND_HAVE + b' ' + have + b'\n')
-            if can_read():
+            if can_read is not None and can_read():
                 pkt = proto.read_pkt_line()
                 parts = pkt.rstrip(b'\n').split(b' ')
                 if parts[0] == b'ACK':
@@ -574,6 +627,7 @@ class GitClient(object):
                             parts[2])
             have = next(graph_walker)
         proto.write_pkt_line(COMMAND_DONE + b'\n')
+        return (new_shallow, new_unshallow)
 
     def _handle_upload_pack_tail(self, proto, capabilities, graph_walker,
                                  pack_data, progress=None, rbufsize=_RBUFSIZE):
@@ -613,6 +667,31 @@ class GitClient(object):
                 pack_data(data)
 
 
+def check_wants(wants, refs):
+    """Check that a set of wants is valid.
+
+    :param wants: Set of object SHAs to fetch
+    :param refs: Refs dictionary to check against
+    """
+    missing = set(wants) - {
+            v for (k, v) in refs.items()
+            if not k.endswith(ANNOTATED_TAG_SUFFIX)}
+    if missing:
+        raise InvalidWants(missing)
+
+
+def remote_error_from_stderr(stderr):
+    """Return an appropriate exception based on stderr output. """
+    if stderr is None:
+        return HangupException()
+    for l in stderr.readlines():
+        if l.startswith(b'ERROR: '):
+            return GitProtocolError(
+                l[len(b'ERROR: '):].decode('utf-8', 'replace'))
+        return GitProtocolError(l.decode('utf-8', 'replace'))
+    return HangupException()
+
+
 class TraditionalGitClient(GitClient):
     """Traditional Git client."""
 
@@ -654,9 +733,12 @@ class TraditionalGitClient(GitClient):
         :return: new_refs dictionary containing the changes that were made
             {refname: new_ref}, including deleted refs.
         """
-        proto, unused_can_read = self._connect(b'receive-pack', path)
+        proto, unused_can_read, stderr = self._connect(b'receive-pack', path)
         with proto:
-            old_refs, server_capabilities = read_pkt_refs(proto)
+            try:
+                old_refs, server_capabilities = read_pkt_refs(proto)
+            except HangupException:
+                raise remote_error_from_stderr(stderr)
             negotiated_capabilities = \
                 self._negotiate_receive_pack_capabilities(server_capabilities)
             if CAPABILITY_REPORT_STATUS in negotiated_capabilities:
@@ -713,7 +795,7 @@ class TraditionalGitClient(GitClient):
             return new_refs
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-                   progress=None):
+                   progress=None, depth=None):
         """Retrieve a pack from a git smart server.
 
         :param path: Remote path to fetch from
@@ -723,11 +805,15 @@ class TraditionalGitClient(GitClient):
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
+        :param depth: Shallow fetch depth
         :return: FetchPackResult object
         """
-        proto, can_read = self._connect(b'upload-pack', path)
+        proto, can_read, stderr = self._connect(b'upload-pack', path)
         with proto:
-            refs, server_capabilities = read_pkt_refs(proto)
+            try:
+                refs, server_capabilities = read_pkt_refs(proto)
+            except HangupException:
+                raise remote_error_from_stderr(stderr)
             negotiated_capabilities, symrefs, agent = (
                     self._negotiate_upload_pack_capabilities(
                             server_capabilities))
@@ -746,25 +832,31 @@ class TraditionalGitClient(GitClient):
             if not wants:
                 proto.write_pkt_line(None)
                 return FetchPackResult(refs, symrefs, agent)
-            self._handle_upload_pack_head(
-                proto, negotiated_capabilities, graph_walker, wants, can_read)
+            check_wants(wants, refs)
+            (new_shallow, new_unshallow) = self._handle_upload_pack_head(
+                proto, negotiated_capabilities, graph_walker, wants, can_read,
+                depth=depth)
             self._handle_upload_pack_tail(
                 proto, negotiated_capabilities, graph_walker, pack_data,
                 progress)
-            return FetchPackResult(refs, symrefs, agent)
+            return FetchPackResult(
+                    refs, symrefs, agent, new_shallow, new_unshallow)
 
     def get_refs(self, path):
         """Retrieve the current refs from a git smart server."""
         # stock `git ls-remote` uses upload-pack
-        proto, _ = self._connect(b'upload-pack', path)
+        proto, _, stderr = self._connect(b'upload-pack', path)
         with proto:
-            refs, _ = read_pkt_refs(proto)
+            try:
+                refs, _ = read_pkt_refs(proto)
+            except HangupException:
+                raise remote_error_from_stderr(stderr)
             proto.write_pkt_line(None)
             return refs
 
     def archive(self, path, committish, write_data, progress=None,
                 write_error=None, format=None, subdirs=None, prefix=None):
-        proto, can_read = self._connect(b'upload-archive', path)
+        proto, can_read, stderr = self._connect(b'upload-archive', path)
         with proto:
             if format is not None:
                 proto.write_pkt_line(b"argument --format=" + format)
@@ -775,13 +867,17 @@ class TraditionalGitClient(GitClient):
             if prefix is not None:
                 proto.write_pkt_line(b"argument --prefix=" + prefix)
             proto.write_pkt_line(None)
-            pkt = proto.read_pkt_line()
+            try:
+                pkt = proto.read_pkt_line()
+            except HangupException:
+                raise remote_error_from_stderr(stderr)
             if pkt == b"NACK\n":
                 return
             elif pkt == b"ACK\n":
                 pass
             elif pkt.startswith(b"ERR "):
-                raise GitProtocolError(pkt[4:].rstrip(b"\n"))
+                raise GitProtocolError(
+                        pkt[4:].rstrip(b"\n").decode('utf-8', 'replace'))
             else:
                 raise AssertionError("invalid response %r" % pkt)
             ret = proto.read_pkt_line()
@@ -828,7 +924,8 @@ class TCPGitClient(TraditionalGitClient):
             try:
                 s.connect(sockaddr)
                 break
-            except socket.error as err:
+            except socket.error as e:
+                err = e
                 if s is not None:
                     s.close()
                 s = None
@@ -851,7 +948,7 @@ class TCPGitClient(TraditionalGitClient):
         # TODO(jelmer): Alternative to ascii?
         proto.send_cmd(
             b'git-' + cmd, path, b'host=' + self._host.encode('ascii'))
-        return proto, lambda: _fileno_can_read(s)
+        return proto, lambda: _fileno_can_read(s), None
 
 
 class SubprocessWrapper(object):
@@ -865,6 +962,10 @@ class SubprocessWrapper(object):
             self.read = BufferedReader(proc.stdout).read
         self.write = proc.stdin.write
 
+    @property
+    def stderr(self):
+        return self.proc.stderr
+
     def can_read(self):
         if sys.platform == 'win32':
             from msvcrt import get_osfhandle
@@ -899,14 +1000,6 @@ def find_git_command():
 class SubprocessGitClient(TraditionalGitClient):
     """Git client that talks to a server using a subprocess."""
 
-    def __init__(self, **kwargs):
-        self._connection = None
-        self._stderr = None
-        self._stderr = kwargs.get('stderr')
-        if 'stderr' in kwargs:
-            del kwargs['stderr']
-        super(SubprocessGitClient, self).__init__(**kwargs)
-
     @classmethod
     def from_parsedurl(cls, parsedurl, **kwargs):
         return cls(**kwargs)
@@ -921,12 +1014,13 @@ class SubprocessGitClient(TraditionalGitClient):
         if self.git_command is None:
             git_command = find_git_command()
         argv = git_command + [service.decode('ascii'), path]
-        p = SubprocessWrapper(
-            subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
+        p = subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE,
-                             stderr=self._stderr))
-        return Protocol(p.read, p.write, p.close,
-                        report_activity=self._report_activity), p.can_read
+                             stderr=subprocess.PIPE)
+        pw = SubprocessWrapper(p)
+        return (Protocol(pw.read, pw.write, pw.close,
+                         report_activity=self._report_activity),
+                pw.can_read, p.stderr)
 
 
 class LocalGitClient(GitClient):
@@ -1010,7 +1104,8 @@ class LocalGitClient(GitClient):
 
         return new_refs
 
-    def fetch(self, path, target, determine_wants=None, progress=None):
+    def fetch(self, path, target, determine_wants=None, progress=None,
+              depth=None):
         """Fetch into a target repository.
 
         :param path: Path to fetch from (as bytestring)
@@ -1019,16 +1114,17 @@ class LocalGitClient(GitClient):
             to fetch. Receives dictionary of name->sha, should return
             list of shas to fetch. Defaults to all shas.
         :param progress: Optional progress function
+        :param depth: Shallow fetch depth
         :return: FetchPackResult object
         """
         with self._open_repo(path) as r:
             refs = r.fetch(target, determine_wants=determine_wants,
-                           progress=progress)
+                           progress=progress, depth=depth)
             return FetchPackResult(refs, r.refs.get_symrefs(),
                                    agent_string())
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-                   progress=None):
+                   progress=None, depth=None):
         """Retrieve a pack from a git smart server.
 
         :param path: Remote path to fetch from
@@ -1038,11 +1134,12 @@ class LocalGitClient(GitClient):
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
+        :param depth: Shallow fetch depth
         :return: FetchPackResult object
         """
         with self._open_repo(path) as r:
             objects_iter = r.fetch_objects(
-                determine_wants, graph_walker, progress)
+                determine_wants, graph_walker, progress=progress, depth=depth)
             symrefs = r.refs.get_symrefs()
             agent = agent_string()
 
@@ -1128,7 +1225,8 @@ class SubprocessSSHVendor(SSHVendor):
 
         proc = subprocess.Popen(args + [command], bufsize=0,
                                 stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE)
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
         return SubprocessWrapper(proc)
 
 
@@ -1164,7 +1262,8 @@ class PLinkSSHVendor(SSHVendor):
 
         proc = subprocess.Popen(args + [command], bufsize=0,
                                 stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE)
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
         return SubprocessWrapper(proc)
 
 
@@ -1236,7 +1335,7 @@ class SSHGitClient(TraditionalGitClient):
             **kwargs)
         return (Protocol(con.read, con.write, con.close,
                          report_activity=self._report_activity),
-                con.can_read)
+                con.can_read, getattr(con, 'stderr', None))
 
 
 def default_user_agent_string():
@@ -1329,7 +1428,6 @@ class HttpGitClient(GitClient):
         self._username = username
         self._password = password
         self.dumb = dumb
-        self.headers = {}
 
         if pool_manager is None:
             self.pool_manager = default_urllib3_manager(config)
@@ -1353,14 +1451,17 @@ class HttpGitClient(GitClient):
     def from_parsedurl(cls, parsedurl, **kwargs):
         password = parsedurl.password
         if password is not None:
-            password = urlunquote(password)
+            kwargs['password'] = urlunquote(password)
         username = parsedurl.username
         if username is not None:
-            username = urlunquote(username)
-        # TODO(jelmer): This also strips the username
-        parsedurl = parsedurl._replace(netloc=parsedurl.hostname)
-        return cls(urlparse.urlunparse(parsedurl),
-                   password=password, username=username, **kwargs)
+            kwargs['username'] = urlunquote(username)
+        netloc = parsedurl.hostname
+        if parsedurl.port:
+            netloc = "%s:%s" % (netloc, parsedurl.port)
+        if parsedurl.username:
+            netloc = "%s@%s" % (parsedurl.username, netloc)
+        parsedurl = parsedurl._replace(netloc=netloc)
+        return cls(urlparse.urlunparse(parsedurl), **kwargs)
 
     def __repr__(self):
         return "%s(%r, dumb=%r)" % (
@@ -1413,10 +1514,7 @@ class HttpGitClient(GitClient):
         # `BytesIO`, if we can guarantee that the entire response is consumed
         # before issuing the next to still allow for connection reuse from the
         # pool.
-        if resp.getheader("Content-Encoding") == "gzip":
-            read = gzip.GzipFile(fileobj=BytesIO(resp.data)).read
-        else:
-            read = BytesIO(resp.data).read
+        read = BytesIO(resp.data).read
 
         resp.content_type = resp.getheader("Content-Type")
         resp.redirect_location = resp.get_redirect_location()
@@ -1530,13 +1628,14 @@ class HttpGitClient(GitClient):
             resp.close()
 
     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
-                   progress=None):
+                   progress=None, depth=None):
         """Retrieve a pack from a git smart server.
 
         :param determine_wants: Callback that returns list of commits to fetch
         :param graph_walker: Object with next() and ack().
         :param pack_data: Callback called for each bit of data in the pack
         :param progress: Callback for progress reports (strings)
+        :param depth: Depth for request
         :return: FetchPackResult object
         """
         url = self._get_url(path)
@@ -1552,19 +1651,24 @@ class HttpGitClient(GitClient):
             return FetchPackResult(refs, symrefs, agent)
         if self.dumb:
             raise NotImplementedError(self.send_pack)
+        check_wants(wants, refs)
         req_data = BytesIO()
         req_proto = Protocol(None, req_data.write)
-        self._handle_upload_pack_head(
+        (new_shallow, new_unshallow) = self._handle_upload_pack_head(
                 req_proto, negotiated_capabilities, graph_walker, wants,
-                lambda: False)
+                can_read=None, depth=depth)
         resp, read = self._smart_request(
             "git-upload-pack", url, data=req_data.getvalue())
         try:
             resp_proto = Protocol(read, None)
+            if new_shallow is None and new_unshallow is None:
+                (new_shallow, new_unshallow) = _read_shallow_updates(
+                        resp_proto)
             self._handle_upload_pack_tail(
                 resp_proto, negotiated_capabilities, graph_walker, pack_data,
                 progress)
-            return FetchPackResult(refs, symrefs, agent)
+            return FetchPackResult(
+                    refs, symrefs, agent, new_shallow, new_unshallow)
         finally:
             resp.close()
 

dulwich/config.py

@@ -31,11 +31,18 @@ import os
 import sys
 
 from collections import (
-    Iterable,
     OrderedDict,
-    MutableMapping,
     )
-
+try:
+    from collections.abc import (
+        Iterable,
+        MutableMapping,
+        )
+except ImportError:  # python < 3.7
+    from collections import (
+        Iterable,
+        MutableMapping,
+        )
 
 from dulwich.file import GitFile
 

dulwich/contrib/README.md

@@ -0,0 +1,3 @@
+This directory contains code that some may find useful. Code here is not an official
+part of Dulwich, and may no longer work. Unlike the rest of Dulwich, it is not regularly
+tested.

dulwich/contrib/paramiko_vendor.py

@@ -32,55 +32,20 @@ This implementation is experimental and does not have any tests.
 
 import paramiko
 import paramiko.client
-import threading
 
 
 class _ParamikoWrapper(object):
-    STDERR_READ_N = 2048  # 2k
 
-    def __init__(self, client, channel, progress_stderr=None):
+    def __init__(self, client, channel):
         self.client = client
         self.channel = channel
-        self.progress_stderr = progress_stderr
-        self.should_monitor = bool(progress_stderr) or True
-        self.monitor_thread = None
-        self.stderr = b''
 
         # Channel must block
         self.channel.setblocking(True)
 
-        # Start
-        if self.should_monitor:
-            self.monitor_thread = threading.Thread(
-                target=self.monitor_stderr)
-            self.monitor_thread.start()
-
-    def monitor_stderr(self):
-        while self.should_monitor:
-            # Block and read
-            data = self.read_stderr(self.STDERR_READ_N)
-
-            # Socket closed
-            if not data:
-                self.should_monitor = False
-                break
-
-            # Emit data
-            if self.progress_stderr:
-                self.progress_stderr(data)
-
-            # Append to buffer
-            self.stderr += data
-
-    def stop_monitoring(self):
-        # Stop StdErr thread
-        if self.should_monitor:
-            self.should_monitor = False
-            self.monitor_thread.join()
-
-            # Get left over data
-            data = self.channel.in_stderr_buffer.empty()
-            self.stderr += data
+    @property
+    def stderr(self):
+        return self.channel.makefile_stderr()
 
     def can_read(self):
         return self.channel.recv_ready()
@@ -88,9 +53,6 @@ class _ParamikoWrapper(object):
     def write(self, data):
         return self.channel.sendall(data)
 
-    def read_stderr(self, n):
-        return self.channel.recv_stderr(n)
-
     def read(self, n=None):
         data = self.channel.recv(n)
         data_len = len(data)
@@ -107,7 +69,6 @@ class _ParamikoWrapper(object):
 
     def close(self):
         self.channel.close()
-        self.stop_monitoring()
 
 
 class ParamikoSSHVendor(object):
@@ -118,7 +79,6 @@ class ParamikoSSHVendor(object):
 
     def run_command(self, host, command,
                     username=None, port=None,
-                    progress_stderr=None,
                     password=None, pkey=None,
                     key_filename=None, **kwargs):
 
@@ -148,5 +108,4 @@ class ParamikoSSHVendor(object):
         # Run commands
         channel.exec_command(command)
 
-        return _ParamikoWrapper(
-            client, channel, progress_stderr=progress_stderr)
+        return _ParamikoWrapper(client, channel)

dulwich/ignore.py

@@ -96,7 +96,7 @@ def translate(pat):
     if not pat.endswith(b'/'):
         res += b'/?'
 
-    return res + b'\Z'
+    return res + b'\\Z'
 
 
 def read_ignore_patterns(f):

dulwich/line_ending.py

@@ -0,0 +1,181 @@
+# line_ending.py -- Line ending conversion functions
+# Copyright (C) 2018-2018 Boris Feld <boris.feld@comet.ml>
+#
+# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+# General Public License as public by the Free Software Foundation; version 2.0
+# or (at your option) any later version. You can redistribute it and/or
+# modify it under the terms of either of these two licenses.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# You should have received a copy of the licenses; if not, see
+# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+# License, Version 2.0.
+#
+""" All line-ending related functions, from conversions to config processing
+
+Line-ending normalization is a complex beast. Here is some notes and details
+about how it seems to work.
+
+The normalization is a two-fold process that happens at two moments:
+
+- When reading a file from the index and to the working directory. For example
+  when doing a `git clone` or `git checkout` call. We call this process the
+  read filter in this module.
+- When writing a file to the index from the working directory. For example
+  when doing a `git add` call. We call this process the write filter in this
+  module.
+
+One thing to know is that Git does line-ending normalization only on text
+files. How does Git know that a file is text? We can either mark a file as a
+text file, a binary file or ask Git to automatically decides. Git has an
+heuristic to detect if a file is a text file or a binary file. It seems based
+on the percentage of non-printable characters in files.
+
+The code for this heuristic is here:
+https://git.kernel.org/pub/scm/git/git.git/tree/convert.c#n46
+
+Dulwich have an implementation with a slightly different heuristic, the
+`is_binary` function in `dulwich.patch`.
+
+The binary detection heuristic implementation is close to the one in JGit:
+https://github.com/eclipse/jgit/blob/f6873ffe522bbc3536969a3a3546bf9a819b92bf/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java#L300
+
+There is multiple variables that impact the normalization.
+
+First, a repository can contains a `.gitattributes` file (or more than one...)
+that can further customize the operation on some file patterns, for example:
+
+    *.txt text
+
+Force all `.txt` files to be treated as text files and to have their lines
+endings normalized.
+
+    *.jpg -text
+
+Force all `.jpg` files to be treated as binary files and to not have their
+lines endings converted.
+
+    *.vcproj text eol=crlf
+
+Force all `.vcproj` files to be treated as text files and to have their lines
+endings converted into `CRLF` in working directory no matter the native EOL of
+the platform.
+
+    *.sh text eol=lf
+
+Force all `.sh` files to be treated as text files and to have their lines
+endings converted into `LF` in working directory no matter the native EOL of
+the platform.
+
+If the `eol` attribute is not defined, Git uses the `core.eol` configuration
+value described later.
+
+    * text=auto
+
+Force all files to be scanned by the text file heuristic detection and to have
+their line endings normalized in case they are detected as text files.
+
+Git also have a obsolete attribute named `crlf` that can be translated to the
+corresponding text attribute value.
+
+Then there are some configuration option (that can be defined at the
+repository or user level):
+
+- core.autocrlf
+- core.eol
+
+`core.autocrlf` is taken into account for all files that doesn't have a `text`
+attribute defined in `.gitattributes`; it takes three possible values:
+
+    - `true`: This forces all files on the working directory to have CRLF
+      line-endings in the working directory and convert line-endings to LF
+      when writing to the index. When autocrlf is set to true, eol value is
+      ignored.
+    - `input`: Quite similar to the `true` value but only force the write
+      filter, ie line-ending of new files added to the index will get their
+      line-endings converted to LF.
+    - `false` (default): No normalization is done.
+
+`core.eol` is the top-level configuration to define the line-ending to use
+when applying the read_filer. It takes three possible values:
+
+    - `lf`: When normalization is done, force line-endings to be `LF` in the
+      working directory.
+    - `crlf`: When normalization is done, force line-endings to be `CRLF` in
+      the working directory.
+    - `native` (default): When normalization is done, force line-endings to be
+      the platform's native line ending.
+
+One thing to remember is when line-ending normalization is done on a file, Git
+always normalize line-ending to `LF` when writing to the index.
+
+There are sources that seems to indicate that Git won't do line-ending
+normalization when a file contains mixed line-endings. I think this logic
+might be in text / binary detection heuristic but couldn't find it yet.
+
+Sources:
+- https://git-scm.com/docs/git-config#git-config-coreeol
+- https://git-scm.com/docs/git-config#git-config-coreautocrlf
+- https://git-scm.com/docs/gitattributes#_checking_out_and_checking_in
+- https://adaptivepatchwork.com/2012/03/01/mind-the-end-of-your-line/
+"""
+
+CRLF = b"\r\n"
+LF = b"\n"
+
+
+def convert_crlf_to_lf(text_hunk):
+    """Convert CRLF in text hunk into LF
+
+    :param text_hunk: A bytes string representing a text hunk
+    :return: The text hunk with the same type, with CRLF replaced into LF
+    """
+    return text_hunk.replace(CRLF, LF)
+
+
+def convert_lf_to_crlf(text_hunk):
+    """Convert LF in text hunk into CRLF
+
+    :param text_hunk: A bytes string representing a text hunk
+    :return: The text hunk with the same type, with LF replaced into CRLF
+    """
+    # TODO find a more efficient way of doing it
+    intermediary = text_hunk.replace(CRLF, LF)
+    return intermediary.replace(LF, CRLF)
+
+
+def get_checkout_filter_autocrlf(core_autocrlf):
+    """ Returns the correct checkout filter base on autocrlf value
+
+    :param core_autocrlf: The bytes configuration value of core.autocrlf.
+        Valid values are: b'true', b'false' or b'input'.
+    :return: Either None if no filter has to be applied or a function
+        accepting a single argument, a binary text hunk
+    """
+
+    if core_autocrlf == b"true":
+        return convert_lf_to_crlf
+
+    return None
+
+
+def get_checkin_filter_autocrlf(core_autocrlf):
+    """ Returns the correct checkin filter base on autocrlf value
+
+    :param core_autocrlf: The bytes configuration value of core.autocrlf.
+        Valid values are: b'true', b'false' or b'input'.
+    :return: Either None if no filter has to be applied or a function
+        accepting a single argument, a binary text hunk
+    """
+
+    if core_autocrlf == b"true" or core_autocrlf == b"input":
+        return convert_crlf_to_lf
+
+    # Checking filter should never be `convert_lf_to_crlf`
+    return None

dulwich/object_store.py

@@ -65,6 +65,7 @@ from dulwich.pack import (
     PackIndexer,
     PackStreamCopier,
     )
+from dulwich.refs import ANNOTATED_TAG_SUFFIX
 
 INFODIR = 'info'
 PACKDIR = 'pack'
@@ -75,7 +76,8 @@ class BaseObjectStore(object):
 
     def determine_wants_all(self, refs):
         return [sha for (ref, sha) in refs.items()
-                if sha not in self and not ref.endswith(b"^{}") and
+                if sha not in self and
+                not ref.endswith(ANNOTATED_TAG_SUFFIX) and
                 not sha == ZERO_SHA]
 
     def iter_shas(self, shas):
@@ -192,7 +194,8 @@ class BaseObjectStore(object):
 
     def find_missing_objects(self, haves, wants, progress=None,
                              get_tagged=None,
-                             get_parents=lambda commit: commit.parents):
+                             get_parents=lambda commit: commit.parents,
+                             depth=None):
         """Find the missing objects required for a set of revisions.
 
         :param haves: Iterable over SHAs already in common.
@@ -1163,7 +1166,7 @@ class ObjectStoreGraphWalker(object):
     :ivar get_parents: Function to retrieve parents in the local repo
     """
 
-    def __init__(self, local_heads, get_parents):
+    def __init__(self, local_heads, get_parents, shallow=None):
         """Create a new instance.
 
         :param local_heads: Heads to start search with
@@ -1172,6 +1175,9 @@ class ObjectStoreGraphWalker(object):
         self.heads = set(local_heads)
         self.get_parents = get_parents
         self.parents = {}
+        if shallow is None:
+            shallow = set()
+        self.shallow = shallow
 
     def ack(self, sha):
         """Ack that a revision and its ancestors are present in the source."""
@@ -1315,3 +1321,14 @@ class OverlayObjectStore(BaseObjectStore):
                 return True
         else:
             return False
+
+
+def read_packs_file(f):
+    """Yield the packs listed in a packs file."""
+    for line in f.read().splitlines():
+        if not line:
+            continue
+        (kind, name) = line.split(b" ", 1)
+        if kind != b"P":
+            continue
+        yield name.decode(sys.getfilesystemencoding())

dulwich/objects.py

@@ -67,6 +67,8 @@ S_IFGITLINK = 0o160000
 
 MAX_TIME = 9223372036854775807  # (2**63) - 1 - signed long int max
 
+BEGIN_PGP_SIGNATURE = b"-----BEGIN PGP SIGNATURE-----"
+
 
 def S_ISGITLINK(m):
     """Check if a mode indicates a submodule.
@@ -520,27 +522,31 @@ class ShaFile(object):
         return "<%s %s>" % (self.__class__.__name__, self.id)
 
     def __ne__(self, other):
+        """Check whether this object does not match the other."""
         return not isinstance(other, ShaFile) or self.id != other.id
 
     def __eq__(self, other):
         """Return True if the SHAs of the two objects match.
-
-        It doesn't make sense to talk about an order on ShaFiles, so we don't
-        override the rich comparison methods (__le__, etc.).
         """
         return isinstance(other, ShaFile) and self.id == other.id
 
     def __lt__(self, other):
+        """Return whether SHA of this object is less than the other.
+        """
         if not isinstance(other, ShaFile):
             raise TypeError
         return self.id < other.id
 
     def __le__(self, other):
+        """Check whether SHA of this object is less than or equal to the other.
+        """
         if not isinstance(other, ShaFile):
             raise TypeError
         return self.id <= other.id
 
     def __cmp__(self, other):
+        """Compare the SHA of this object with that of the other object.
+        """
         if not isinstance(other, ShaFile):
             raise TypeError
         return cmp(self.id, other.id)  # noqa: F821
@@ -687,7 +693,7 @@ class Tag(ShaFile):
 
     __slots__ = ('_tag_timezone_neg_utc', '_name', '_object_sha',
                  '_object_class', '_tag_time', '_tag_timezone',
-                 '_tagger', '_message')
+                 '_tagger', '_message', '_signature')
 
     def __init__(self):
         super(Tag, self).__init__()
@@ -695,6 +701,7 @@ class Tag(ShaFile):
         self._tag_time = None
         self._tag_timezone = None
         self._tag_timezone_neg_utc = False
+        self._signature = None
 
     @classmethod
     def from_path(cls, filename):
@@ -753,6 +760,8 @@ class Tag(ShaFile):
         if self._message is not None:
             chunks.append(b'\n')  # To close headers
             chunks.append(self._message)
+        if self._signature is not None:
+            chunks.append(self._signature)
         return chunks
 
     def _deserialize(self, chunks):
@@ -777,7 +786,18 @@ class Tag(ShaFile):
                  (self._tag_timezone,
                   self._tag_timezone_neg_utc)) = parse_time_entry(value)
             elif field is None:
-                self._message = value
+                if value is None:
+                    self._message = None
+                    self._signature = None
+                else:
+                    try:
+                        sig_idx = value.index(BEGIN_PGP_SIGNATURE)
+                    except ValueError:
+                        self._message = value
+                        self._signature = None
+                    else:
+                        self._message = value[:sig_idx]
+                        self._signature = value[sig_idx:]
             else:
                 raise ObjectFormatException("Unknown field %s" % field)
 
@@ -806,7 +826,10 @@ class Tag(ShaFile):
             "tag_timezone",
             "The timezone that tag_time is in.")
     message = serializable_property(
-            "message", "The message attached to this tag")
+            "message", "the message attached to this tag")
+
+    signature = serializable_property(
+            "signature", "Optional detached GPG signature")
 
 
 class TreeEntry(namedtuple('TreeEntry', ['path', 'mode', 'sha'])):
@@ -1112,7 +1135,7 @@ def parse_time_entry(value):
     :return: Tuple of (author, time, (timezone, timezone_neg_utc))
     """
     try:
-        sep = value.index(b'> ')
+        sep = value.rindex(b'> ')
     except ValueError:
         return (value, None, (None, False))
     try:

dulwich/objectspec.py

@@ -87,12 +87,12 @@ def parse_reftuple(lh_container, rh_container, refspec):
     :return: A tuple with left and right ref
     :raise KeyError: If one of the refs can not be found
     """
+    refspec = to_bytes(refspec)
     if refspec.startswith(b"+"):
         force = True
         refspec = refspec[1:]
     else:
         force = False
-    refspec = to_bytes(refspec)
     if b":" in refspec:
         (lh, rh) = refspec.split(b":")
     else:

dulwich/porcelain.py

@@ -61,7 +61,7 @@ from contextlib import (
     closing,
     contextmanager,
 )
-from io import BytesIO
+from io import BytesIO, RawIOBase
 import datetime
 import os
 import posixpath
@@ -139,8 +139,27 @@ from dulwich.server import (
 GitStatus = namedtuple('GitStatus', 'staged unstaged untracked')
 
 
-default_bytes_out_stream = getattr(sys.stdout, 'buffer', sys.stdout)
-default_bytes_err_stream = getattr(sys.stderr, 'buffer', sys.stderr)
+class NoneStream(RawIOBase):
+    """Fallback if stdout or stderr are unavailable, does nothing."""
+    def read(self, size=-1):
+        return None
+
+    def readall(self):
+        return None
+
+    def readinto(self, b):
+        return None
+
+    def write(self, b):
+        return None
+
+
+default_bytes_out_stream = getattr(
+        sys.stdout, 'buffer', sys.stdout
+    ) or NoneStream()
+default_bytes_err_stream = getattr(
+        sys.stderr, 'buffer', sys.stderr
+    ) or NoneStream()
 
 
 DEFAULT_ENCODING = 'utf-8'
@@ -188,6 +207,8 @@ def path_to_tree_path(repopath, path):
     treepath = os.path.relpath(path, repopath)
     if treepath.startswith(b'..'):
         raise ValueError('Path not in repo')
+    if os.path.sep != '/':
+        treepath = treepath.replace(os.path.sep.encode('ascii'), b'/')
     return treepath
 
 
@@ -288,7 +309,7 @@ def init(path=".", bare=False):
 
 def clone(source, target=None, bare=False, checkout=None,
           errstream=default_bytes_err_stream, outstream=None,
-          origin=b"origin", **kwargs):
+          origin=b"origin", depth=None, **kwargs):
     """Clone a local or remote git repository.
 
     :param source: Path or URL for source repository
@@ -298,6 +319,7 @@ def clone(source, target=None, bare=False, checkout=None,
     :param errstream: Optional stream to write progress to
     :param outstream: Optional stream to write progress to (deprecated)
     :param origin: Name of remote from the repository used to clone
+    :param depth: Depth to fetch at
     :return: The new repository
     """
     # TODO(jelmer): This code overlaps quite a bit with Repo.clone
@@ -328,7 +350,7 @@ def clone(source, target=None, bare=False, checkout=None,
     try:
         fetch_result = fetch(
             r, source, origin, errstream=errstream, message=reflog_message,
-            **kwargs)
+            depth=depth, **kwargs)
         target_config = r.get_config()
         if not isinstance(source, bytes):
             source = source.encode(DEFAULT_ENCODING)
@@ -659,7 +681,8 @@ def tag(*args, **kwargs):
 
 def tag_create(
         repo, tag, author=None, message=None, annotated=False,
-        objectish="HEAD", tag_time=None, tag_timezone=None):
+        objectish="HEAD", tag_time=None, tag_timezone=None,
+        sign=False):
     """Creates a tag in git via dulwich calls:
 
     :param repo: Path to repository
@@ -670,6 +693,7 @@ def tag_create(
     :param objectish: object the tag should point at, defaults to HEAD
     :param tag_time: Optional time for annotated tag
     :param tag_timezone: Optional timezone for annotated tag
+    :param sign: GPG Sign the tag
     """
 
     with open_repo_closing(repo) as r:
@@ -680,7 +704,7 @@ def tag_create(
             tag_obj = Tag()
             if author is None:
                 # TODO(jelmer): Don't use repo private method.
-                author = r._get_user_identity()
+                author = r._get_user_identity(r.get_config_stack())
             tag_obj.tagger = author
             tag_obj.message = message
             tag_obj.name = tag
@@ -694,6 +718,10 @@ def tag_create(
             elif isinstance(tag_timezone, str):
                 tag_timezone = parse_timezone(tag_timezone)
             tag_obj.tag_timezone = tag_timezone
+            if sign:
+                import gpg
+                with gpg.Context(armor=True) as c:
+                    tag_obj.signature, result = c.sign(tag_obj.as_raw_string())
             r.object_store.add_object(tag_obj)
             tag_id = tag_obj.id
         else:
@@ -1065,7 +1093,8 @@ def branch_list(repo):
 
 
 def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout,
-          errstream=default_bytes_err_stream, message=None, **kwargs):
+          errstream=default_bytes_err_stream, message=None, depth=None,
+          **kwargs):
     """Fetch objects from a remote server.
 
     :param repo: Path to the repository
@@ -1074,6 +1103,7 @@ def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout,
     :param outstream: Output stream (defaults to stdout)
     :param errstream: Error stream (defaults to stderr)
     :param message: Reflog message (defaults to b"fetch: from <remote_name>")
+    :param depth: Depth to fetch at
     :return: Dictionary with refs on the remote
     """
     if message is None:
@@ -1081,7 +1111,8 @@ def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout,
     with open_repo_closing(repo) as r:
         client, path = get_transport_and_path(
             remote_location, config=r.get_config_stack(), **kwargs)
-        fetch_result = client.fetch(path, r, progress=errstream.write)
+        fetch_result = client.fetch(path, r, progress=errstream.write,
+                                    depth=depth)
         stripped_refs = strip_peeled_refs(fetch_result.refs)
         branches = {
             n[len(b'refs/heads/'):]: v for (n, v) in stripped_refs.items()
@@ -1364,3 +1395,35 @@ def describe(repo):
 
         # Return plain commit if no parent tag can be found
         return 'g{}'.format(latest_commit.id.decode('ascii')[:7])
+
+
+def get_object_by_path(repo, path, committish=None):
+    """Get an object by path.
+
+    :param repo: A path to the repository
+    :param path: Path to look up
+    :param committish: Commit to look up path in
+    :return: A `ShaFile` object
+    """
+    if committish is None:
+        committish = "HEAD"
+    # Get the repository
+    with open_repo_closing(repo) as r:
+        commit = parse_commit(repo, committish)
+        base_tree = commit.tree
+        if not isinstance(path, bytes):
+            path = path.encode(commit.encoding or DEFAULT_ENCODING)
+        (mode, sha) = tree_lookup_path(
+            r.object_store.__getitem__,
+            base_tree, path)
+        return r[sha]
+
+
+def write_tree(repo):
+    """Write a tree object from the index.
+
+    :param repo: Repository for which to write tree
+    :return: tree id for the tree that was written
+    """
+    with open_repo_closing(repo) as r:
+        return r.open_index().commit(r.object_store)

dulwich/refs.py

@@ -617,7 +617,7 @@ class DiskRefsContainer(RefsContainer):
                     # Read only the first 40 bytes
                     return header + f.read(40 - len(SYMREF))
         except IOError as e:
-            if e.errno in (errno.ENOENT, errno.EISDIR):
+            if e.errno in (errno.ENOENT, errno.EISDIR, errno.ENOTDIR):
                 return None
             raise
 
@@ -687,6 +687,16 @@ class DiskRefsContainer(RefsContainer):
         except (KeyError, IndexError):
             realname = name
         filename = self.refpath(realname)
+
+        # make sure none of the ancestor folders is in packed refs
+        probe_ref = os.path.dirname(realname)
+        packed_refs = self.get_packed_refs()
+        while probe_ref:
+            if packed_refs.get(probe_ref, None) is not None:
+                raise OSError(errno.ENOTDIR,
+                              'Not a directory: {}'.format(filename))
+            probe_ref = os.path.dirname(probe_ref)
+
         ensure_dir_exists(os.path.dirname(filename))
         with GitFile(filename, 'wb') as f:
             if old_ref is not None:
@@ -800,7 +810,7 @@ class DiskRefsContainer(RefsContainer):
             parent_filename = self.refpath(parent)
             try:
                 os.rmdir(parent_filename)
-            except OSError as e:
+            except OSError:
                 # this can be caused by the parent directory being
                 # removed by another process, being not empty, etc.
                 # in any case, this is non fatal because we already

dulwich/repo.py

@@ -73,6 +73,7 @@ from dulwich.hooks import (
     )
 
 from dulwich.refs import (  # noqa: F401
+    ANNOTATED_TAG_SUFFIX,
     check_ref_format,
     RefsContainer,
     DictRefsContainer,
@@ -251,6 +252,10 @@ class BaseRepo(object):
         """
         raise NotImplementedError(self._put_named_file)
 
+    def _del_named_file(self, path):
+        """Delete a file in the contrl directory with the given name."""
+        raise NotImplementedError(self._del_named_file)
+
     def open_index(self):
         """Open the index for this repository.
 
@@ -259,24 +264,26 @@ class BaseRepo(object):
         """
         raise NotImplementedError(self.open_index)
 
-    def fetch(self, target, determine_wants=None, progress=None):
+    def fetch(self, target, determine_wants=None, progress=None, depth=None):
         """Fetch objects into another repository.
 
         :param target: The target repository
         :param determine_wants: Optional function to determine what refs to
             fetch.
         :param progress: Optional progress function
+        :param depth: Optional shallow fetch depth
         :return: The local refs
         """
         if determine_wants is None:
             determine_wants = target.object_store.determine_wants_all
         count, pack_data = self.fetch_pack_data(
-                determine_wants, target.get_graph_walker(), progress)
+                determine_wants, target.get_graph_walker(), progress=progress,
+                depth=depth)
         target.object_store.add_pack_data(count, pack_data, progress)
         return self.get_refs()
 
     def fetch_pack_data(self, determine_wants, graph_walker, progress,
-                        get_tagged=None):
+                        get_tagged=None, depth=None):
         """Fetch the pack data required for a set of revisions.
 
         :param determine_wants: Function that takes a dictionary with heads
@@ -288,15 +295,16 @@ class BaseRepo(object):
             updated progress strings.
         :param get_tagged: Function that returns a dict of pointed-to sha ->
             tag sha for including tags.
+        :param depth: Shallow fetch depth
         :return: count and iterator over pack data
         """
         # TODO(jelmer): Fetch pack data directly, don't create objects first.
         objects = self.fetch_objects(determine_wants, graph_walker, progress,
-                                     get_tagged)
+                                     get_tagged, depth=depth)
         return pack_objects_to_data(objects)
 
     def fetch_objects(self, determine_wants, graph_walker, progress,
-                      get_tagged=None):
+                      get_tagged=None, depth=None):
         """Fetch the missing objects required for a set of revisions.
 
         :param determine_wants: Function that takes a dictionary with heads
@@ -308,9 +316,28 @@ class BaseRepo(object):
             updated progress strings.
         :param get_tagged: Function that returns a dict of pointed-to sha ->
             tag sha for including tags.
+        :param depth: Shallow fetch depth
         :return: iterator over objects, with __len__ implemented
         """
-        wants = determine_wants(self.get_refs())
+        if depth not in (None, 0):
+            raise NotImplementedError("depth not supported yet")
+
+        refs = {}
+        for ref, sha in self.get_refs().items():
+            try:
+                obj = self.object_store[sha]
+            except KeyError:
+                warnings.warn(
+                    'ref %s points at non-present sha %s' % (
+                        ref.decode('utf-8', 'replace'), sha.decode('ascii')),
+                    UserWarning)
+                continue
+            else:
+                if isinstance(obj, Tag):
+                    refs[ref + ANNOTATED_TAG_SUFFIX] = obj.object[1]
+                refs[ref] = sha
+
+        wants = determine_wants(refs)
         if not isinstance(wants, list):
             raise TypeError("determine_wants() did not return a list")
 
@@ -361,7 +388,8 @@ class BaseRepo(object):
         """
         if heads is None:
             heads = self.refs.as_dict(b'refs/heads').values()
-        return ObjectStoreGraphWalker(heads, self.get_parents)
+        return ObjectStoreGraphWalker(
+            heads, self.get_parents, shallow=self.get_shallow())
 
     def get_refs(self):
         """Get dictionary with all refs.
@@ -458,7 +486,26 @@ class BaseRepo(object):
 
         :return: Set of shallow commits.
         """
-        return set()
+        f = self.get_named_file('shallow')
+        if f is None:
+            return set()
+        with f:
+            return set(l.strip() for l in f)
+
+    def update_shallow(self, new_shallow, new_unshallow):
+        """Update the list of shallow objects.
+
+        :param new_shallow: Newly shallow objects
+        :param new_unshallow: Newly no longer shallow objects
+        """
+        shallow = self.get_shallow()
+        if new_shallow:
+            shallow.update(new_shallow)
+        if new_unshallow:
+            shallow.difference_update(new_unshallow)
+        self._put_named_file(
+            'shallow',
+            b''.join([sha + b'\n' for sha in shallow]))
 
     def get_peeled(self, ref):
         """Get the peeled value of a ref.
@@ -565,12 +612,11 @@ class BaseRepo(object):
         else:
             raise ValueError(name)
 
-    def _get_user_identity(self):
+    def _get_user_identity(self, config):
         """Determine the identity to use for new commits.
         """
         user = os.environ.get("GIT_COMMITTER_NAME")
         email = os.environ.get("GIT_COMMITTER_EMAIL")
-        config = self.get_config_stack()
         if user is None:
             try:
                 user = config.get(("user", ), "name")
@@ -612,6 +658,13 @@ class BaseRepo(object):
         for sha in to_remove:
             del self._graftpoints[sha]
 
+    def _read_heads(self, name):
+        f = self.get_named_file(name)
+        if f is None:
+            return []
+        with f:
+            return [l.strip() for l in f.readlines() if l.strip()]
+
     def do_commit(self, message=None, committer=None,
                   author=None, commit_timestamp=None,
                   commit_timezone=None, author_timestamp=None,
@@ -652,11 +705,11 @@ class BaseRepo(object):
         except KeyError:  # no hook defined, silent fallthrough
             pass
 
+        config = self.get_config_stack()
         if merge_heads is None:
-            # FIXME: Read merge heads from .git/MERGE_HEADS
-            merge_heads = []
+            merge_heads = self._read_heads('MERGE_HEADS')
         if committer is None:
-            committer = self._get_user_identity()
+            committer = self._get_user_identity(config)
         check_user_identity(committer)
         c.committer = committer
         if commit_timestamp is None:
@@ -680,6 +733,11 @@ class BaseRepo(object):
         if author_timezone is None:
             author_timezone = commit_timezone
         c.author_timezone = author_timezone
+        if encoding is None:
+            try:
+                encoding = config.get(('i18n', ), 'commitEncoding')
+            except KeyError:
+                pass  # No dice
         if encoding is not None:
             c.encoding = encoding
         if message is None:
@@ -720,6 +778,8 @@ class BaseRepo(object):
                 # commit and all its objects as garbage.
                 raise CommitError("%s changed during commit" % (ref,))
 
+        self._del_named_file('MERGE_HEADS')
+
         try:
             self.hooks['post-commit'].execute()
         except HookError as e:  # silent failure
@@ -816,7 +876,8 @@ class Repo(BaseRepo):
             if e.errno != errno.EEXIST:
                 raise
         if committer is None:
-            committer = self._get_user_identity()
+            config = self.get_config_stack()
+            committer = self._get_user_identity(config)
         check_user_identity(committer)
         if timestamp is None:
             timestamp = int(time.time())
@@ -890,6 +951,14 @@ class Repo(BaseRepo):
         with GitFile(os.path.join(self.controldir(), path), 'wb') as f:
             f.write(contents)
 
+    def _del_named_file(self, path):
+        try:
+            os.unlink(os.path.join(self.controldir(), path))
+        except (IOError, OSError) as e:
+            if e.errno == errno.ENOENT:
+                return
+            raise
+
     def get_named_file(self, path, basedir=None):
         """Get a file from the control dir with a specific name.
 
@@ -914,13 +983,6 @@ class Repo(BaseRepo):
                 return None
             raise
 
-    def get_shallow(self):
-        f = self.get_named_file('shallow')
-        if f is None:
-            return set()
-        with f:
-            return set(l.strip() for l in f)
-
     def index_path(self):
         """Return path to the index file."""
         return os.path.join(self.controldir(), INDEX_FILENAME)
@@ -1241,6 +1303,12 @@ class MemoryRepo(BaseRepo):
         """
         self._named_files[path] = contents
 
+    def _del_named_file(self, path):
+        try:
+            del self._named_files[path]
+        except KeyError:
+            pass
+
     def get_named_file(self, path):
         """Get a file from the control dir with a specific name.
 

dulwich/server.py

@@ -46,6 +46,7 @@ import collections
 import os
 import socket
 import sys
+import time
 import zlib
 
 try:
@@ -53,6 +54,7 @@ try:
 except ImportError:
     import socketserver as SocketServer
 
+from dulwich.archive import tar_stream
 from dulwich.errors import (
     ApplyDeltaError,
     ChecksumMismatch,
@@ -329,6 +331,8 @@ class UploadPackHandler(PackHandler):
                 # all relevant tags.
                 # TODO: fix behavior when missing
                 return {}
+        # TODO(jelmer): Integrate this with the refs logic in
+        # Repo.fetch_objects
         tagged = {}
         for name, sha in refs.items():
             peeled_sha = repo.get_peeled(name)
@@ -370,12 +374,10 @@ class UploadPackHandler(PackHandler):
                 self._done_received):
             return
 
-        self.progress(b"dul-daemon says what\n")
         self.progress(
                 ("counting objects: %d, done.\n" % len(objects_iter)).encode(
                     'ascii'))
         write_pack_objects(ProtocolFile(None, write), objects_iter)
-        self.progress(b"how was that, then?\n")
         # we are done
         self.proto.write_pkt_line(None)
 
@@ -551,6 +553,13 @@ class _ProtocolGraphWalker(object):
         values = set(heads.values())
         if self.advertise_refs or not self.http_req:
             for i, (ref, sha) in enumerate(sorted(heads.items())):
+                try:
+                    peeled_sha = self.get_peeled(ref)
+                except KeyError:
+                    # Skip refs that are inaccessible
+                    # TODO(jelmer): Integrate with Repo.fetch_objects refs
+                    # logic.
+                    continue
                 line = sha + b' ' + ref
                 if not i:
                     line += (b'\x00' +
@@ -558,7 +567,6 @@ class _ProtocolGraphWalker(object):
                                  self.handler.capabilities() +
                                  symref_capabilities(symrefs.items())))
                 self.proto.write_pkt_line(line + b'\n')
-                peeled_sha = self.get_peeled(ref)
                 if peeled_sha != sha:
                     self.proto.write_pkt_line(
                         peeled_sha + b' ' + ref + ANNOTATED_TAG_SUFFIX + b'\n')
@@ -929,7 +937,7 @@ class ReceivePackHandler(PackHandler):
                         self.repo.refs.set_if_equals(ref, oldsha, sha)
                     except all_exceptions:
                         ref_status = b'failed to write'
-            except KeyError as e:
+            except KeyError:
                 ref_status = b'bad ref'
             status.append((ref, ref_status))
 
@@ -1005,19 +1013,48 @@ class ReceivePackHandler(PackHandler):
 
 class UploadArchiveHandler(Handler):
 
-    def __init__(self, backend, proto, http_req=None):
+    def __init__(self, backend, args, proto, http_req=None):
         super(UploadArchiveHandler, self).__init__(backend, proto, http_req)
+        self.repo = backend.open_repository(args[0])
 
     def handle(self):
-        # TODO(jelmer)
-        raise NotImplementedError(self.handle)
+        def write(x):
+            return self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, x)
+        arguments = []
+        for pkt in self.proto.read_pkt_seq():
+            (key, value) = pkt.split(b' ', 1)
+            if key != b'argument':
+                raise GitProtocolError('unknown command %s' % key)
+            arguments.append(value.rstrip(b'\n'))
+        prefix = b''
+        format = 'tar'
+        i = 0
+        store = self.repo.object_store
+        while i < len(arguments):
+            argument = arguments[i]
+            if argument == b'--prefix':
+                i += 1
+                prefix = arguments[i]
+            elif argument == b'--format':
+                i += 1
+                format = arguments[i].decode('ascii')
+            else:
+                commit_sha = self.repo.refs[argument]
+                tree = store[store[commit_sha].tree]
+            i += 1
+        self.proto.write_pkt_line(b'ACK\n')
+        self.proto.write_pkt_line(None)
+        for chunk in tar_stream(
+                store, tree, mtime=time.time(), prefix=prefix, format=format):
+            write(chunk)
+        self.proto.write_pkt_line(None)
 
 
 # Default handler classes for git services.
 DEFAULT_HANDLERS = {
   b'git-upload-pack': UploadPackHandler,
   b'git-receive-pack': ReceivePackHandler,
-  # b'git-upload-archive': UploadArchiveHandler,
+  b'git-upload-archive': UploadArchiveHandler,
 }
 
 

dulwich/tests/__init__.py

@@ -110,6 +110,7 @@ def self_test_suite():
         'hooks',
         'ignore',
         'index',
+        'line_ending',
         'lru_cache',
         'mailmap',
         'objects',

dulwich/tests/compat/test_client.py

@@ -210,6 +210,18 @@ class DulwichClientTestBase(object):
                 dest.refs.set_if_equals(r[0], None, r[1])
             self.assertDestEqualsSrc()
 
+    def test_fetch_pack_depth(self):
+        c = self._client()
+        with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
+            result = c.fetch(self._build_path('/server_new.export'), dest,
+                             depth=1)
+            for r in result.refs.items():
+                dest.refs.set_if_equals(r[0], None, r[1])
+            self.assertEqual(
+                    dest.get_shallow(),
+                    set([b'35e0b59e187dd72a0af294aedffc213eaa4d03ff',
+                         b'514dc6d3fbfe77361bcaef320c4d21b72bc10be9']))
+
     def test_repeat(self):
         c = self._client()
         with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
@@ -380,7 +392,7 @@ class DulwichSubprocessClientTest(CompatTestCase, DulwichClientTestBase):
         CompatTestCase.tearDown(self)
 
     def _client(self):
-        return client.SubprocessGitClient(stderr=subprocess.PIPE)
+        return client.SubprocessGitClient()
 
     def _build_path(self, path):
         return self.gitroot + path
@@ -500,6 +512,7 @@ class GitHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
             data = self.rfile.read(nbytes)
         else:
             data = None
+            env['CONTENT_LENGTH'] = '0'
         # throw away additional data [see bug #427345]
         while select.select([self.rfile._sock], [], [], 0)[0]:
             if not self.rfile._sock.recv(1):

dulwich/tests/compat/test_pack.py

@@ -46,7 +46,7 @@ from dulwich.tests.compat.utils import (
     run_git_or_fail,
     )
 
-_NON_DELTA_RE = re.compile(b'non delta: (?P<non_delta>\d+) objects')
+_NON_DELTA_RE = re.compile(b'non delta: (?P<non_delta>\\d+) objects')
 
 
 def _git_verify_pack_object_list(output):

dulwich/tests/test_client.py

@@ -42,6 +42,7 @@ from dulwich import (
     client,
     )
 from dulwich.client import (
+    InvalidWants,
     LocalGitClient,
     TraditionalGitClient,
     TCPGitClient,
@@ -53,6 +54,7 @@ from dulwich.client import (
     SubprocessSSHVendor,
     PLinkSSHVendor,
     UpdateRefsError,
+    check_wants,
     default_urllib3_manager,
     get_transport_and_path,
     get_transport_and_path_from_url,
@@ -98,7 +100,7 @@ class DummyClient(TraditionalGitClient):
         TraditionalGitClient.__init__(self)
 
     def _connect(self, service, path):
-        return Protocol(self.read, self.write), self.can_read
+        return Protocol(self.read, self.write), self.can_read, None
 
 
 class DummyPopen():
@@ -132,7 +134,7 @@ class GitClientTests(TestCase):
         agent_cap = (
             'agent=dulwich/%d.%d.%d' % dulwich.__version__).encode('ascii')
         self.assertEqual(set([b'multi_ack', b'side-band-64k', b'ofs-delta',
-                              b'thin-pack', b'multi_ack_detailed',
+                              b'thin-pack', b'multi_ack_detailed', b'shallow',
                               agent_cap]),
                          set(self.client._fetch_capabilities))
         self.assertEqual(set([b'ofs-delta', b'report-status', b'side-band-64k',
@@ -190,6 +192,20 @@ class GitClientTests(TestCase):
         self.assertEqual({}, ret.symrefs)
         self.assertEqual(self.rout.getvalue(), b'0000')
 
+    def test_fetch_pack_sha_not_in_ref(self):
+        self.rin.write(
+            b'008855dcc6bf963f922e1ed5c4bbaaefcfacef57b1d7 HEAD\x00multi_ack '
+            b'thin-pack side-band side-band-64k ofs-delta shallow no-progress '
+            b'include-tag\n'
+            b'0000')
+        self.rin.seek(0)
+        self.assertRaises(
+                InvalidWants, self.client.fetch_pack,
+                b'bla',
+                lambda heads: ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'],
+                None, None,
+                None)
+
     def test_send_pack_no_sideband64k_with_update_ref_error(self):
         # No side-bank-64k reported by server shouldn't try to parse
         # side band data
@@ -216,7 +232,7 @@ class GitClientTests(TestCase):
         commit.encoding = b'UTF-8'
         commit.message = b'test message'
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {b'refs/foo/bar': commit.id, }
 
         def generate_pack_data(have, want, ofs_delta=False):
@@ -224,7 +240,7 @@ class GitClientTests(TestCase):
 
         self.assertRaises(UpdateRefsError,
                           self.client.send_pack, "blah",
-                          determine_wants, generate_pack_data)
+                          update_refs, generate_pack_data)
 
     def test_send_pack_none(self):
         self.rin.write(
@@ -234,7 +250,7 @@ class GitClientTests(TestCase):
             b'0000')
         self.rin.seek(0)
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {
                 b'refs/heads/master':
                     b'310ca9477129b8586fa2afc779c1f57cf64bba6c'
@@ -243,7 +259,7 @@ class GitClientTests(TestCase):
         def generate_pack_data(have, want, ofs_delta=False):
             return 0, []
 
-        self.client.send_pack(b'/', determine_wants, generate_pack_data)
+        self.client.send_pack(b'/', update_refs, generate_pack_data)
         self.assertEqual(self.rout.getvalue(), b'0000')
 
     def test_send_pack_keep_and_delete(self):
@@ -256,13 +272,13 @@ class GitClientTests(TestCase):
             b'0000')
         self.rin.seek(0)
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {b'refs/heads/master': b'0' * 40}
 
         def generate_pack_data(have, want, ofs_delta=False):
             return 0, []
 
-        self.client.send_pack(b'/', determine_wants, generate_pack_data)
+        self.client.send_pack(b'/', update_refs, generate_pack_data)
         self.assertIn(
             self.rout.getvalue(),
             [b'007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
@@ -281,13 +297,13 @@ class GitClientTests(TestCase):
             b'0000')
         self.rin.seek(0)
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {b'refs/heads/master': b'0' * 40}
 
         def generate_pack_data(have, want, ofs_delta=False):
             return 0, []
 
-        self.client.send_pack(b'/', determine_wants, generate_pack_data)
+        self.client.send_pack(b'/', update_refs, generate_pack_data)
         self.assertIn(
             self.rout.getvalue(),
             [b'007f310ca9477129b8586fa2afc779c1f57cf64bba6c '
@@ -306,7 +322,7 @@ class GitClientTests(TestCase):
             b'0000')
         self.rin.seek(0)
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {
                 b'refs/heads/blah12':
                 b'310ca9477129b8586fa2afc779c1f57cf64bba6c',
@@ -319,7 +335,7 @@ class GitClientTests(TestCase):
 
         f = BytesIO()
         write_pack_objects(f, {})
-        self.client.send_pack('/', determine_wants, generate_pack_data)
+        self.client.send_pack('/', update_refs, generate_pack_data)
         self.assertIn(
             self.rout.getvalue(),
             [b'007f0000000000000000000000000000000000000000 '
@@ -350,7 +366,7 @@ class GitClientTests(TestCase):
         commit.encoding = b'UTF-8'
         commit.message = b'test message'
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {
                 b'refs/heads/blah12': commit.id,
                 b'refs/heads/master':
@@ -362,7 +378,7 @@ class GitClientTests(TestCase):
 
         f = BytesIO()
         write_pack_data(f, *generate_pack_data(None, None))
-        self.client.send_pack(b'/', determine_wants, generate_pack_data)
+        self.client.send_pack(b'/', update_refs, generate_pack_data)
         self.assertIn(
             self.rout.getvalue(),
             [b'007f0000000000000000000000000000000000000000 ' + commit.id +
@@ -384,7 +400,7 @@ class GitClientTests(TestCase):
                 self.rin.write(("%04x" % (len(pkt)+4)).encode('ascii') + pkt)
         self.rin.seek(0)
 
-        def determine_wants(refs):
+        def update_refs(refs):
             return {b'refs/heads/master': b'0' * 40}
 
         def generate_pack_data(have, want, ofs_delta=False):
@@ -392,7 +408,7 @@ class GitClientTests(TestCase):
 
         self.assertRaises(UpdateRefsError,
                           self.client.send_pack, b"/",
-                          determine_wants, generate_pack_data)
+                          update_refs, generate_pack_data)
         self.assertEqual(self.rout.getvalue(), b'0000')
 
 
@@ -549,6 +565,28 @@ class TestGetTransportAndPath(TestCase):
         self.assertEqual('user', c._username)
         self.assertEqual('passwd', c._password)
 
+    def test_http_auth_with_username(self):
+        url = 'https://github.com/jelmer/dulwich'
+
+        c, path = get_transport_and_path(
+                url, username='user2', password='blah')
+
+        self.assertTrue(isinstance(c, HttpGitClient))
+        self.assertEqual('/jelmer/dulwich', path)
+        self.assertEqual('user2', c._username)
+        self.assertEqual('blah', c._password)
+
+    def test_http_auth_with_username_and_in_url(self):
+        url = 'https://user:passwd@github.com/jelmer/dulwich'
+
+        c, path = get_transport_and_path(
+                url, username='user2', password='blah')
+
+        self.assertTrue(isinstance(c, HttpGitClient))
+        self.assertEqual('/jelmer/dulwich', path)
+        self.assertEqual('user', c._username)
+        self.assertEqual('passwd', c._password)
+
     def test_http_no_auth(self):
         url = 'https://github.com/jelmer/dulwich'
 
@@ -635,6 +673,14 @@ class TestGetTransportAndPathFromUrl(TestCase):
         url = 'https://github.com/jelmer/dulwich'
         c, path = get_transport_and_path_from_url(url)
         self.assertTrue(isinstance(c, HttpGitClient))
+        self.assertEqual('https://github.com', c.get_url(b'/'))
+        self.assertEqual('/jelmer/dulwich', path)
+
+    def test_http_port(self):
+        url = 'https://github.com:9090/jelmer/dulwich'
+        c, path = get_transport_and_path_from_url(url)
+        self.assertEqual('https://github.com:9090', c.get_url(b'/'))
+        self.assertTrue(isinstance(c, HttpGitClient))
         self.assertEqual('/jelmer/dulwich', path)
 
     def test_file(self):
@@ -1156,3 +1202,25 @@ class RsyncUrlTests(TestCase):
 
     def test_path(self):
         self.assertRaises(ValueError, parse_rsync_url, '/path')
+
+
+class CheckWantsTests(TestCase):
+
+    def test_fine(self):
+        check_wants(
+            [b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'],
+            {b'refs/heads/blah': b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'})
+
+    def test_missing(self):
+        self.assertRaises(
+            InvalidWants, check_wants,
+            [b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'],
+            {b'refs/heads/blah': b'3f3dc7a53fb752a6961d3a56683df46d4d3bf262'})
+
+    def test_annotated(self):
+        self.assertRaises(
+            InvalidWants, check_wants,
+            [b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'],
+            {b'refs/heads/blah': b'3f3dc7a53fb752a6961d3a56683df46d4d3bf262',
+             b'refs/heads/blah^{}':
+                b'2f3dc7a53fb752a6961d3a56683df46d4d3bf262'})

dulwich/tests/test_diff_tree.py

@@ -493,8 +493,7 @@ class RenameDetectionTest(DiffTestCase):
 
     def _do_test_count_blocks(self, count_blocks):
         blob = make_object(Blob, data=b'a\nb\na\n')
-        self.assertEqual({hash(b'a\n'): 4, hash(b'b\n'): 2},
-                         count_blocks(blob))
+        self.assertBlockCountEqual({b'a\n': 4, b'b\n': 2}, count_blocks(blob))
 
     test_count_blocks = functest_builder(_do_test_count_blocks,
                                          _count_blocks_py)
@@ -503,17 +502,21 @@ class RenameDetectionTest(DiffTestCase):
 
     def _do_test_count_blocks_no_newline(self, count_blocks):
         blob = make_object(Blob, data=b'a\na')
-        self.assertEqual({hash(b'a\n'): 2, hash(b'a'): 1}, _count_blocks(blob))
+        self.assertBlockCountEqual({b'a\n': 2, b'a': 1}, _count_blocks(blob))
 
     test_count_blocks_no_newline = functest_builder(
         _do_test_count_blocks_no_newline, _count_blocks_py)
     test_count_blocks_no_newline_extension = ext_functest_builder(
         _do_test_count_blocks_no_newline, _count_blocks)
 
+    def assertBlockCountEqual(self, expected, got):
+        self.assertEqual(
+            {(hash(l) & 0xffffffff): c for (l, c) in expected.items()},
+            {(h & 0xffffffff): c for (h, c) in got.items()})
+
     def _do_test_count_blocks_chunks(self, count_blocks):
         blob = ShaFile.from_raw_chunks(Blob.type_num, [b'a\nb', b'\na\n'])
-        self.assertEqual({hash(b'a\n'): 4, hash(b'b\n'): 2},
-                         _count_blocks(blob))
+        self.assertBlockCountEqual({b'a\n': 4, b'b\n': 2}, _count_blocks(blob))
 
     test_count_blocks_chunks = functest_builder(_do_test_count_blocks_chunks,
                                                 _count_blocks_py)
@@ -524,9 +527,12 @@ class RenameDetectionTest(DiffTestCase):
         a = b'a' * 64
         data = a + b'xxx\ny\n' + a + b'zzz\n'
         blob = make_object(Blob, data=data)
-        self.assertEqual({hash(b'a' * 64): 128, hash(b'xxx\n'): 4,
-                          hash(b'y\n'): 2, hash(b'zzz\n'): 4},
-                         _count_blocks(blob))
+        self.assertBlockCountEqual(
+            {b'a' * 64: 128,
+             b'xxx\n': 4,
+             b'y\n': 2,
+             b'zzz\n': 4},
+            _count_blocks(blob))
 
     test_count_blocks_long_lines = functest_builder(
         _do_test_count_blocks_long_lines, _count_blocks_py)
@@ -725,8 +731,8 @@ class RenameDetectionTest(DiffTestCase):
     def test_content_rename_one_to_one(self):
         b11 = make_object(Blob, data=b'a\nb\nc\nd\n')
         b12 = make_object(Blob, data=b'a\nb\nc\ne\n')
-        b21 = make_object(Blob, data=b'e\nf\ng\n\h')
-        b22 = make_object(Blob, data=b'e\nf\ng\n\i')
+        b21 = make_object(Blob, data=b'e\nf\ng\n\nh')
+        b22 = make_object(Blob, data=b'e\nf\ng\n\ni')
         tree1 = self.commit_tree([(b'a', b11), (b'b', b21)])
         tree2 = self.commit_tree([(b'c', b12), (b'd', b22)])
         self.assertEqual(

dulwich/tests/test_ignore.py

@@ -106,10 +106,10 @@ class ReadIgnorePatterns(TestCase):
 
 # and an empty line:
 
-\#not a comment
+\\#not a comment
 !negative
 with trailing whitespace 
-with escaped trailing whitespace\ 
+with escaped trailing whitespace\\ 
 """)  # noqa: W291
         self.assertEqual(list(read_ignore_patterns(f)), [
             b'\\#not a comment',

dulwich/tests/test_line_ending.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+# test_line_ending.py -- Tests for the line ending functions
+# encoding: utf-8
+# Copyright (C) 2018-2019 Boris Feld <boris.feld@comet.ml>
+#
+# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
+# General Public License as public by the Free Software Foundation; version 2.0
+# or (at your option) any later version. You can redistribute it and/or
+# modify it under the terms of either of these two licenses.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# You should have received a copy of the licenses; if not, see
+# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
+# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
+# License, Version 2.0.
+#
+
+"""Tests for the line ending conversion."""
+
+from dulwich.line_ending import (
+    convert_crlf_to_lf,
+    convert_lf_to_crlf,
+    get_checkin_filter_autocrlf,
+    get_checkout_filter_autocrlf,
+)
+from dulwich.tests import TestCase
+
+
+class LineEndingConversion(TestCase):
+    """Test the line ending conversion functions in various cases"""
+
+    def test_convert_crlf_to_lf_no_op(self):
+        self.assertEqual(convert_crlf_to_lf(b"foobar"), b"foobar")
+
+    def test_convert_crlf_to_lf(self):
+        self.assertEqual(
+            convert_crlf_to_lf(b"line1\r\nline2"), b"line1\nline2"
+        )
+
+    def test_convert_crlf_to_lf_mixed(self):
+        self.assertEqual(
+            convert_crlf_to_lf(b"line1\r\n\nline2"), b"line1\n\nline2"
+        )
+
+    def test_convert_lf_to_crlf_no_op(self):
+        self.assertEqual(convert_lf_to_crlf(b"foobar"), b"foobar")
+
+    def test_convert_lf_to_crlf(self):
+        self.assertEqual(
+            convert_lf_to_crlf(b"line1\nline2"), b"line1\r\nline2"
+        )
+
+    def test_convert_lf_to_crlf_mixed(self):
+        self.assertEqual(
+            convert_lf_to_crlf(b"line1\r\n\nline2"), b"line1\r\n\r\nline2"
+        )
+
+
+class GetLineEndingAutocrlfFilters(TestCase):
+
+    def test_get_checkin_filter_autocrlf_default(self):
+        checkin_filter = get_checkin_filter_autocrlf(b"false")
+
+        self.assertEqual(checkin_filter, None)
+
+    def test_get_checkin_filter_autocrlf_true(self):
+        checkin_filter = get_checkin_filter_autocrlf(b"true")
+
+        self.assertEqual(checkin_filter, convert_crlf_to_lf)
+
+    def test_get_checkin_filter_autocrlf_input(self):
+        checkin_filter = get_checkin_filter_autocrlf(b"input")
+
+        self.assertEqual(checkin_filter, convert_crlf_to_lf)
+
+    def test_get_checkout_filter_autocrlf_default(self):
+        checkout_filter = get_checkout_filter_autocrlf(b"false")
+
+        self.assertEqual(checkout_filter, None)
+
+    def test_get_checkout_filter_autocrlf_true(self):
+        checkout_filter = get_checkout_filter_autocrlf(b"true")
+
+        self.assertEqual(checkout_filter, convert_lf_to_crlf)
+
+    def test_get_checkout_filter_autocrlf_input(self):
+        checkout_filter = get_checkout_filter_autocrlf(b"input")
+
+        self.assertEqual(checkout_filter, None)

dulwich/tests/test_object_store.py

@@ -47,6 +47,7 @@ from dulwich.object_store import (
     OverlayObjectStore,
     ObjectStoreGraphWalker,
     commit_tree_changes,
+    read_packs_file,
     tree_lookup_path,
     )
 from dulwich.pack import (
@@ -645,3 +646,10 @@ class CommitTreeChangesTests(TestCase):
         self.assertEqual(set(new_tree), {b'a', b'ad', b'c'})
         ad_tree = self.store[new_tree[b'ad'][1]]
         self.assertEqual(set(ad_tree), {b'b', b'c'})
+
+
+class TestReadPacksFile(TestCase):
+
+    def test_read_packs(self):
+        self.assertEqual(["pack-1.pack"], list(read_packs_file(BytesIO(b"""P pack-1.pack
+"""))))

dulwich/tests/test_objects.py

@@ -204,6 +204,9 @@ class BlobReadTests(TestCase):
         self.assertEqual(
                 t.message,
                 b'This is a signed tag\n'
+                )
+        self.assertEqual(
+                t.signature,
                 b'-----BEGIN PGP SIGNATURE-----\n'
                 b'Version: GnuPG v1.4.9 (GNU/Linux)\n'
                 b'\n'
@@ -673,6 +676,27 @@ class CommitParseTests(ShaFileCheckTests):
             with self.assertRaises(ObjectFormatException):
                 commit.check()
 
+    def test_mangled_author_line(self):
+        """Mangled author line should successfully parse"""
+        author_line = (
+            b'Karl MacMillan <kmacmill@redhat.com> <"Karl MacMillan '
+            b'<kmacmill@redhat.com>"> 1197475547 -0500'
+        )
+        expected_identity = (
+            b'Karl MacMillan <kmacmill@redhat.com> <"Karl MacMillan '
+            b'<kmacmill@redhat.com>">'
+        )
+        commit = Commit.from_string(
+            self.make_commit_text(author=author_line)
+        )
+
+        # The commit parses properly
+        self.assertEqual(commit.author, expected_identity)
+
+        # But the check fails because the author identity is bogus
+        with self.assertRaises(ObjectFormatException):
+            commit.check()
+
     def test_parse_gpgsig(self):
         c = Commit.from_string(b"""tree aaff74984cccd156a469afa7d9ab10e4777beb24
 author Jelmer Vernooij <jelmer@samba.org> 1412179807 +0200

dulwich/tests/test_objectspec.py

@@ -174,6 +174,11 @@ class ParseReftupleTests(TestCase):
         self.assertEqual((b"refs/heads/foo", None, False),
                          parse_reftuple(r, r, b"refs/heads/foo:"))
 
+    def test_default_with_string(self):
+        r = {b"refs/heads/foo": "bla"}
+        self.assertEqual((b"refs/heads/foo", b"refs/heads/foo", False),
+                         parse_reftuple(r, r, "foo"))
+
 
 class ParseReftuplesTests(TestCase):
 

dulwich/tests/test_porcelain.py

@@ -224,6 +224,20 @@ class CloneTests(PorcelainTestCase):
             self.repo.path, target_path, checkout=True, errstream=errstream)
         r.close()
 
+    def test_no_head_no_checkout_outstream_errstream_autofallback(self):
+        f1_1 = make_object(Blob, data=b'f1')
+        commit_spec = [[1]]
+        trees = {1: [(b'f1', f1_1), (b'f2', f1_1)]}
+
+        (c1, ) = build_commit_graph(self.repo.object_store, commit_spec, trees)
+        self.repo.refs[b"refs/heads/master"] = c1.id
+        target_path = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, target_path)
+        errstream = porcelain.NoneStream()
+        r = porcelain.clone(
+            self.repo.path, target_path, checkout=True, errstream=errstream)
+        r.close()
+
 
 class InitTests(TestCase):
 
@@ -1490,6 +1504,7 @@ class DescribeTests(PorcelainTestCase):
 
 
 class HelperTests(PorcelainTestCase):
+
     def test_path_to_tree_path_base(self):
         self.assertEqual(
             b'bar', porcelain.path_to_tree_path('/home/foo', '/home/foo/bar'))
@@ -1526,3 +1541,36 @@ class HelperTests(PorcelainTestCase):
                 os.path.join(os.getcwd(), '..'), 'baz'))
         finally:
             os.chdir(cwd)
+
+
+class GetObjectBypathTests(PorcelainTestCase):
+
+    def test_simple(self):
+        fullpath = os.path.join(self.repo.path, 'foo')
+        with open(fullpath, 'w') as f:
+            f.write("BAR")
+        porcelain.add(repo=self.repo.path, paths=[fullpath])
+        porcelain.commit(
+                self.repo.path, message=b"Some message",
+                author=b"Joe <joe@example.com>",
+                committer=b"Bob <bob@example.com>")
+        self.assertEqual(
+            b"BAR",
+            porcelain.get_object_by_path(self.repo, 'foo').data)
+
+    def test_missing(self):
+        self.assertRaises(
+            KeyError,
+            porcelain.get_object_by_path, self.repo, 'foo')
+
+
+class WriteTreeTests(PorcelainTestCase):
+
+    def test_simple(self):
+        fullpath = os.path.join(self.repo.path, 'foo')
+        with open(fullpath, 'w') as f:
+            f.write("BAR")
+        porcelain.add(repo=self.repo.path, paths=[fullpath])
+        self.assertEqual(
+            b'd2092c8a9f311f0311083bf8d177f2ca0ab5b241',
+            porcelain.write_tree(self.repo))

dulwich/tests/test_refs.py

@@ -335,10 +335,37 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
 
     def test_setitem(self):
         RefsContainerTests.test_setitem(self)
-        f = open(os.path.join(self._refs.path, b'refs', b'some', b'ref'), 'rb')
-        self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
-                         f.read()[:40])
-        f.close()
+        path = os.path.join(self._refs.path, b'refs', b'some', b'ref')
+        with open(path, 'rb') as f:
+            self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
+                             f.read()[:40])
+
+        self.assertRaises(
+            OSError, self._refs.__setitem__,
+            b'refs/some/ref/sub', b'42d06bd4b77fed026b154d16493e5deab78f02ec')
+
+    def test_setitem_packed(self):
+        with open(os.path.join(self._refs.path, b'packed-refs'), 'w') as f:
+            f.write('# pack-refs with: peeled fully-peeled sorted \n')
+            f.write(
+                '42d06bd4b77fed026b154d16493e5deab78f02ec refs/heads/packed\n')
+
+        # It's allowed to set a new ref on a packed ref, the new ref will be
+        # placed outside on refs/
+        self._refs[b'refs/heads/packed'] = (
+            b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8'
+        )
+        packed_ref_path = os.path.join(
+            self._refs.path, b'refs', b'heads', b'packed')
+        with open(packed_ref_path, 'rb') as f:
+            self.assertEqual(
+                b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8',
+                f.read()[:40])
+
+        self.assertRaises(
+            OSError, self._refs.__setitem__,
+            b'refs/heads/packed/sub',
+            b'42d06bd4b77fed026b154d16493e5deab78f02ec')
 
     def test_setitem_symbolic(self):
         ones = b'1' * 40
@@ -486,6 +513,13 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
                          self._refs.read_ref(b'refs/heads/packed'))
         self.assertEqual(None, self._refs.read_ref(b'nonexistant'))
 
+    def test_read_loose_ref(self):
+        self._refs[b'refs/heads/foo'] = (
+            b'df6800012397fb85c56e7418dd4eb9405dee075c'
+        )
+
+        self.assertEqual(None, self._refs.read_ref(b'refs/heads/foo/bar'))
+
     def test_non_ascii(self):
         try:
             encoded_ref = u'refs/tags/schön'.encode(
@@ -506,6 +540,9 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
         self.assertEqual(expected_refs, self._repo.get_refs())
 
     def test_cyrillic(self):
+        if sys.platform == 'win32':
+            raise SkipTest(
+                    "filesystem encoding doesn't support arbitrary bytes")
         # reported in https://github.com/dulwich/dulwich/issues/608
         name = b'\xcd\xee\xe2\xe0\xff\xe2\xe5\xf2\xea\xe01'
         encoded_ref = b'refs/heads/' + name

dulwich/tests/test_repository.py

@@ -266,6 +266,35 @@ class RepositoryRootTests(TestCase):
                 r.get_walker(b'2a72d929692c41d8554c07f6301757ba18a65d91')],
             [b'2a72d929692c41d8554c07f6301757ba18a65d91'])
 
+    def test_fetch(self):
+        r = self.open_repo('a.git')
+        tmp_dir = self.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmp_dir)
+        t = Repo.init(tmp_dir)
+        self.addCleanup(t.close)
+        r.fetch(t)
+        self.assertIn(b'a90fa2d900a17e99b433217e988c4eb4a2e9a097', t)
+        self.assertIn(b'a90fa2d900a17e99b433217e988c4eb4a2e9a097', t)
+        self.assertIn(b'a90fa2d900a17e99b433217e988c4eb4a2e9a097', t)
+        self.assertIn(b'28237f4dc30d0d462658d6b937b08a0f0b6ef55a', t)
+        self.assertIn(b'b0931cadc54336e78a1d980420e3268903b57a50', t)
+
+    def test_fetch_ignores_missing_refs(self):
+        r = self.open_repo('a.git')
+        missing = b'1234566789123456789123567891234657373833'
+        r.refs[b'refs/heads/blah'] = missing
+        tmp_dir = self.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmp_dir)
+        t = Repo.init(tmp_dir)
+        self.addCleanup(t.close)
+        r.fetch(t)
+        self.assertIn(b'a90fa2d900a17e99b433217e988c4eb4a2e9a097', t)
+        self.assertIn(b'a90fa2d900a17e99b433217e988c4eb4a2e9a097', t)
+        self.assertIn(b'a90fa2d900a17e99b433217e988c4eb4a2e9a097', t)
+        self.assertIn(b'28237f4dc30d0d462658d6b937b08a0f0b6ef55a', t)
+        self.assertIn(b'b0931cadc54336e78a1d980420e3268903b57a50', t)
+        self.assertNotIn(missing, t)
+
     def test_clone(self):
         r = self.open_repo('a.git')
         tmp_dir = self.mkdtemp()
@@ -662,13 +691,28 @@ class BuildRepoRootTests(TestCase):
         self.assertEqual([], r[commit_sha].parents)
         self._root_commit = commit_sha
 
-    def test_shallow(self):
+    def test_get_shallow(self):
         self.assertEqual(set(), self._repo.get_shallow())
         with open(os.path.join(self._repo.path, '.git', 'shallow'), 'wb') as f:
             f.write(b'a90fa2d900a17e99b433217e988c4eb4a2e9a097\n')
         self.assertEqual({b'a90fa2d900a17e99b433217e988c4eb4a2e9a097'},
                          self._repo.get_shallow())
 
+    def test_update_shallow(self):
+        self._repo.update_shallow(None, None)  # no op
+        self.assertEqual(set(), self._repo.get_shallow())
+        self._repo.update_shallow(
+                [b'a90fa2d900a17e99b433217e988c4eb4a2e9a097'],
+                None)
+        self.assertEqual(
+                {b'a90fa2d900a17e99b433217e988c4eb4a2e9a097'},
+                self._repo.get_shallow())
+        self._repo.update_shallow(
+                [b'a90fa2d900a17e99b433217e988c4eb4a2e9a097'],
+                [b'f9e39b120c68182a4ba35349f832d0e4e61f485c'])
+        self.assertEqual({b'a90fa2d900a17e99b433217e988c4eb4a2e9a097'},
+                         self._repo.get_shallow())
+
     def test_build_repo(self):
         r = self._repo
         self.assertEqual(b'ref: refs/heads/master', r.refs.read_ref(b'HEAD'))
@@ -710,6 +754,34 @@ class BuildRepoRootTests(TestCase):
         self.assertTrue(stat.S_ISLNK(b_mode))
         self.assertEqual(b'a', r[b_id].data)
 
+    def test_commit_merge_heads_file(self):
+        tmp_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmp_dir)
+        r = Repo.init(tmp_dir)
+        with open(os.path.join(r.path, 'a'), 'w') as f:
+            f.write('initial text')
+        c1 = r.do_commit(
+            b'initial commit',
+            committer=b'Test Committer <test@nodomain.com>',
+            author=b'Test Author <test@nodomain.com>',
+            commit_timestamp=12395, commit_timezone=0,
+            author_timestamp=12395, author_timezone=0)
+        with open(os.path.join(r.path, 'a'), 'w') as f:
+            f.write('merged text')
+        with open(os.path.join(r.path, '.git', 'MERGE_HEADS'), 'w') as f:
+            f.write('c27a2d21dd136312d7fa9e8baabb82561a1727d0\n')
+        r.stage(['a'])
+        commit_sha = r.do_commit(
+            b'deleted a',
+            committer=b'Test Committer <test@nodomain.com>',
+            author=b'Test Author <test@nodomain.com>',
+            commit_timestamp=12395, commit_timezone=0,
+            author_timestamp=12395, author_timezone=0)
+        self.assertEqual([
+            c1,
+            b'c27a2d21dd136312d7fa9e8baabb82561a1727d0'],
+            r[commit_sha].parents)
+
     def test_commit_deleted(self):
         r = self._repo
         os.remove(os.path.join(r.path, 'a'))
@@ -748,6 +820,19 @@ class BuildRepoRootTests(TestCase):
             encoding=b"iso8859-1")
         self.assertEqual(b"iso8859-1", r[commit_sha].encoding)
 
+    def test_commit_encoding_from_config(self):
+        r = self._repo
+        c = r.get_config()
+        c.set(('i18n',), 'commitEncoding', 'iso8859-1')
+        c.write_to_path()
+        commit_sha = r.do_commit(
+            b'commit with strange character \xee',
+            committer=b'Test Committer <test@nodomain.com>',
+            author=b'Test Author <test@nodomain.com>',
+            commit_timestamp=12395, commit_timezone=0,
+            author_timestamp=12395, author_timezone=0)
+        self.assertEqual(b"iso8859-1", r[commit_sha].encoding)
+
     def test_commit_config_identity(self):
         # commit falls back to the users' identity if it wasn't specified
         r = self._repo

requirements.txt

@@ -0,0 +1 @@
+urllib3[secure]>=1.23

setup.cfg

@@ -1,28 +1,5 @@
-[metadata]
-name = dulwich
-author = Jelmer Vernooij
-author_email = jelmer@jelmer.uk
-home-page = https://www.dulwich.io/
-description = file:README.md
-summary = Python Git Library
-classifiers = 
-	Development Status :: 4 - Beta
-	License :: OSI Approved :: Apache Software License
-	Programming Language :: Python :: 2.7
-	Programming Language :: Python :: 3.3
-	Programming Language :: Python :: 3.4
-	Programming Language :: Python :: 3.5
-	Programming Language :: Python :: 3.6
-	Programming Language :: Python :: Implementation :: CPython
-	Programming Language :: Python :: Implementation :: PyPy
-	Operating System :: POSIX
-	Operating System :: Microsoft :: Windows
-	Topic :: Software Development :: Version Control
-keyword = git, vcs
-project_urls = 
-	Bug Tracker = https://github.com/dulwich/dulwich/issues
-
-[build_ext]
+[flake8]
+exclude = build,.git,build-pypy,.tox
 
 [egg_info]
 tag_build = 

setup.py

@@ -11,10 +11,11 @@ except ImportError:
 else:
     has_setuptools = True
 from distutils.core import Distribution
+import io
 import os
 import sys
 
-dulwich_version_string = '0.19.6'
+dulwich_version_string = '0.19.10'
 
 include_dirs = []
 # Windows MSVC support
@@ -76,22 +77,49 @@ setup_kwargs = {}
 if has_setuptools:
     setup_kwargs['extras_require'] = {
         'fastimport': ['fastimport'],
-        'https': ['urllib3[secure]>=1.21'],
+        'https': ['urllib3[secure]>=1.23'],
         }
-    setup_kwargs['install_requires'] = ['urllib3>=1.21', 'certifi']
+    setup_kwargs['install_requires'] = ['urllib3>=1.23', 'certifi']
     setup_kwargs['include_package_data'] = True
     setup_kwargs['test_suite'] = 'dulwich.tests.test_suite'
     setup_kwargs['tests_require'] = tests_require
 
+with io.open(os.path.join(os.path.dirname(__file__), "README.rst"),
+             encoding="utf-8") as f:
+    description = f.read()
 
 setup(name='dulwich',
+      author="Jelmer Vernooij",
+      author_email="jelmer@jelmer.uk",
+      url="https://www.dulwich.io/",
+      long_description=description,
+      description="Python Git Library",
       version=dulwich_version_string,
       license='Apachev2 or later or GPLv2',
+      project_urls={
+          "Bug Tracker": "https://github.com/dulwich/dulwich/issues",
+          "Repository": "https://www.dulwich.io/code/",
+          "GitHub": "https://github.com/dulwich/dulwich",
+      },
+      keywords="git vcs",
       packages=['dulwich', 'dulwich.tests', 'dulwich.tests.compat',
                 'dulwich.contrib'],
       package_data={'': ['../docs/tutorial/*.txt']},
       scripts=['bin/dulwich', 'bin/dul-receive-pack', 'bin/dul-upload-pack'],
       ext_modules=ext_modules,
       distclass=DulwichDistribution,
+      classifiers=[
+          'Development Status :: 4 - Beta',
+          'License :: OSI Approved :: Apache Software License',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3.4',
+          'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
+          'Programming Language :: Python :: Implementation :: CPython',
+          'Programming Language :: Python :: Implementation :: PyPy',
+          'Operating System :: POSIX',
+          'Operating System :: Microsoft :: Windows',
+          'Topic :: Software Development :: Version Control',
+      ],
       **setup_kwargs
       )