New upstream version 0.19.3

.coveragerc

@@ -1,6 +0,0 @@
-[run]
-branch = True
-
-[report]
-exclude_lines =
-    raise NotImplementedError

.gitignore

@@ -1,23 +0,0 @@
-_trial_temp
-build
-build-pypy
-MANIFEST
-dist
-apidocs
-*,cover
-.testrepository
-*.pyc
-*.pyd
-*.pyo
-*.so
-*~
-*.swp
-*.swh
-*.swn
-*.swo
-docs/tutorial/index.html
-dulwich.egg-info/
-.tox/
-.idea/
-.coverage
-htmlcov/

.mailmap

@@ -1,23 +0,0 @@
-Jelmer Vernooij <jelmer@jelmer.uk>
-Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@jelmer.uk>
-Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@samba.org>
-Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@debian.org>
-Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@canonical.com>
-Jelmer Vernooij <jelmer@jelmer.uk> <jelmer@google.com>
-Martin <gzlist@googlemail.com> <martin.packman@canonical.com>
-Dave Borowitz <dborowitz@google.com> <ddborowitz@gmail.com>
-Dave Borowitz <dborowitz@google.com> <dborowitz@google.com>
-John Carr <john.carr@unrouted.co.uk>
-Mark Mikofski <bwanamarko@yahoo.com> <mark.mikofski@sunpowercorp.com>
-Mark Mikofski <bwanamarko@yahoo.com> <bwana.marko@yahoo.com>
-David Carr <david@carrclan.us>
-Jon Bain <jsbain@yahoo.com> <jsbain@yahoo.com>
-James Westby <jw+debian@jameswestby.net> <jw+debian@jameswestby.net>
-David Keijser <david.keijser@klarna.com> <keijser@gmail.com>
-Benoît HERVIER <khertan@khertan.net> <khertan@khertan.net>
-Ryan Faulkner <rfaulk@yahoo-inc.com> <rfaulkner@wikimedia.org>
-David Bennett <davbennett@google.com> <david@dbinit.com>
-Risto Kankkunen <risto.kankkunen@iki.fi> <risto.kankkunen@f-secure.com>
-Augie Fackler <durin42@gmail.com> <raf@durin42.com>
-Damien Tournoud <damien@commerceguys.com> <damien@platform.sh>
-Marcin Kuźmiński <marcin@python-blog.com> <marcin@python-works.com>

.travis.yml

@@ -4,13 +4,13 @@ cache: pip
 
 python:
   - 2.7
-  - 3.3
   - 3.4
   - 3.5
   - 3.5-dev
   - 3.6
   - 3.6-dev
   - pypy3.3-5.2-alpha1
+  - pypy3.5
 
 env:
   - PYTHONHASHSEED=random
@@ -20,17 +20,19 @@ matrix:
   include:
     - python: pypy
       env: TEST_REQUIRE=fastimport
+    - python: 3.3
+      env: TEST_REQUIRE=fastimport
 
 install:
   - travis_retry pip install -U pip coverage codecov flake8 $TEST_REQUIRE
 
 script:
   # Test without c extensions
-  - python -m coverage run -p --source=dulwich -m unittest dulwich.tests.test_suite
+  - python -m coverage run -p -m unittest dulwich.tests.test_suite
 
   # Test with c extensions
   - python setup.py build_ext -i
-  - python -m coverage run -p --source=dulwich -m unittest dulwich.tests.test_suite
+  - python -m coverage run -p -m unittest dulwich.tests.test_suite
 
   # Style
   - make style

Makefile

@@ -5,7 +5,7 @@ FLAKE8 ?= flake8
 SETUP = $(PYTHON) setup.py
 PYDOCTOR ?= pydoctor
 TESTRUNNER ?= unittest
-RUNTEST = PYTHONHASHSEED=random PYTHONPATH=.:$(PYTHONPATH) $(PYTHON) -m $(TESTRUNNER) $(TEST_OPTIONS)
+RUNTEST = PYTHONHASHSEED=random PYTHONPATH=$(shell pwd)$(if $(PYTHONPATH),:$(PYTHONPATH),) $(PYTHON) -m $(TESTRUNNER) $(TEST_OPTIONS)
 COVERAGE = python3-coverage
 
 DESTDIR=/
@@ -65,7 +65,7 @@ before-push: check
 	git diff origin/master | $(PEP8) --diff
 
 coverage:
-	$(COVERAGE) run --source=dulwich -m unittest dulwich.tests.test_suite dulwich.contrib.test_suite
+	$(COVERAGE) run -m unittest dulwich.tests.test_suite dulwich.contrib.test_suite
 
 coverage-html: coverage
 	$(COVERAGE) html

NEWS

@@ -1,3 +1,46 @@
+0.19.3	2018-06-17
+
+ IMPROVEMENTS
+
+  * Add really basic `dulwich.porcelain.fsck` implementation.
+    (Jelmer Vernooij)
+
+  * When the `DULWICH_PDB` environment variable is set, make
+    SIGQUIT open pdb in the 'dulwich' command.
+
+  * Add `checkout` argument to `Repo.clone`.
+    (Jelmer Vernooij, #503)
+
+  * Add `Repo.get_shallow` method. (Jelmer Vernooij)
+
+  * Add basic `dulwich.stash` module. (Jelmer Vernooij)
+
+  * Support a `prefix` argument to `dulwich.archive.tar_stream`.
+    (Jelmer Vernooij)
+
+ BUG FIXES
+
+  * Fix handling of encoding for tags. (Jelmer Vernooij, #608)
+
+  * Fix tutorial tests on Python 3. (Jelmer Vernooij, #573)
+
+  * Fix remote refs created by `porcelain.fetch`. (Daniel Andersson, #623)
+
+  * More robust pack creation on Windows. (Daniel Andersson)
+
+  * Fix recursive option for `porcelain.ls_tree`. (Romain Keramitas)
+
+ TESTS
+
+  * Some improvements to paramiko tests. (Filipp Frizzy)
+
+0.19.2	2018-04-07
+
+ BUG FIXES
+
+  * Fix deprecated Index.iterblobs method.
+    (Jelmer Vernooij)
+
 0.19.1	2018-04-05
 
  IMPROVEMENTS
@@ -29,7 +72,7 @@
  API CHANGES
 
   * Index.iterblobs has been renamed to Index.iterobjects.
-   (Jelmer Vernooij)
+    (Jelmer Vernooij)
 
 0.19.0	2018-03-10
 
@@ -73,7 +116,7 @@
  BUG FIXES
 
   * Fix handling of empty repositories in ``porcelain.clone``.
-   (#570, Jelmer Vernooij)
+    (#570, Jelmer Vernooij)
 
   * Raise an error when attempting to add paths that are not under the
     repository. (Jelmer Vernooij)

PKG-INFO

@@ -1,21 +1,13 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.19.1
+Version: 0.19.3
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
+Author: Jelmer Vernooij
 Author-email: jelmer@jelmer.uk
 License: Apachev2 or later or GPLv2
-Description: 
-              Python implementation of the Git file formats and protocols,
-              without the need to have git installed.
-        
-              All functionality is available in pure Python. Optional
-              C extensions can be built for improved performance.
-        
-              The project is named after the part of London that Mr. and Mrs. Git live
-              in in the particular Monty Python sketch.
-              
-Keywords: git
+Project-URL: Bug Tracker, https://github.com/dulwich/dulwich/issues
+Description: UNKNOWN
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License

README.md

@@ -1,5 +1,5 @@
 [![Build Status](https://travis-ci.org/dulwich/dulwich.png?branch=master)](https://travis-ci.org/dulwich/dulwich)
-[![Windows Build status](https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true)](https://ci.appveyor.com/project/dulwich/dulwich/branch/master)
+[![Windows Build status](https://ci.appveyor.com/api/projects/status/mob7g4vnrfvvoweb?svg=true)](https://ci.appveyor.com/project/jelmer/dulwich/branch/master)
 
 This is the Dulwich project.
 

bin/dulwich

@@ -1,7 +1,7 @@
 #!/usr/bin/python -u
 #
 # dulwich - Simple command-line interface to Dulwich
-# Copyright (C) 2008-2011 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2011 Jelmer Vernooij <jelmer@jelmer.uk>
 # vim: expandtab
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
@@ -37,6 +37,13 @@ import signal
 def signal_int(signal, frame):
     sys.exit(1)
 
+
+def signal_quit(signal, frame):
+    import pdb
+    pdb.set_trace()
+
+if 'DULWICH_PDB' in os.environ:
+    signal.signal(signal.SIGQUIT, signal_quit)
 signal.signal(signal.SIGINT, signal_int)
 
 from dulwich import porcelain
@@ -59,12 +66,18 @@ class Command(object):
 class cmd_archive(Command):
 
     def run(self, args):
-        opts, args = getopt(args, "", [])
-        client, path = get_transport_and_path(args.pop(0))
-        location = args.pop(0)
+        parser = optparse.OptionParser()
+        parser.add_option("--remote", type=str,
+                          help="Retrieve archive from specified remote repo")
+        options, args = parser.parse_args(args)
         committish = args.pop(0)
-        porcelain.archive(location, committish, outstream=sys.stdout,
-            errstream=sys.stderr)
+        if options.remote:
+            client, path = get_transport_and_path(options.remote)
+            client.archive(path, committish, sys.stdout.write,
+                    write_error=sys.stderr.write)
+        else:
+            porcelain.archive('.', committish, outstream=sys.stdout,
+                errstream=sys.stderr)
 
 
 class cmd_add(Command):
@@ -112,6 +125,15 @@ class cmd_fetch(Command):
             print("%s -> %s" % item)
 
 
+class cmd_fsck(Command):
+
+    def run(self, args):
+        opts, args = getopt(args, "", [])
+        opts = dict(opts)
+        for (obj, msg) in porcelain.fsck('.'):
+            print("%s: %s" % (obj, msg))
+
+
 class cmd_log(Command):
 
     def run(self, args):
@@ -547,6 +569,42 @@ class cmd_check_mailmap(Command):
             print(canonical_identity)
 
 
+class cmd_stash_list(Command):
+
+    def run(self, args):
+        parser = optparse.OptionParser()
+        options, args = parser.parse_args(args)
+        for i, entry in porcelain.stash_list('.'):
+            print("stash@{%d}: %s" % (i, entry.message.rstrip('\n')))
+
+
+class cmd_stash_push(Command):
+
+    def run(self, args):
+        parser = optparse.OptionParser()
+        options, args = parser.parse_args(args)
+        porcelain.stash_push('.')
+        print("Saved working directory and index state")
+
+
+class cmd_stash_pop(Command):
+
+    def run(self, args):
+        parser = optparse.OptionParser()
+        options, args = parser.parse_args(args)
+        porcelain.stash_pop('.')
+        print("Restrored working directory and index state")
+
+
+class cmd_stash(SuperCommand):
+
+    subcommands = {
+        "list": cmd_stash_list,
+        "pop": cmd_stash_pop,
+        "push": cmd_stash_push,
+    }
+
+
 class cmd_help(Command):
 
     def run(self, args):
@@ -584,6 +642,7 @@ commands = {
     "dump-index": cmd_dump_index,
     "fetch-pack": cmd_fetch_pack,
     "fetch": cmd_fetch,
+    "fsck": cmd_fsck,
     "help": cmd_help,
     "init": cmd_init,
     "log": cmd_log,
@@ -599,6 +658,7 @@ commands = {
     "rev-list": cmd_rev_list,
     "rm": cmd_rm,
     "show": cmd_show,
+    "stash": cmd_stash,
     "status": cmd_status,
     "symbolic-ref": cmd_symbolic_ref,
     "tag": cmd_tag,

build.cmd

@@ -1,21 +0,0 @@
-@echo off
-:: To build extensions for 64 bit Python 3, we need to configure environment
-:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
-:: MS Windows SDK for Windows 7 and .NET Framework 4
-::
-:: More details at:
-:: https://github.com/cython/cython/wiki/CythonExtensionsOnWindows
-
-IF "%DISTUTILS_USE_SDK%"=="1" (
-    ECHO Configuring environment to build with MSVC on a 64bit architecture
-    ECHO Using Windows SDK 7.1
-    "C:\Program Files\Microsoft SDKs\Windows\v7.1\Setup\WindowsSdkVer.exe" -q -version:v7.1
-    CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release
-    SET MSSdk=1
-    REM Need the following to allow tox to see the SDK compiler
-    SET TOX_TESTENV_PASSENV=DISTUTILS_USE_SDK MSSdk INCLUDE LIB
-) ELSE (
-    ECHO Using default MSVC build environment
-)
-
-CALL %*

devscripts/PREAMBLE.c

@@ -1,15 +0,0 @@
- * Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
- * General Public License as public by the Free Software Foundation; version 2.0
- * or (at your option) any later version. You can redistribute it and/or
- * modify it under the terms of either of these two licenses.
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * You should have received a copy of the licenses; if not, see
- * <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
- * and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
- * License, Version 2.0.

devscripts/replace-preamble.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/zsh
-perl -i -p0e "s{\Q$(cat PREAMBLE.py.old)\E}{$(cat devscripts/PREAMBLE.py)}g" dulwich/**/*.py bin/dul*
-perl -i -p0e "s{\Q$(cat PREAMBLE.c.old)\E}{$(cat devscripts/PREAMBLE.c)}g" dulwich/*.c

docs/tutorial/.gitignore

@@ -1,2 +0,0 @@
-*.html
-myrepo

docs/tutorial/object-store.txt

@@ -169,8 +169,10 @@ The diff between the previous head and the new one can be printed using
 write_tree_diff::
 
   >>> from dulwich.patch import write_tree_diff
-  >>> import sys
-  >>> write_tree_diff(sys.stdout, repo.object_store, commit.tree, tree.id)
+  >>> from io import BytesIO
+  >>> out = BytesIO()
+  >>> write_tree_diff(out, repo.object_store, commit.tree, tree.id)
+  >>> import sys; _ = sys.stdout.write(out.getvalue().decode('ascii'))
   diff --git a/spam b/spam
   index c55063a..16ee268 100644
   --- a/spam

docs/tutorial/remote.txt

@@ -5,7 +5,7 @@ Most of the tests in this file require a Dulwich server, so let's start one:
     >>> from dulwich.repo import Repo
     >>> from dulwich.server import DictBackend, TCPGitServer
     >>> import threading
-    >>> repo = Repo.init(b"remote", mkdir=True)
+    >>> repo = Repo.init("remote", mkdir=True)
     >>> cid = repo.do_commit(b"message", committer=b"Jelmer <jelmer@samba.org>")
     >>> backend = DictBackend({b'/': repo})
     >>> dul_server = TCPGitServer(backend, b'localhost', 0)
@@ -32,7 +32,7 @@ Dulwich provides support for accessing remote repositories in
 one manually::
 
    >>> from dulwich.client import TCPGitClient
-   >>> client = TCPGitClient(server_address.encode('ascii'), server_port)
+   >>> client = TCPGitClient(server_address, server_port)
 
 Retrieving raw pack files
 -------------------------
@@ -76,7 +76,7 @@ in which case Dulwich takes care of providing the right graph walker, and
 importing the received pack file into the local repository::
 
    >>> from dulwich.repo import Repo
-   >>> local = Repo.init(b"local", mkdir=True)
+   >>> local = Repo.init("local", mkdir=True)
    >>> remote_refs = client.fetch(b"/", local)
 
 Let's shut down the server now that all tests have been run::

docs/tutorial/repo.txt

@@ -53,7 +53,7 @@ so only non-bare repositories will have an index, too. To open the index, simply
 call::
 
     >>> index = repo.open_index()
-    >>> print(index.path.decode(sys.getfilesystemencoding()))
+    >>> print(index.path)
     myrepo/.git/index
 
 Since the repository was just created, the index will be empty::

dulwich.egg-info/PKG-INFO

@@ -1,21 +1,13 @@
 Metadata-Version: 2.1
 Name: dulwich
-Version: 0.19.1
+Version: 0.19.3
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
+Author: Jelmer Vernooij
 Author-email: jelmer@jelmer.uk
 License: Apachev2 or later or GPLv2
-Description: 
-              Python implementation of the Git file formats and protocols,
-              without the need to have git installed.
-        
-              All functionality is available in pure Python. Optional
-              C extensions can be built for improved performance.
-        
-              The project is named after the part of London that Mr. and Mrs. Git live
-              in in the particular Monty Python sketch.
-              
-Keywords: git
+Project-URL: Bug Tracker, https://github.com/dulwich/dulwich/issues
+Description: UNKNOWN
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License

dulwich.egg-info/SOURCES.txt

@@ -1,6 +1,3 @@
-.coveragerc
-.gitignore
-.mailmap
 .testr.conf
 .travis.yml
 AUTHORS
@@ -13,25 +10,19 @@ README.md
 README.swift.md
 TODO
 appveyor.yml
-build.cmd
 dulwich.cfg
-requirements.txt
 setup.cfg
 setup.py
 tox.ini
 bin/dul-receive-pack
 bin/dul-upload-pack
 bin/dulwich
-devscripts/PREAMBLE.c
-devscripts/PREAMBLE.py
-devscripts/replace-preamble.sh
 docs/Makefile
 docs/conf.py
 docs/index.txt
 docs/make.bat
 docs/performance.txt
 docs/protocol.txt
-docs/tutorial/.gitignore
 docs/tutorial/Makefile
 docs/tutorial/conclusion.txt
 docs/tutorial/encoding.txt
@@ -81,7 +72,6 @@ dulwich.egg-info/SOURCES.txt
 dulwich.egg-info/dependency_links.txt
 dulwich.egg-info/requires.txt
 dulwich.egg-info/top_level.txt
-dulwich/contrib/README.md
 dulwich/contrib/__init__.py
 dulwich/contrib/paramiko_vendor.py
 dulwich/contrib/release_robot.py
@@ -116,6 +106,7 @@ dulwich/tests/test_reflog.py
 dulwich/tests/test_refs.py
 dulwich/tests/test_repository.py
 dulwich/tests/test_server.py
+dulwich/tests/test_stash.py
 dulwich/tests/test_utils.py
 dulwich/tests/test_walk.py
 dulwich/tests/test_web.py
@@ -211,4 +202,5 @@ dulwich/tests/data/trees/70/c190eb48fa8bbb50ddc692a17b44cb781af7f6
 examples/clone.py
 examples/config.py
 examples/diff.py
-examples/latest_change.py
+examples/latest_change.py
+examples/memoryrepo.py

dulwich/__init__.py

@@ -1,6 +1,6 @@
 # __init__.py -- The git module of dulwich
 # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
-# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
@@ -22,4 +22,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 19, 1)
+__version__ = (0, 19, 3)

dulwich/_objects.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
+ * Copyright (C) 2009 Jelmer Vernooij <jelmer@jelmer.uk>
  *
  * Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
  * General Public License as public by the Free Software Foundation; version 2.0

dulwich/_pack.c

@@ -1,5 +1,5 @@
 /* 
- * Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
+ * Copyright (C) 2009 Jelmer Vernooij <jelmer@jelmer.uk>
  *
  * Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
  * General Public License as public by the Free Software Foundation; version 2.0

dulwich/archive.py

@@ -68,7 +68,7 @@ class ChunkedBytesIO(object):
         return b''.join(buf)
 
 
-def tar_stream(store, tree, mtime, format=''):
+def tar_stream(store, tree, mtime, prefix=b'', format=''):
     """Generate a tar stream for the contents of a Git tree.
 
     Returns a generator that lazily assembles a .tar.gz archive, yielding it in
@@ -96,7 +96,7 @@ def tar_stream(store, tree, mtime, format=''):
             buf.write(struct.pack('<L', mtime))
             buf.seek(0, SEEK_END)
 
-        for entry_abspath, entry in _walk_tree(store, tree):
+        for entry_abspath, entry in _walk_tree(store, tree, prefix):
             try:
                 blob = store[entry.sha]
             except KeyError:

dulwich/client.py

@@ -1,5 +1,5 @@
 # client.py -- Implementation of the client side git protocols
-# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
@@ -765,10 +765,17 @@ class TraditionalGitClient(GitClient):
             return refs
 
     def archive(self, path, committish, write_data, progress=None,
-                write_error=None):
+                write_error=None, format=None, subdirs=None, prefix=None):
         proto, can_read = self._connect(b'upload-archive', path)
         with proto:
+            if format is not None:
+                proto.write_pkt_line(b"argument --format=" + format)
             proto.write_pkt_line(b"argument " + committish)
+            if subdirs is not None:
+                for subdir in subdirs:
+                    proto.write_pkt_line(b"argument " + subdir)
+            if prefix is not None:
+                proto.write_pkt_line(b"argument --prefix=" + prefix)
             proto.write_pkt_line(None)
             pkt = proto.read_pkt_line()
             if pkt == b"NACK\n":

dulwich/config.py

@@ -1,5 +1,5 @@
 # config.py - Reading and writing Git config files
-# Copyright (C) 2011-2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2011-2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/contrib/README.md

@@ -1,3 +0,0 @@
-This directory contains code that some may find useful. Code here is not an official
-part of Dulwich, and may no longer work. Unlike the rest of Dulwich, it is not regularly
-tested.

dulwich/contrib/__init__.py

@@ -1,5 +1,5 @@
 # __init__.py -- Contrib module for Dulwich
-# Copyright (C) 2014 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2014 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/contrib/paramiko_vendor.py

@@ -97,7 +97,7 @@ class _ParamikoWrapper(object):
 
         # Closed socket
         if not data:
-            return
+            return b''
 
         # Read more if needed
         if n and data_len < n:

dulwich/errors.py

@@ -1,6 +1,6 @@
 # errors.py -- errors for dulwich
 # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
-# Copyright (C) 2009-2012 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2009-2012 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/fastexport.py

@@ -1,5 +1,5 @@
 # __init__.py -- Fast export/import functionality
-# Copyright (C) 2010-2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2010-2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/file.py

@@ -47,7 +47,7 @@ def _fancy_rename(oldname, newname):
 
     # destination file exists
     try:
-        (fd, tmpfile) = tempfile.mkstemp(".tmp", prefix=oldname+".", dir=".")
+        (fd, tmpfile) = tempfile.mkstemp(".tmp", prefix=oldname, dir=".")
         os.close(fd)
         os.remove(tmpfile)
     except OSError:
@@ -118,7 +118,10 @@ class _GitFile(object):
 
     def __init__(self, filename, mode, bufsize):
         self._filename = filename
-        self._lockfilename = '%s.lock' % self._filename
+        if isinstance(self._filename, bytes):
+            self._lockfilename = self._filename + b'.lock'
+        else:
+            self._lockfilename = self._filename + '.lock'
         try:
             fd = os.open(
                 self._lockfilename,
@@ -167,15 +170,15 @@ class _GitFile(object):
         os.fsync(self._file.fileno())
         self._file.close()
         try:
-            try:
-                os.rename(self._lockfilename, self._filename)
-            except OSError as e:
-                if sys.platform == 'win32' and e.errno == errno.EEXIST:
+            if getattr(os, 'replace', None) is not None:
+                os.replace(self._lockfilename, self._filename)
+            else:
+                if sys.platform != 'win32':
+                    os.rename(self._lockfilename, self._filename)
+                else:
                     # Windows versions prior to Vista don't support atomic
                     # renames
                     _fancy_rename(self._lockfilename, self._filename)
-                else:
-                    raise
         finally:
             self.abort()
 

dulwich/index.py

@@ -1,5 +1,5 @@
 # index.py -- File parser/writer for the git index file
-# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
@@ -275,8 +275,8 @@ class Index(object):
 
     def iterblobs(self):
         import warnings
-        warnings.warn(PendingDeprecationWarning, 'Use iterobjects() instead.')
-        return self.iterblobs()
+        warnings.warn('Use iterobjects() instead.', PendingDeprecationWarning)
+        return self.iterobjects()
 
     def clear(self):
         """Remove all contents from this index."""

dulwich/mailmap.py

@@ -1,5 +1,5 @@
 # mailmap.py -- Mailmap reader
-# Copyright (C) 2018 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2018 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/object_store.py

@@ -1,5 +1,5 @@
 # object_store.py -- Object store for git objects
-# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #                         and others
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
@@ -129,14 +129,14 @@ class BaseObjectStore(object):
         """
         raise NotImplementedError(self.add_object)
 
-    def add_objects(self, objects):
+    def add_objects(self, objects, progress=None):
         """Add a set of objects to this object store.
 
         :param objects: Iterable over a list of (object, path) tuples
         """
         raise NotImplementedError(self.add_objects)
 
-    def add_pack_data(self, count, pack_data):
+    def add_pack_data(self, count, pack_data, progress=None):
         """Add pack data to this object store.
 
         :param num_items: Number of items to add
@@ -147,7 +147,7 @@ class BaseObjectStore(object):
             return
         f, commit, abort = self.add_pack()
         try:
-            write_pack_data(f, count, pack_data)
+            write_pack_data(f, count, pack_data, progress)
         except BaseException:
             abort()
             raise
@@ -460,14 +460,16 @@ class PackBasedObjectStore(BaseObjectStore):
                 pass
         raise KeyError(hexsha)
 
-    def add_objects(self, objects):
+    def add_objects(self, objects, progress=None):
         """Add a set of objects to this object store.
 
         :param objects: Iterable over (object, path) tuples, should support
             __len__.
         :return: Pack object of the objects written.
         """
-        return self.add_pack_data(*pack_objects_to_data(objects))
+        return self.add_pack_data(
+                *pack_objects_to_data(objects),
+                progress=progress)
 
 
 class DiskObjectStore(PackBasedObjectStore):
@@ -649,14 +651,16 @@ class DiskObjectStore(PackBasedObjectStore):
         # Move the pack in.
         entries.sort()
         pack_base_name = self._get_pack_basepath(entries)
+        target_pack = pack_base_name + '.pack'
         if sys.platform == 'win32':
+            # Windows might have the target pack file lingering. Attempt
+            # removal, silently passing if the target does not exist.
             try:
-                os.rename(path, pack_base_name + '.pack')
-            except WindowsError:
-                os.remove(pack_base_name + '.pack')
-                os.rename(path, pack_base_name + '.pack')
-        else:
-            os.rename(path, pack_base_name + '.pack')
+                os.remove(target_pack)
+            except (IOError, OSError) as e:
+                if e.errno != errno.ENOENT:
+                    raise
+        os.rename(path, target_pack)
 
         # Write the index.
         index_file = GitFile(pack_base_name + '.idx', 'wb')
@@ -713,9 +717,16 @@ class DiskObjectStore(PackBasedObjectStore):
             return self._pack_cache[basename]
         except KeyError:
             pass
-        else:
-            os.unlink(path)
-        os.rename(path, basename + ".pack")
+        target_pack = basename + '.pack'
+        if sys.platform == 'win32':
+            # Windows might have the target pack file lingering. Attempt
+            # removal, silently passing if the target does not exist.
+            try:
+                os.remove(target_pack)
+            except (IOError, OSError) as e:
+                if e.errno != errno.ENOENT:
+                    raise
+        os.rename(path, target_pack)
         final_pack = Pack(basename)
         self._add_known_pack(basename, final_pack)
         return final_pack
@@ -828,7 +839,7 @@ class MemoryObjectStore(BaseObjectStore):
         """
         self._data[obj.id] = obj.copy()
 
-    def add_objects(self, objects):
+    def add_objects(self, objects, progress=None):
         """Add a set of objects to this object store.
 
         :param objects: Iterable over a list of (object, path) tuples
@@ -1262,10 +1273,10 @@ class OverlayObjectStore(BaseObjectStore):
             raise NotImplementedError(self.add_object)
         return self.add_store.add_object(object)
 
-    def add_objects(self, objects):
+    def add_objects(self, objects, progress=None):
         if self.add_store is None:
             raise NotImplementedError(self.add_object)
-        return self.add_store.add_objects(objects)
+        return self.add_store.add_objects(objects, progress)
 
     @property
     def packs(self):

dulwich/objects.py

@@ -1,6 +1,6 @@
 # objects.py -- Access to base git objects
 # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
-# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
@@ -1014,8 +1014,10 @@ class Tree(ShaFile):
         for name, mode, sha in parse_tree(b''.join(self._chunked_text),
                                           True):
             check_hexsha(sha, 'invalid sha %s' % sha)
-            if b'/' in name or name in (b'', b'.', b'..'):
-                raise ObjectFormatException('invalid name %s' % name)
+            if b'/' in name or name in (b'', b'.', b'..', b'.git'):
+                raise ObjectFormatException(
+                        'invalid name %s' %
+                        name.decode('utf-8', 'replace'))
 
             if mode not in allowed_modes:
                 raise ObjectFormatException('invalid mode %06o' % mode)

dulwich/objectspec.py

@@ -1,5 +1,5 @@
 # objectspec.py -- Object specification
-# Copyright (C) 2014 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2014 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/pack.py

@@ -1,6 +1,6 @@
 # pack.py -- For dealing with packed git objects.
 # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
-# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
@@ -1617,19 +1617,24 @@ def write_pack_objects(f, objects, delta_window_size=None, deltify=None):
     return write_pack_data(f, pack_contents_count, pack_contents)
 
 
-def write_pack_data(f, num_records, records):
+def write_pack_data(f, num_records, records, progress=None):
     """Write a new pack data file.
 
     :param f: File to write to
     :param num_records: Number of records
     :param records: Iterator over type_num, object_id, delta_base, raw
+    :param progress: Function to report progress to
     :return: Dict mapping id -> (offset, crc32 checksum), pack checksum
     """
     # Write the pack
     entries = {}
     f = SHA1Writer(f)
     write_pack_header(f, num_records)
-    for type_num, object_id, delta_base, raw in records:
+    for i, (type_num, object_id, delta_base, raw) in enumerate(records):
+        if progress is not None:
+            progress((
+                'writing pack data: %d/%d\r' %
+                (i, num_records)).encode('ascii'))
         offset = f.offset()
         if delta_base is not None:
             try:

dulwich/patch.py

@@ -1,5 +1,5 @@
 # patch.py -- For dealing with packed-style patches.
-# Copyright (C) 2009-2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2009-2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/porcelain.py

@@ -1,5 +1,5 @@
 # porcelain.py -- Porcelain-like layer on top of Dulwich
-# Copyright (C) 2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
@@ -292,6 +292,7 @@ def clone(source, target=None, bare=False, checkout=None,
     :param origin: Name of remote from the repository used to clone
     :return: The new repository
     """
+    # TODO(jelmer): This code overlaps quite a bit with Repo.clone
     if outstream is not None:
         import warnings
         warnings.warn(
@@ -317,22 +318,10 @@ def clone(source, target=None, bare=False, checkout=None,
         r = Repo.init_bare(target)
     else:
         r = Repo.init(target)
+
+    reflog_message = b'clone: from ' + source.encode('utf-8')
     try:
-        fetch_result = client.fetch(
-            host_path, r, determine_wants=r.object_store.determine_wants_all,
-            progress=errstream.write)
-        ref_message = b"clone: from " + source.encode('utf-8')
-        r.refs.import_refs(
-            b'refs/remotes/' + origin,
-            {n[len(b'refs/heads/'):]: v for (n, v) in fetch_result.refs.items()
-                if n.startswith(b'refs/heads/')},
-            message=ref_message)
-        r.refs.import_refs(
-            b'refs/tags',
-            {n[len(b'refs/tags/'):]: v for (n, v) in fetch_result.refs.items()
-                if n.startswith(b'refs/tags/') and
-                not n.endswith(ANNOTATED_TAG_SUFFIX)},
-            message=ref_message)
+        fetch_result = fetch(r, host_path, origin, message=reflog_message)
         target_config = r.get_config()
         if not isinstance(source, bytes):
             source = source.encode(DEFAULT_ENCODING)
@@ -344,7 +333,7 @@ def clone(source, target=None, bare=False, checkout=None,
         # TODO(jelmer): Support symref capability,
         # https://github.com/jelmer/dulwich/issues/485
         try:
-            head = r[fetch_result.refs[b"HEAD"]]
+            head = r[fetch_result[b'HEAD']]
         except KeyError:
             head = None
         else:
@@ -377,7 +366,7 @@ def add(repo=".", paths=None):
             paths = [paths]
         for p in paths:
             relpath = os.path.relpath(p, r.path)
-            if relpath.startswith('../'):
+            if relpath.startswith('..' + os.path.sep):
                 raise ValueError('path %r is not in repo' % relpath)
             # FIXME: Support patterns, directories.
             if ignore_manager.is_ignored(relpath):
@@ -847,7 +836,7 @@ def status(repo=".", ignored=False):
     """Returns staged, unstaged, and untracked changes relative to the HEAD.
 
     :param repo: Path to repository or repository object
-    :param ignored: Whether to include ignoed files in `untracked`
+    :param ignored: Whether to include ignored files in `untracked`
     :return: GitStatus tuple,
         staged -    list of staged paths (diff index/HEAD)
         unstaged -  list of unstaged paths (diff index/working-tree)
@@ -1069,7 +1058,7 @@ def branch_list(repo):
 
 
 def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout,
-          errstream=default_bytes_err_stream, **kwargs):
+          errstream=default_bytes_err_stream, message=None, **kwargs):
     """Fetch objects from a remote server.
 
     :param repo: Path to the repository
@@ -1077,14 +1066,26 @@ def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout,
     :param remote_name: Name for remote server
     :param outstream: Output stream (defaults to stdout)
     :param errstream: Error stream (defaults to stderr)
+    :param message: Reflog message (defaults to b"fetch: from <remote_name>")
     :return: Dictionary with refs on the remote
     """
+    if message is None:
+        message = b'fetch: from ' + remote_location.encode("utf-8")
     with open_repo_closing(repo) as r:
         client, path = get_transport_and_path(
-                remote_location, config=r.get_config_stack(), **kwargs)
+            remote_location, config=r.get_config_stack(), **kwargs)
         fetch_result = client.fetch(path, r, progress=errstream.write)
-        ref_name = b'refs/remotes/' + remote_name
-        r.refs.import_refs(ref_name, strip_peeled_refs(fetch_result.refs))
+        stripped_refs = strip_peeled_refs(fetch_result.refs)
+        branches = {
+            n[len(b'refs/heads/'):]: v for (n, v) in stripped_refs.items()
+            if n.startswith(b'refs/heads/')}
+        r.refs.import_refs(
+            b'refs/remotes/' + remote_name, branches, message=message)
+        tags = {
+            n[len(b'refs/tags/'):]: v for (n, v) in stripped_refs.items()
+            if n.startswith(b'refs/tags/') and
+            not n.endswith(ANNOTATED_TAG_SUFFIX)}
+        r.refs.import_refs(b'refs/tags', tags, message=message)
     return fetch_result.refs
 
 
@@ -1148,7 +1149,7 @@ def ls_tree(repo, treeish=b"HEAD", outstream=sys.stdout, recursive=False,
                 outstream.write(name + b"\n")
             else:
                 outstream.write(pretty_format_tree_entry(name, mode, sha))
-            if stat.S_ISDIR(mode):
+            if stat.S_ISDIR(mode) and recursive:
                 list_tree(store, sha, name)
     with open_repo_closing(repo) as r:
         tree = parse_tree(r, treeish)
@@ -1239,3 +1240,45 @@ def check_mailmap(repo, contact):
                 raise
             mailmap = Mailmap()
         return mailmap.lookup(contact)
+
+
+def fsck(repo):
+    """Check a repository.
+
+    :param repo: A path to the repository
+    :return: Iterator over errors/warnings
+    """
+    with open_repo_closing(repo) as r:
+        # TODO(jelmer): check pack files
+        # TODO(jelmer): check graph
+        # TODO(jelmer): check refs
+        for sha in r.object_store:
+            o = r.object_store[sha]
+            try:
+                o.check()
+            except Exception as e:
+                yield (sha, e)
+
+
+def stash_list(repo):
+    """List all stashes in a repository."""
+    with open_repo_closing(repo) as r:
+        from dulwich.stash import Stash
+        stash = Stash.from_repo(r)
+        return enumerate(list(stash.stashes()))
+
+
+def stash_push(repo):
+    """Push a new stash onto the stack."""
+    with open_repo_closing(repo) as r:
+        from dulwich.stash import Stash
+        stash = Stash.from_repo(r)
+        stash.push()
+
+
+def stash_pop(repo):
+    """Pop a new stash from the stack."""
+    with open_repo_closing(repo) as r:
+        from dulwich.stash import Stash
+        stash = Stash.from_repo(r)
+        stash.pop()

dulwich/protocol.py

@@ -1,6 +1,6 @@
 # protocol.py -- Shared parts of the git protocols
 # Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
-# Copyright (C) 2008-2012 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2012 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/refs.py

@@ -1,5 +1,5 @@
 # refs.py -- For dealing with git refs
-# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
@@ -475,8 +475,14 @@ class DiskRefsContainer(RefsContainer):
 
     def __init__(self, path, worktree_path=None, logger=None):
         super(DiskRefsContainer, self).__init__(logger=logger)
+        if getattr(path, 'encode', None) is not None:
+            path = path.encode(sys.getfilesystemencoding())
         self.path = path
-        self.worktree_path = worktree_path or path
+        if worktree_path is None:
+            worktree_path = path
+        if getattr(worktree_path, 'encode', None) is not None:
+            worktree_path = worktree_path.encode(sys.getfilesystemencoding())
+        self.worktree_path = worktree_path
         self._packed_refs = None
         self._peeled_refs = None
 
@@ -486,11 +492,14 @@ class DiskRefsContainer(RefsContainer):
     def subkeys(self, base):
         subkeys = set()
         path = self.refpath(base)
-        for root, dirs, files in os.walk(path):
-            dir = root[len(path):].strip(os.path.sep).replace(os.path.sep, "/")
+        for root, unused_dirs, files in os.walk(path):
+            dir = root[len(path):]
+            if os.path.sep != '/':
+                dir = dir.replace(os.path.sep.encode(
+                    sys.getfilesystemencoding()), b"/")
+            dir = dir.strip(b'/')
             for filename in files:
-                refname = (("%s/%s" % (dir, filename))
-                           .strip("/").encode(sys.getfilesystemencoding()))
+                refname = b"/".join(([dir] if dir else []) + [filename])
                 # check_ref_format requires at least one /, so we prepend the
                 # base before calling it.
                 if check_ref_format(base + b'/' + refname):
@@ -505,12 +514,14 @@ class DiskRefsContainer(RefsContainer):
         if os.path.exists(self.refpath(b'HEAD')):
             allkeys.add(b'HEAD')
         path = self.refpath(b'')
-        for root, dirs, files in os.walk(self.refpath(b'refs')):
-            dir = root[len(path):].strip(os.path.sep).replace(os.path.sep, "/")
+        refspath = self.refpath(b'refs')
+        for root, unused_dirs, files in os.walk(refspath):
+            dir = root[len(path):]
+            if os.path.sep != '/':
+                dir = dir.replace(
+                    os.path.sep.encode(sys.getfilesystemencoding()), b"/")
             for filename in files:
-                refname = (
-                    "%s/%s" % (dir, filename)).encode(
-                            sys.getfilesystemencoding())
+                refname = b"/".join([dir, filename])
                 if check_ref_format(refname):
                     allkeys.add(refname)
         allkeys.update(self.get_packed_refs())
@@ -520,14 +531,13 @@ class DiskRefsContainer(RefsContainer):
         """Return the disk path of a ref.
 
         """
-        if (getattr(self.path, "encode", None) and
-                getattr(name, "decode", None)):
-            name = name.decode(sys.getfilesystemencoding())
         if os.path.sep != "/":
-            name = name.replace("/", os.path.sep)
+            name = name.replace(
+                    b"/",
+                    os.path.sep.encode(sys.getfilesystemencoding()))
         # TODO: as the 'HEAD' reference is working tree specific, it
         # should actually not be a part of RefsContainer
-        if name == 'HEAD':
+        if name == b'HEAD':
             return os.path.join(self.worktree_path, name)
         else:
             return os.path.join(self.path, name)
@@ -546,7 +556,7 @@ class DiskRefsContainer(RefsContainer):
             # None if and only if _packed_refs is also None.
             self._packed_refs = {}
             self._peeled_refs = {}
-            path = os.path.join(self.path, 'packed-refs')
+            path = os.path.join(self.path, b'packed-refs')
             try:
                 f = GitFile(path, 'rb')
             except IOError as e:
@@ -614,7 +624,7 @@ class DiskRefsContainer(RefsContainer):
     def _remove_packed_ref(self, name):
         if self._packed_refs is None:
             return
-        filename = os.path.join(self.path, 'packed-refs')
+        filename = os.path.join(self.path, b'packed-refs')
         # reread cached refs from disk, while holding the lock
         f = GitFile(filename, 'wb')
         try:
@@ -643,19 +653,17 @@ class DiskRefsContainer(RefsContainer):
         self._check_refname(name)
         self._check_refname(other)
         filename = self.refpath(name)
+        f = GitFile(filename, 'wb')
         try:
-            f = GitFile(filename, 'wb')
-            try:
-                f.write(SYMREF + other + b'\n')
-            except (IOError, OSError):
-                f.abort()
-                raise
-            else:
-                sha = self.follow(name)[-1]
-                self._log(name, sha, sha, committer=committer,
-                          timestamp=timestamp, timezone=timezone,
-                          message=message)
-        finally:
+            f.write(SYMREF + other + b'\n')
+            sha = self.follow(name)[-1]
+            self._log(name, sha, sha, committer=committer,
+                      timestamp=timestamp, timezone=timezone,
+                      message=message)
+        except BaseException:
+            f.abort()
+            raise
+        else:
             f.close()
 
     def set_if_equals(self, name, old_ref, new_ref, committer=None,
@@ -882,7 +890,7 @@ def write_info_refs(refs, store):
 
 
 def is_local_branch(x):
-    return x.startswith(b'refs/heads/')
+    return x.startswith(LOCAL_BRANCH_PREFIX)
 
 
 def strip_peeled_refs(refs):

dulwich/repo.py

@@ -1,6 +1,6 @@
 # repo.py -- For dealing with git repositories.
 # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
-# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
@@ -62,6 +62,9 @@ from dulwich.objects import (
     Tag,
     Tree,
     )
+from dulwich.pack import (
+    pack_objects_to_data,
+    )
 
 from dulwich.hooks import (
     PreCommitShellHook,
@@ -267,11 +270,31 @@ class BaseRepo(object):
         """
         if determine_wants is None:
             determine_wants = target.object_store.determine_wants_all
-        target.object_store.add_objects(
-            self.fetch_objects(determine_wants, target.get_graph_walker(),
-                               progress))
+        count, pack_data = self.fetch_pack_data(
+                determine_wants, target.get_graph_walker(), progress)
+        target.object_store.add_pack_data(count, pack_data, progress)
         return self.get_refs()
 
+    def fetch_pack_data(self, determine_wants, graph_walker, progress,
+                        get_tagged=None):
+        """Fetch the pack data required for a set of revisions.
+
+        :param determine_wants: Function that takes a dictionary with heads
+            and returns the list of heads to fetch.
+        :param graph_walker: Object that can iterate over the list of revisions
+            to fetch and has an "ack" method that will be called to acknowledge
+            that a revision is present.
+        :param progress: Simple progress function that will be called with
+            updated progress strings.
+        :param get_tagged: Function that returns a dict of pointed-to sha ->
+            tag sha for including tags.
+        :return: count and iterator over pack data
+        """
+        # TODO(jelmer): Fetch pack data directly, don't create objects first.
+        objects = self.fetch_objects(determine_wants, graph_walker, progress,
+                                     get_tagged)
+        return pack_objects_to_data(objects)
+
     def fetch_objects(self, determine_wants, graph_walker, progress,
                       get_tagged=None):
         """Fetch the missing objects required for a set of revisions.
@@ -430,6 +453,13 @@ class BaseRepo(object):
         backends = [self.get_config()] + StackedConfig.default_backends()
         return StackedConfig(backends, writable=backends[0])
 
+    def get_shallow(self):
+        """Get the set of shallow commits.
+
+        :return: Set of shallow commits.
+        """
+        return set()
+
     def get_peeled(self, ref):
         """Get the peeled value of a ref.
 
@@ -884,6 +914,13 @@ class Repo(BaseRepo):
                 return None
             raise
 
+    def get_shallow(self):
+        f = self.get_named_file('shallow')
+        if f is None:
+            return set()
+        with f:
+            return set(l.strip() for l in f)
+
     def index_path(self):
         """Return path to the index file."""
         return os.path.join(self.controldir(), INDEX_FILENAME)
@@ -951,7 +988,7 @@ class Repo(BaseRepo):
         index.write()
 
     def clone(self, target_path, mkdir=True, bare=False,
-              origin=b"origin"):
+              origin=b"origin", checkout=None):
         """Clone this repository.
 
         :param target_path: Target path
@@ -964,6 +1001,8 @@ class Repo(BaseRepo):
         if not bare:
             target = self.init(target_path, mkdir=mkdir)
         else:
+            if checkout:
+                raise ValueError("checkout and bare are incompatible")
             target = self.init_bare(target_path, mkdir=mkdir)
         self.fetch(target)
         encoded_path = self.path
@@ -995,7 +1034,9 @@ class Repo(BaseRepo):
                                          message=ref_message)
             target[b'HEAD'] = head_sha
 
-            if not bare:
+            if checkout is None:
+                checkout = (not bare)
+            if checkout:
                 # Checkout HEAD to target dir
                 target.reset_index()
 

dulwich/server.py

@@ -1,6 +1,6 @@
 # server.py -- Implementation of the server side git protocols
 # Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk>
-# Coprygith (C) 2011-2012 Jelmer Vernooij <jelmer@samba.org>
+# Coprygith (C) 2011-2012 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/stash.py

@@ -69,7 +69,7 @@ class Stash(object):
     def pop(self, index):
         raise NotImplementedError(self.drop)
 
-    def create(self, committer=None, author=None, message=None):
+    def push(self, committer=None, author=None, message=None):
         """Create a new stash.
 
         :param committer: Optional committer name to use

dulwich/tests/__init__.py

@@ -124,6 +124,7 @@ def self_test_suite():
         'refs',
         'repository',
         'server',
+        'stash',
         'utils',
         'walk',
         'web',
@@ -134,6 +135,13 @@ def self_test_suite():
 
 
 def tutorial_test_suite():
+    import dulwich.client  # noqa: F401
+    import dulwich.config  # noqa: F401
+    import dulwich.index  # noqa: F401
+    import dulwich.reflog  # noqa: F401
+    import dulwich.repo  # noqa: F401
+    import dulwich.server  # noqa: F401
+    import dulwich.patch  # noqa: F401
     tutorial = [
         'introduction',
         'file-format',
@@ -146,22 +154,23 @@ def tutorial_test_suite():
 
     def setup(test):
         test.__old_cwd = os.getcwd()
-        test.__dulwich_tempdir = tempfile.mkdtemp()
-        os.chdir(test.__dulwich_tempdir)
+        test.tempdir = tempfile.mkdtemp()
+        test.globs.update({'tempdir': test.tempdir})
+        os.chdir(test.tempdir)
 
     def teardown(test):
         os.chdir(test.__old_cwd)
-        shutil.rmtree(test.__dulwich_tempdir)
+        shutil.rmtree(test.tempdir)
     return doctest.DocFileSuite(
+            module_relative=True, package='dulwich.tests',
             setUp=setup, tearDown=teardown, *tutorial_files)
 
 
 def nocompat_test_suite():
     result = unittest.TestSuite()
     result.addTests(self_test_suite())
+    result.addTests(tutorial_test_suite())
     from dulwich.contrib import test_suite as contrib_test_suite
-    if sys.version_info[0] == 2:
-        result.addTests(tutorial_test_suite())
     result.addTests(contrib_test_suite())
     return result
 
@@ -176,7 +185,7 @@ def compat_test_suite():
 def test_suite():
     result = unittest.TestSuite()
     result.addTests(self_test_suite())
-    if sys.version_info[0] == 2 and sys.platform != 'win32':
+    if sys.platform != 'win32':
         result.addTests(tutorial_test_suite())
     from dulwich.tests.compat import test_suite as compat_test_suite
     result.addTests(compat_test_suite())

dulwich/tests/compat/__init__.py

@@ -1,5 +1,5 @@
 # __init__.py -- Compatibility tests for dulwich
-# Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2010 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/tests/test_archive.py

@@ -75,6 +75,12 @@ class ArchiveTests(TestCase):
         self.addCleanup(tf.close)
         self.assertEqual(["somename"], tf.getnames())
 
+    def test_prefix(self):
+        stream = self._get_example_tar_stream(mtime=0, prefix=b'blah')
+        tf = tarfile.TarFile(fileobj=stream)
+        self.addCleanup(tf.close)
+        self.assertEqual(["blah/somename"], tf.getnames())
+
     def test_gzip_mtime(self):
         stream = self._get_example_tar_stream(mtime=1234, format='gz')
         expected_mtime = struct.pack('<L', 1234)

dulwich/tests/test_blackbox.py

@@ -1,5 +1,5 @@
 # test_blackbox.py -- blackbox tests
-# Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2010 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/tests/test_client.py

@@ -1,5 +1,5 @@
 # test_client.py -- Tests for the git protocol, client side
-# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2009 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/tests/test_config.py

@@ -1,5 +1,5 @@
 # test_config.py -- Tests for reading and writing configuration files
-# Copyright (C) 2011 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2011 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/tests/test_fastexport.py

@@ -1,5 +1,5 @@
 # test_fastexport.py -- Fast export/import functionality
-# Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2010 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/tests/test_index.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # test_index.py -- Tests for the git index
 # encoding: utf-8
-# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
@@ -30,6 +30,7 @@ import stat
 import struct
 import sys
 import tempfile
+import warnings
 
 from dulwich.index import (
     Index,
@@ -62,6 +63,28 @@ from dulwich.tests import (
     TestCase,
     skipIf,
     )
+from dulwich.tests.utils import (
+    setup_warning_catcher,
+    )
+
+
+def can_symlink():
+    """Return whether running process can create symlinks."""
+    if sys.platform != 'win32':
+        # Platforms other than Windows should allow symlinks without issues.
+        return True
+
+    if not hasattr(os, 'symlink'):
+        # Older Python versions do not have `os.symlink` on Windows.
+        return False
+
+    test_source = tempfile.mkdtemp()
+    test_target = test_source + 'can_symlink'
+    try:
+        os.symlink(test_source, test_target)
+    except OSError:
+        return False
+    return True
 
 
 class IndexTestCase(TestCase):
@@ -80,6 +103,32 @@ class SimpleIndexTestCase(IndexTestCase):
     def test_iter(self):
         self.assertEqual([b'bla'], list(self.get_simple_index("index")))
 
+    def test_iterobjects(self):
+        self.assertEqual(
+                [(b'bla', b'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391', 33188)],
+                list(self.get_simple_index("index").iterobjects()))
+
+    def test_iterblobs(self):
+        warnings.simplefilter("always", UserWarning)
+        self.addCleanup(warnings.resetwarnings)
+        warnings_list, restore_warnings = setup_warning_catcher()
+        self.addCleanup(restore_warnings)
+
+        self.assertEqual(
+                [(b'bla', b'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391', 33188)],
+                list(self.get_simple_index("index").iterblobs()))
+
+        expected_warning = PendingDeprecationWarning(
+            'Use iterobjects() instead.')
+        for w in warnings_list:
+            if (type(w) == type(expected_warning) and
+                    w.args == expected_warning.args):
+                break
+        else:
+            raise AssertionError(
+                'Expected warning %r not in %r' %
+                (expected_warning, warnings_list))
+
     def test_getitem(self):
         self.assertEqual(
                 ((1230680220, 0), (1230680220, 0), 2050, 3761020,
@@ -414,7 +463,7 @@ class BuildIndexTests(TestCase):
             with open(filea_path, 'rb') as fh:
                 self.assertEqual(b'file a', fh.read())
 
-    @skipIf(not getattr(os, 'symlink', None), 'Requires symlink support')
+    @skipIf(not can_symlink(), 'Requires symlink support')
     def test_symlink(self):
         repo_dir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, repo_dir)

dulwich/tests/test_object_store.py

@@ -1,5 +1,5 @@
 # test_object_store.py -- tests for object_store.py
-# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/tests/test_objects.py

@@ -887,6 +887,7 @@ class TreeTests(ShaFileCheckTests):
         self.assertCheckFails(t, b'100644 .\0' + sha)
         self.assertCheckFails(t, b'100644 a/a\0' + sha)
         self.assertCheckFails(t, b'100644 ..\0' + sha)
+        self.assertCheckFails(t, b'100644 .git\0' + sha)
 
         # modes
         self.assertCheckSucceeds(t, b'100644 a\0' + sha)

dulwich/tests/test_objectspec.py

@@ -1,5 +1,5 @@
 # test_objectspec.py -- tests for objectspec.py
-# Copyright (C) 2014 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2014 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/tests/test_pack.py

@@ -1,6 +1,6 @@
 # test_pack.py -- Tests for the handling of git packs.
 # Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
-# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/tests/test_patch.py

@@ -1,5 +1,5 @@
 # test_patch.py -- tests for patch.py
-# Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2010 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/tests/test_porcelain.py

@@ -1,5 +1,5 @@
 # test_porcelain.py -- porcelain tests
-# Copyright (C) 2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
@@ -134,6 +134,7 @@ class CloneTests(PorcelainTestCase):
         self.addCleanup(shutil.rmtree, target_path)
         r = porcelain.clone(self.repo.path, target_path,
                             checkout=False, errstream=errstream)
+        self.addCleanup(r.close)
         self.assertEqual(r.path, target_path)
         target_repo = Repo(target_path)
         self.assertEqual(0, len(target_repo.open_index()))
@@ -1093,6 +1094,10 @@ class FetchTests(PorcelainTestCase):
         porcelain.fetch(target_path, self.repo.path,
                         outstream=outstream, errstream=errstream)
 
+        # Assert that fetch updated the local image of the remote
+        self.assert_correct_remote_refs(
+            target_repo.get_refs(), self.repo.get_refs())
+
         # Check the target repo for pushed changes
         with Repo(target_path) as r:
             self.assertTrue(self.repo[b'HEAD'].id in r)
@@ -1134,12 +1139,32 @@ class FetchTests(PorcelainTestCase):
         porcelain.fetch(target_path, self.repo.path, remote_name=remote_name,
                         outstream=outstream, errstream=errstream)
 
+        # Assert that fetch updated the local image of the remote
+        self.assert_correct_remote_refs(
+            target_repo.get_refs(), self.repo.get_refs())
+
         # Check the target repo for pushed changes, as well as updates
         # for the refs
         with Repo(target_path) as r:
             self.assertTrue(self.repo[b'HEAD'].id in r)
             self.assertNotEqual(self.repo.get_refs(), target_refs)
 
+    def assert_correct_remote_refs(
+            self, local_refs, remote_refs, remote_name=b'origin'):
+        """Assert that known remote refs corresponds to actual remote refs."""
+        local_ref_prefix = b'refs/heads'
+        remote_ref_prefix = b'refs/remotes/' + remote_name
+
+        locally_known_remote_refs = {
+            k[len(remote_ref_prefix) + 1:]: v for k, v in local_refs.items()
+            if k.startswith(remote_ref_prefix)}
+
+        normalized_remote_refs = {
+            k[len(local_ref_prefix) + 1:]: v for k, v in remote_refs.items()
+            if k.startswith(local_ref_prefix)}
+
+        self.assertEqual(locally_known_remote_refs, normalized_remote_refs)
+
 
 class RepackTests(PorcelainTestCase):
 
@@ -1181,6 +1206,30 @@ class LsTreeTests(PorcelainTestCase):
                 f.getvalue(),
                 '100644 blob 8b82634d7eae019850bb883f06abf428c58bc9aa\tfoo\n')
 
+    def test_recursive(self):
+        # Create a directory then write a dummy file in it
+        dirpath = os.path.join(self.repo.path, 'adir')
+        filepath = os.path.join(dirpath, 'afile')
+        os.mkdir(dirpath)
+        with open(filepath, 'w') as f:
+            f.write('origstuff')
+        porcelain.add(repo=self.repo.path, paths=[filepath])
+        porcelain.commit(repo=self.repo.path, message=b'test status',
+                         author=b'author <email>',
+                         committer=b'committer <email>')
+        f = StringIO()
+        porcelain.ls_tree(self.repo, b"HEAD", outstream=f)
+        self.assertEqual(
+                f.getvalue(),
+                '40000 tree b145cc69a5e17693e24d8a7be0016ed8075de66d\tadir\n')
+        f = StringIO()
+        porcelain.ls_tree(self.repo, b"HEAD", outstream=f, recursive=True)
+        self.assertEqual(
+                f.getvalue(),
+                '40000 tree b145cc69a5e17693e24d8a7be0016ed8075de66d\tadir\n'
+                '100644 blob 8b82634d7eae019850bb883f06abf428c58bc9aa\tadir'
+                '/afile\n')
+
 
 class LsRemoteTests(PorcelainTestCase):
 
@@ -1250,7 +1299,7 @@ class UpdateHeadTests(PorcelainTestCase):
         porcelain.update_head(self.repo, "blah")
         self.assertEqual(c1.id, self.repo.head())
         self.assertEqual(b'ref: refs/heads/blah',
-                         self.repo.refs.read_ref('HEAD'))
+                         self.repo.refs.read_ref(b'HEAD'))
 
     def test_set_to_branch_detached(self):
         [c1] = build_commit_graph(self.repo.object_store, [[1]])
@@ -1292,3 +1341,22 @@ Jelmer Vernooij <jelmer@debian.org>
             b'Jelmer Vernooij <jelmer@debian.org>',
             porcelain.check_mailmap(
                 self.repo, b'Jelmer Vernooij <jelmer@samba.org>'))
+
+
+class FsckTests(PorcelainTestCase):
+
+    def test_none(self):
+        self.assertEqual(
+                [],
+                list(porcelain.fsck(self.repo)))
+
+    def test_git_dir(self):
+        obj = Tree()
+        a = Blob()
+        a.data = b"foo"
+        obj.add(b".git", 0o100644, a.id)
+        self.repo.object_store.add_objects(
+            [(a, None), (obj, None)])
+        self.assertEqual(
+                [(obj.id, 'invalid name .git')],
+                [(sha, str(e)) for (sha, e) in porcelain.fsck(self.repo)])

dulwich/tests/test_protocol.py

@@ -1,5 +1,5 @@
 # test_protocol.py -- Tests for the git protocol
-# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2009 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/tests/test_reflog.py

@@ -1,6 +1,6 @@
 # test_reflog.py -- tests for reflog.py
 # encoding: utf-8
-# Copyright (C) 2015 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2015 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

dulwich/tests/test_refs.py

@@ -1,6 +1,6 @@
 # test_refs.py -- tests for refs.py
 # encoding: utf-8
-# Copyright (C) 2013 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2013 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
@@ -335,7 +335,7 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
 
     def test_setitem(self):
         RefsContainerTests.test_setitem(self)
-        f = open(os.path.join(self._refs.path, 'refs', 'some', 'ref'), 'rb')
+        f = open(os.path.join(self._refs.path, b'refs', b'some', b'ref'), 'rb')
         self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
                          f.read()[:40])
         f.close()
@@ -346,13 +346,13 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
         self.assertEqual(ones, self._refs[b'HEAD'])
 
         # ensure HEAD was not modified
-        f = open(os.path.join(self._refs.path, 'HEAD'), 'rb')
+        f = open(os.path.join(self._refs.path, b'HEAD'), 'rb')
         v = next(iter(f)).rstrip(b'\n\r')
         f.close()
         self.assertEqual(b'ref: refs/heads/master', v)
 
         # ensure the symbolic link was written through
-        f = open(os.path.join(self._refs.path, 'refs', 'heads', 'master'),
+        f = open(os.path.join(self._refs.path, b'refs', b'heads', b'master'),
                  'rb')
         self.assertEqual(ones, f.read()[:40])
         f.close()
@@ -365,9 +365,9 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
 
         # ensure lockfile was deleted
         self.assertFalse(os.path.exists(
-            os.path.join(self._refs.path, 'refs', 'heads', 'master.lock')))
+            os.path.join(self._refs.path, b'refs', b'heads', b'master.lock')))
         self.assertFalse(os.path.exists(
-            os.path.join(self._refs.path, 'HEAD.lock')))
+            os.path.join(self._refs.path, b'HEAD.lock')))
 
     def test_add_if_new_packed(self):
         # don't overwrite packed ref
@@ -406,7 +406,7 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
 
     def test_delitem(self):
         RefsContainerTests.test_delitem(self)
-        ref_file = os.path.join(self._refs.path, 'refs', 'heads', 'master')
+        ref_file = os.path.join(self._refs.path, b'refs', b'heads', b'master')
         self.assertFalse(os.path.exists(ref_file))
         self.assertFalse(b'refs/heads/master' in self._refs.get_packed_refs())
 
@@ -417,7 +417,8 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
         self.assertRaises(KeyError, lambda: self._refs[b'HEAD'])
         self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
                          self._refs[b'refs/heads/master'])
-        self.assertFalse(os.path.exists(os.path.join(self._refs.path, 'HEAD')))
+        self.assertFalse(
+            os.path.exists(os.path.join(self._refs.path, b'HEAD')))
 
     def test_remove_if_equals_symref(self):
         # HEAD is a symref, so shouldn't equal its dereferenced value
@@ -433,9 +434,9 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
                          self._refs.read_loose_ref(b'HEAD'))
 
         self.assertFalse(os.path.exists(
-            os.path.join(self._refs.path, 'refs', 'heads', 'master.lock')))
+            os.path.join(self._refs.path, b'refs', b'heads', b'master.lock')))
         self.assertFalse(os.path.exists(
-            os.path.join(self._refs.path, 'HEAD.lock')))
+            os.path.join(self._refs.path, b'HEAD.lock')))
 
     def test_remove_packed_without_peeled(self):
         refs_file = os.path.join(self._repo.path, 'packed-refs')
@@ -475,7 +476,9 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
         except UnicodeEncodeError:
             raise SkipTest(
                     "filesystem encoding doesn't support special character")
-        p = os.path.join(self._repo.path, 'refs', 'tags', u'schön')
+        p = os.path.join(
+                self._repo.path.encode(sys.getfilesystemencoding()),
+                encoded_ref)
         with open(p, 'w') as f:
             f.write('00' * 20)
 
@@ -485,6 +488,28 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
 
         self.assertEqual(expected_refs, self._repo.get_refs())
 
+    def test_cyrillic(self):
+        # reported in https://github.com/dulwich/dulwich/issues/608
+        name = b'\xcd\xee\xe2\xe0\xff\xe2\xe5\xf2\xea\xe01'
+        encoded_ref = b'refs/heads/' + name
+        with open(os.path.join(
+            self._repo.path.encode(
+                sys.getfilesystemencoding()), encoded_ref), 'w') as f:
+            f.write('00' * 20)
+
+        expected_refs = set(_TEST_REFS.keys())
+        expected_refs.add(encoded_ref)
+
+        self.assertEqual(expected_refs,
+                         set(self._repo.refs.allkeys()))
+        self.assertEqual({r[len(b'refs/'):] for r in expected_refs
+                          if r.startswith(b'refs/')},
+                         set(self._repo.refs.subkeys(b'refs/')))
+        expected_refs.remove(b'refs/heads/loop')
+        expected_refs.add(b'HEAD')
+        self.assertEqual(expected_refs,
+                         set(self._repo.get_refs().keys()))
+
 
 _TEST_REFS_SERIALIZED = (
     b'42d06bd4b77fed026b154d16493e5deab78f02ec\t'

dulwich/tests/test_repository.py

@@ -325,6 +325,20 @@ class RepositoryRootTests(TestCase):
         self.addCleanup(shutil.rmtree, tmp_dir)
         r.clone(tmp_dir, mkdir=False, bare=True)
 
+    def test_clone_bare(self):
+        r = self.open_repo('a.git')
+        tmp_dir = self.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmp_dir)
+        t = r.clone(tmp_dir, mkdir=False)
+        t.close()
+
+    def test_clone_checkout_and_bare(self):
+        r = self.open_repo('a.git')
+        tmp_dir = self.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmp_dir)
+        self.assertRaises(ValueError, r.clone, tmp_dir, mkdir=False,
+                          checkout=True, bare=True)
+
     def test_merge_history(self):
         r = self.open_repo('simple_merge.git')
         shas = [e.commit.id for e in r.get_walker()]
@@ -648,6 +662,13 @@ class BuildRepoRootTests(TestCase):
         self.assertEqual([], r[commit_sha].parents)
         self._root_commit = commit_sha
 
+    def test_shallow(self):
+        self.assertEqual(set(), self._repo.get_shallow())
+        with open(os.path.join(self._repo.path, '.git', 'shallow'), 'wb') as f:
+            f.write(b'a90fa2d900a17e99b433217e988c4eb4a2e9a097\n')
+        self.assertEqual({b'a90fa2d900a17e99b433217e988c4eb4a2e9a097'},
+                         self._repo.get_shallow())
+
     def test_build_repo(self):
         r = self._repo
         self.assertEqual(b'ref: refs/heads/master', r.refs.read_ref(b'HEAD'))

devscripts/PREAMBLE.py → dulwich/tests/test_stash.py

@@ -1,3 +1,6 @@
+# test_stash.py -- tests for stash
+# Copyright (C) 2018 Jelmer Vernooij <jelmer@jelmer.uk>
+#
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0
 # or (at your option) any later version. You can redistribute it and/or
@@ -14,3 +17,19 @@
 # and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
 # License, Version 2.0.
 #
+
+"""Tests for stashes."""
+
+from . import TestCase
+
+from ..repo import MemoryRepo
+from ..stash import Stash
+
+
+class StashTests(TestCase):
+    """Tests for stash."""
+
+    def test_obtain(self):
+        repo = MemoryRepo()
+        stash = Stash.from_repo(repo)
+        self.assertIsInstance(stash, Stash)

dulwich/web.py

@@ -1,6 +1,6 @@
 # web.py -- WSGI smart-http server
 # Copyright (C) 2010 Google, Inc.
-# Copyright (C) 2012 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2012 Jelmer Vernooij <jelmer@jelmer.uk>
 #
 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
 # General Public License as public by the Free Software Foundation; version 2.0

examples/memoryrepo.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python
+# This script creates a clone of a remote repository in local memory,
+# then adds a single file and pushes the result back.
+#
+# Example usage:
+#  python examples/memoryrepo.py git+ssh://github.com/jelmer/testrepo
+
+import stat
+import sys
+
+from dulwich import porcelain
+from dulwich.objects import Blob
+from dulwich.repo import MemoryRepo
+
+local_repo = MemoryRepo()
+local_repo.refs.set_symbolic_ref(b'HEAD', b'refs/heads/master')
+print(local_repo.refs.as_dict())
+
+porcelain.fetch(local_repo, sys.argv[1])
+local_repo['refs/heads/master'] = local_repo['refs/remotes/origin/master']
+
+last_tree = local_repo[local_repo['HEAD'].tree]
+new_blob = Blob.from_string(b'Some contents')
+local_repo.object_store.add_object(new_blob)
+last_tree.add(b'test', stat.S_IFREG, new_blob.id)
+local_repo.object_store.add_object(last_tree)
+
+local_repo.do_commit(
+    message=b'Add a file called \'test\'',
+    ref=b'refs/heads/master',
+    tree=last_tree.id)
+
+porcelain.push(local_repo, sys.argv[1], 'master')

requirements.txt

@@ -1 +0,0 @@
-urllib3[secure]==1.22

setup.cfg

@@ -1,3 +1,27 @@
+[metadata]
+name = dulwich
+author = Jelmer Vernooij
+author_email = jelmer@jelmer.uk
+home-page = https://www.dulwich.io/
+description-file = README.md
+summary = Python Git Library
+classifiers = 
+	Development Status :: 4 - Beta
+	License :: OSI Approved :: Apache Software License
+	Programming Language :: Python :: 2.7
+	Programming Language :: Python :: 3.3
+	Programming Language :: Python :: 3.4
+	Programming Language :: Python :: 3.5
+	Programming Language :: Python :: 3.6
+	Programming Language :: Python :: Implementation :: CPython
+	Programming Language :: Python :: Implementation :: PyPy
+	Operating System :: POSIX
+	Operating System :: Microsoft :: Windows
+	Topic :: Software Development :: Version Control
+keyword = git, vcs
+project_urls = 
+	Bug Tracker = https://github.com/dulwich/dulwich/issues
+
 [build_ext]
 
 [egg_info]

setup.py

@@ -14,7 +14,7 @@ from distutils.core import Distribution
 import os
 import sys
 
-dulwich_version_string = '0.19.1'
+dulwich_version_string = '0.19.3'
 
 include_dirs = []
 # Windows MSVC support
@@ -84,47 +84,13 @@ if has_setuptools:
     setup_kwargs['tests_require'] = tests_require
 
 
-if sys.platform == 'win32':
-    # Win32 setup breaks with non-ascii characters.
-    author = "Jelmer Vernooij"
-else:
-    author = "Jelmer Vernooij"
-
 setup(name='dulwich',
-      description='Python Git Library',
-      keywords='git',
       version=dulwich_version_string,
-      url='https://www.dulwich.io/',
       license='Apachev2 or later or GPLv2',
-      author_email='jelmer@jelmer.uk',
-      long_description="""
-      Python implementation of the Git file formats and protocols,
-      without the need to have git installed.
-
-      All functionality is available in pure Python. Optional
-      C extensions can be built for improved performance.
-
-      The project is named after the part of London that Mr. and Mrs. Git live
-      in in the particular Monty Python sketch.
-      """,
       packages=['dulwich', 'dulwich.tests', 'dulwich.tests.compat',
                 'dulwich.contrib'],
       package_data={'': ['../docs/tutorial/*.txt']},
       scripts=['bin/dulwich', 'bin/dul-receive-pack', 'bin/dul-upload-pack'],
-      classifiers=[
-          'Development Status :: 4 - Beta',
-          'License :: OSI Approved :: Apache Software License',
-          'Programming Language :: Python :: 2.7',
-          'Programming Language :: Python :: 3.3',
-          'Programming Language :: Python :: 3.4',
-          'Programming Language :: Python :: 3.5',
-          'Programming Language :: Python :: 3.6',
-          'Programming Language :: Python :: Implementation :: CPython',
-          'Programming Language :: Python :: Implementation :: PyPy',
-          'Operating System :: POSIX',
-          'Operating System :: Microsoft :: Windows',
-          'Topic :: Software Development :: Version Control',
-      ],
       ext_modules=ext_modules,
       distclass=DulwichDistribution,
       **setup_kwargs