New upstream version 0.17.1

CONTRIBUTING.md

@@ -0,0 +1,51 @@
+All functionality should be available in pure Python. Optional C
+implementations may be written for performance reasons, but should never
+replace the Python implementation. The C implementations should follow the
+kernel/git coding style.
+
+Where possible include updates to NEWS along with your improvements.
+
+New functionality and bug fixes should be accompanied by matching unit tests.
+
+Coding style
+------------
+Where possible, please follow PEP8 with regard to coding style.
+
+Furthermore, triple-quotes should always be """, single quotes are ' unless
+using " would result in less escaping within the string.
+
+Public methods, functions and classes should all have doc strings. Please use
+epydoc style docstrings to document parameters and return values.
+You can generate the documentation by running "make doc".
+
+Running the tests
+-----------------
+To run the testsuite, you should be able to simply run "make check". This
+will run the tests using unittest.
+
+ $ make check
+
+Tox configuration is also present as well as a Travis configuration file.
+
+String Types
+------------
+Like Linux, Git treats filenames as arbitrary bytestrings. There is no prescribed
+encoding for these strings, and although it is fairly common to use UTF-8, any
+raw byte strings are supported.
+
+For this reason, Dulwich internally treats git-based filenames as bytestrings. It is up
+to the Dulwich API user to encode and decode them if necessary.
+
+* git-repository related filenames: bytes
+* object sha1 digests (20 bytes long): bytes
+* object sha1 hexdigests (40 bytes long): str (bytestrings on python2, strings on python3)
+
+Merge requests
+--------------
+Please either send pull requests to the maintainer (jelmer@jelmer.uk) or create new pull
+requests on GitHub.
+
+Licensing
+---------
+All contributions should be made under the same license that Dulwich itself comes under:
+both Apache License, version 2.0 or later and GNU General Public License, version 2.0 or later.

MANIFEST.in

@@ -1,11 +1,10 @@
 include NEWS
 include AUTHORS
 include README.md
-include README.swift
+include README.swift.md
 include Makefile
 include COPYING
-include HACKING
+include CONTRIBUTING.md
-include CONTRIBUTING
 include TODO
 include setup.cfg
 include dulwich/stdint.h
@@ -17,4 +16,3 @@ include dulwich.cfg
 include appveyor.yml
 include .testr.conf
 include .travis.yml
-include relicensing-apachev2.txt

Makefile

@@ -5,6 +5,7 @@ SETUP = $(PYTHON) setup.py
 PYDOCTOR ?= pydoctor
 TESTRUNNER ?= unittest
 RUNTEST = PYTHONHASHSEED=random PYTHONPATH=.:$(PYTHONPATH) $(PYTHON) -m $(TESTRUNNER) $(TEST_OPTIONS)
+COVERAGE = python3-coverage
 
 DESTDIR=/
 
@@ -58,3 +59,9 @@ pep8:
 
 before-push: check
 	git diff origin/master | $(PEP8) --diff
+
+coverage:
+	$(COVERAGE) run --source=dulwich -m unittest dulwich.tests.test_suite dulwich.contrib.test_suite
+
+coverage-html: coverage
+	$(COVERAGE) html

NEWS

@@ -1,9 +1,63 @@
+0.17.1	2017-03-01
+
+ IMPROVEMENTS
+
+ * Add basic 'dulwich pull' command. (Jelmer Vernooij)
+
+ BUG FIXES
+
+ * Cope with existing submodules during pull.
+   (Jelmer Vernooij, #505)
+
+0.17.0	2017-03-01
+
+ TEST FIXES
+
+ * Skip test that requires sync to synchronize filesystems if os.sync is
+   not available. (Koen Martens)
+
+ IMPROVEMENTS
+
+ * Implement MemoryRepo.{set_description,get_description}.
+   (Jelmer Vernooij)
+
+ * Raise exception in Repo.stage() when absolute paths are
+   passed in. Allow passing in relative paths to
+   porcelain.add().(Jelmer Vernooij)
+
+ BUG FIXES
+
+ * Handle multi-line quoted values in config files.
+   (Jelmer Vernooij, #495)
+
+ * Allow porcelain.clone of repository without HEAD.
+   (Jelmer Vernooij, #501)
+
+ * Support passing tag ids to Walker()'s include argument.
+   (Jelmer Vernooij)
+
+ * Don't strip trailing newlines from extra headers.
+   (Nicolas Dandrimont)
+
+ * Set bufsize=0 for subprocess interaction with SSH client.
+   Fixes hangs on Python 3. (René Stern, #434)
+
+ * Don't drop first slash for SSH paths, except for those
+   starting with "~". (Jelmer Vernooij, René Stern, #463)
+
+ * Properly log off after retrieving just refs.
+   (Jelmer Vernooij)
+
 0.16.3	2016-01-14
 
  TEST FIXES
 
   * Remove racy check that relies on clock time changing between writes.
-   (Jelmer Vernooij)
+    (Jelmer Vernooij)
+
+ IMPROVEMENTS
+
+  * Add porcelain.remote_add. (Jelmer Vernooij)
 
 0.16.2	2016-01-14
 

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: dulwich
-Version: 0.16.3
+Version: 0.17.1
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij

README.md

@@ -5,9 +5,9 @@ This is the Dulwich project.
 It aims to provide an interface to git repos (both local and remote) that
 doesn't call out to git directly but instead uses pure Python.
 
-Main website: https://www.dulwich.io/
+**Main website**: [www.dulwich.io](https://www.dulwich.io/)
 
-License: Apache License, version 2 or GNU General Public License, version 2 or later.
+**License**: Apache License, version 2 or GNU General Public License, version 2 or later.
 
 The project is named after the part of London that Mr. and Mrs. Git live in
 in the particular Monty Python sketch.
@@ -30,19 +30,19 @@ or if you are installing from pip::
 Further documentation
 ---------------------
 
-The dulwich documentation can be found in doc/ and on the web:
+The dulwich documentation can be found in doc/ and
+[on the web](https://www.dulwich.io/docs/).
 
-https://www.dulwich.io/docs/
+The API reference can be generated using pydoctor, by running "make pydoctor",
-
+or [on the web](https://www.dulwich.io/apidocs).
-The API reference can be generated using pydoctor, by running "make pydoctor", or on the web:
-
-https://www.dulwich.io/apidocs
 
 Help
 ----
 
-There is a #dulwich IRC channel on Freenode, and a dulwich mailing list at
+There is a *#dulwich* IRC channel on the [Freenode](https://www.freenode.net/), and
-https://launchpad.net/~dulwich-users.
+[dulwich-announce](https://groups.google.com/forum/#!forum/dulwich-announce)
+and [dulwich-discuss](https://groups.google.com/forum/#!forum/dulwich-discuss)
+mailing lists.
 
 Supported versions of Python
 ----------------------------

README.swift.md

@@ -0,0 +1,133 @@
+Openstack Swift as backend for Dulwich
+======================================
+Fabien Boucher <fabien.boucher@enovance.com>
+
+The module dulwich/contrib/swift.py implements dulwich.repo.BaseRepo
+in order to being compatible with Openstack Swift.
+We can then use Dulwich as server (Git server) and instead of using
+a regular POSIX file system to store repository objects we use the
+object storage Swift via its own API.
+
+c Git client <---> Dulwich server <---> Openstack Swift API
+
+This implementation is still a work in progress and we can say that
+is a Beta version so you need to be prepared to find bugs.
+
+Configuration file
+------------------
+
+We need to provide some configuration values in order to let Dulwich
+talk and authenticate against Swift. The following config file must
+be used as template:
+
+    [swift]
+    # Authentication URL (Keystone or Swift)
+    auth_url = http://127.0.0.1:5000/v2.0
+    # Authentication version to use
+    auth_ver = 2
+    # The tenant and username separated by a semicolon
+    username = admin;admin
+    # The user password
+    password = pass
+    # The Object storage region to use (auth v2) (Default RegionOne)
+    region_name = RegionOne
+    # The Object storage endpoint URL to use (auth v2) (Default internalURL)
+    endpoint_type = internalURL
+    # Concurrency to use for parallel tasks (Default 10)
+    concurrency = 10
+    # Size of the HTTP pool (Default 10)
+    http_pool_length = 10
+    # Timeout delay for HTTP connections (Default 20)
+    http_timeout = 20
+    # Chunk size to read from pack (Bytes) (Default 12228)
+    chunk_length = 12228
+    # Cache size (MBytes) (Default 20)
+    cache_length = 20
+
+
+Note that for now we use the same tenant to perform the requests
+against Swift. Therefor there is only one Swift account used
+for storing repositories. Each repository will be contained in
+a Swift container.
+
+How to start unittest
+---------------------
+
+There is no need to have a Swift cluster running to run the unitests.
+Just run the following command in the Dulwich source directory:
+
+    $ PYTHONPATH=. python -m dulwich.contrib.test_swift
+
+How to start functional tests
+-----------------------------
+
+We provide some basic tests to perform smoke tests against a real Swift
+cluster. To run those functional tests you need a properly configured
+configuration file. The tests can be run as follow:
+
+    $ DULWICH_SWIFT_CFG=/etc/swift-dul.conf PYTHONPATH=. python -m dulwich.contrib.test_swift_smoke
+
+How to install
+--------------
+
+Install the Dulwich library via the setup.py. The dependencies will be
+automatically retrieved from pypi:
+
+    $ python ./setup.py install
+
+How to run the server
+---------------------
+
+Start the server using the following command:
+
+    $ python -m dulwich.contrib.swift daemon -c /etc/swift-dul.conf -l 127.0.0.1
+
+Note that a lot of request will be performed against the Swift
+cluster so it is better to start the Dulwich server as close
+as possible of the Swift proxy. The best solution is to run
+the server on the Swift proxy node to reduce the latency.
+
+How to use
+----------
+
+Once you have validated that the functional tests is working as expected and
+the server is running we can init a bare repository. Run this
+command with the name of the repository to create:
+
+    $ python -m dulwich.contrib.swift init -c /etc/swift-dul.conf edeploy
+
+The repository name will be the container that will contain all the Git
+objects for the repository. Then standard c Git client can be used to
+perform operations againt this repository.
+
+As an example we can clone the previously empty bare repository:
+
+    $ git clone git://localhost/edeploy
+
+Then push an existing project in it:
+
+    $ git clone https://github.com/enovance/edeploy.git edeployclone
+    $ cd edeployclone
+    $ git remote add alt git://localhost/edeploy
+    $ git push alt master
+    $ git ls-remote alt
+    9dc50a9a9bff1e232a74e365707f22a62492183e        HEAD
+    9dc50a9a9bff1e232a74e365707f22a62492183e        refs/heads/master
+
+The other Git commands can be used the way you do usually against
+a regular repository.
+
+Note the daemon subcommands starts a Git server listening for the
+Git protocol. Therefor there is no authentication or encryption
+at all between the cGIT client and the GIT server (Dulwich).
+
+Note on the .info file for pack object
+--------------------------------------
+
+The Swift interface of Dulwich relies only on the pack format
+to store Git objects. Instead of using only an index (pack-sha.idx)
+along with the pack, we add a second file (pack-sha.info). This file
+is automatically created when a client pushes some references on the
+repository. The purpose of this file is to speed up pack creation
+server side when a client fetches some references. Currently this
+.info format is not optimized and may change in future.

bin/dulwich

@@ -48,372 +48,480 @@ from dulwich.patch import write_tree_diff
 from dulwich.repo import Repo
 
 
-def cmd_archive(args):
+class Command(object):
-    opts, args = getopt(args, "", [])
+    """A Dulwich subcommand."""
-    client, path = get_transport_and_path(args.pop(0))
+
-    location = args.pop(0)
+    def run(self, args):
-    committish = args.pop(0)
+        """Run the command."""
-    porcelain.archive(location, committish, outstream=sys.stdout,
+        raise NotImplementedError(self.run)
-        errstream=sys.stderr)
+
-
+
-
+class cmd_archive(Command):
-def cmd_add(args):
+
-    opts, args = getopt(args, "", [])
+    def run(self, args):
-
+        opts, args = getopt(args, "", [])
-    porcelain.add(".", paths=args)
+        client, path = get_transport_and_path(args.pop(0))
-
+        location = args.pop(0)
-
+        committish = args.pop(0)
-def cmd_rm(args):
+        porcelain.archive(location, committish, outstream=sys.stdout,
-    opts, args = getopt(args, "", [])
+            errstream=sys.stderr)
-
+
-    porcelain.rm(".", paths=args)
+
-
+class cmd_add(Command):
-
+
-def cmd_fetch_pack(args):
+    def run(self, args):
-    opts, args = getopt(args, "", ["all"])
+        opts, args = getopt(args, "", [])
-    opts = dict(opts)
+
-    client, path = get_transport_and_path(args.pop(0))
+        porcelain.add(".", paths=args)
-    r = Repo(".")
+
-    if "--all" in opts:
+
-        determine_wants = r.object_store.determine_wants_all
+class cmd_rm(Command):
-    else:
+
-        determine_wants = lambda x: [y for y in args if not y in r.object_store]
+    def run(self, args):
-    client.fetch(path, r, determine_wants)
+        opts, args = getopt(args, "", [])
-
+
-
+        porcelain.rm(".", paths=args)
-def cmd_fetch(args):
+
-    opts, args = getopt(args, "", [])
+
-    opts = dict(opts)
+class cmd_fetch_pack(Command):
-    client, path = get_transport_and_path(args.pop(0))
+
-    r = Repo(".")
+    def run(self, args):
-    if "--all" in opts:
+        opts, args = getopt(args, "", ["all"])
-        determine_wants = r.object_store.determine_wants_all
+        opts = dict(opts)
-    refs = client.fetch(path, r, progress=sys.stdout.write)
+        client, path = get_transport_and_path(args.pop(0))
-    print("Remote refs:")
+        r = Repo(".")
-    for item in refs.items():
+        if "--all" in opts:
-        print("%s -> %s" % item)
+            determine_wants = r.object_store.determine_wants_all
-
+        else:
-
+            determine_wants = lambda x: [y for y in args if not y in r.object_store]
-def cmd_log(args):
+        client.fetch(path, r, determine_wants)
-    parser = optparse.OptionParser()
+
-    parser.add_option("--reverse", dest="reverse", action="store_true",
+
-                      help="Reverse order in which entries are printed")
+class cmd_fetch(Command):
-    parser.add_option("--name-status", dest="name_status", action="store_true",
+
-                      help="Print name/status for each changed file")
+    def run(self, args):
-    options, args = parser.parse_args(args)
+        opts, args = getopt(args, "", [])
-
+        opts = dict(opts)
-    porcelain.log(".", paths=args, reverse=options.reverse,
+        client, path = get_transport_and_path(args.pop(0))
-                  name_status=options.name_status,
+        r = Repo(".")
-                  outstream=sys.stdout)
+        if "--all" in opts:
-
+            determine_wants = r.object_store.determine_wants_all
-
+        refs = client.fetch(path, r, progress=sys.stdout.write)
-def cmd_diff(args):
+        print("Remote refs:")
-    opts, args = getopt(args, "", [])
+        for item in refs.items():
-
+            print("%s -> %s" % item)
-    if args == []:
+
-        print("Usage: dulwich diff COMMITID")
+
-        sys.exit(1)
+class cmd_log(Command):
-
+
-    r = Repo(".")
+    def run(self, args):
-    commit_id = args[0]
+        parser = optparse.OptionParser()
-    commit = r[commit_id]
+        parser.add_option("--reverse", dest="reverse", action="store_true",
-    parent_commit = r[commit.parents[0]]
+                          help="Reverse order in which entries are printed")
-    write_tree_diff(sys.stdout, r.object_store, parent_commit.tree, commit.tree)
+        parser.add_option("--name-status", dest="name_status", action="store_true",
-
+                          help="Print name/status for each changed file")
-
+        options, args = parser.parse_args(args)
-def cmd_dump_pack(args):
+
-    opts, args = getopt(args, "", [])
+        porcelain.log(".", paths=args, reverse=options.reverse,
-
+                      name_status=options.name_status,
-    if args == []:
+                      outstream=sys.stdout)
-        print("Usage: dulwich dump-pack FILENAME")
+
-        sys.exit(1)
+
-
+class cmd_diff(Command):
-    basename, _ = os.path.splitext(args[0])
+
-    x = Pack(basename)
+    def run(self, args):
-    print("Object names checksum: %s" % x.name())
+        opts, args = getopt(args, "", [])
-    print("Checksum: %s" % sha_to_hex(x.get_stored_checksum()))
+
-    if not x.check():
+        if args == []:
-        print("CHECKSUM DOES NOT MATCH")
+            print("Usage: dulwich diff COMMITID")
-    print("Length: %d" % len(x))
+            sys.exit(1)
-    for name in x:
+
-        try:
+        r = Repo(".")
-            print("\t%s" % x[name])
+        commit_id = args[0]
-        except KeyError as k:
+        commit = r[commit_id]
-            print("\t%s: Unable to resolve base %s" % (name, k))
+        parent_commit = r[commit.parents[0]]
-        except ApplyDeltaError as e:
+        write_tree_diff(sys.stdout, r.object_store, parent_commit.tree, commit.tree)
-            print("\t%s: Unable to apply delta: %r" % (name, e))
+
+
+class cmd_dump_pack(Command):
+
+    def run(self, args):
+        opts, args = getopt(args, "", [])
 
+        if args == []:
+            print("Usage: dulwich dump-pack FILENAME")
+            sys.exit(1)
 
-def cmd_dump_index(args):
+        basename, _ = os.path.splitext(args[0])
-    opts, args = getopt(args, "", [])
+        x = Pack(basename)
+        print("Object names checksum: %s" % x.name())
+        print("Checksum: %s" % sha_to_hex(x.get_stored_checksum()))
+        if not x.check():
+            print("CHECKSUM DOES NOT MATCH")
+        print("Length: %d" % len(x))
+        for name in x:
+            try:
+                print("\t%s" % x[name])
+            except KeyError as k:
+                print("\t%s: Unable to resolve base %s" % (name, k))
+            except ApplyDeltaError as e:
+                print("\t%s: Unable to apply delta: %r" % (name, e))
 
-    if args == []:
-        print("Usage: dulwich dump-index FILENAME")
-        sys.exit(1)
 
-    filename = args[0]
+class cmd_dump_index(Command):
-    idx = Index(filename)
 
-    for o in idx:
+    def run(self, args):
-        print(o, idx[o])
+        opts, args = getopt(args, "", [])
 
+        if args == []:
+            print("Usage: dulwich dump-index FILENAME")
+            sys.exit(1)
 
-def cmd_init(args):
+        filename = args[0]
-    opts, args = getopt(args, "", ["bare"])
+        idx = Index(filename)
-    opts = dict(opts)
 
-    if args == []:
+        for o in idx:
-        path = os.getcwd()
+            print(o, idx[o])
-    else:
-        path = args[0]
 
-    porcelain.init(path, bare=("--bare" in opts))
 
+class cmd_init(Command):
 
-def cmd_clone(args):
+    def run(self, args):
-    opts, args = getopt(args, "", ["bare"])
+        opts, args = getopt(args, "", ["bare"])
-    opts = dict(opts)
+        opts = dict(opts)
 
-    if args == []:
+        if args == []:
-        print("usage: dulwich clone host:path [PATH]")
+            path = os.getcwd()
-        sys.exit(1)
+        else:
+            path = args[0]
 
-    source = args.pop(0)
+        porcelain.init(path, bare=("--bare" in opts))
-    if len(args) > 0:
-        target = args.pop(0)
-    else:
-        target = None
 
-    porcelain.clone(source, target, bare=("--bare" in opts))
 
+class cmd_clone(Command):
 
-def cmd_commit(args):
+    def run(self, args):
-    opts, args = getopt(args, "", ["message"])
+        opts, args = getopt(args, "", ["bare"])
-    opts = dict(opts)
+        opts = dict(opts)
-    porcelain.commit(".", message=opts["--message"])
 
+        if args == []:
+            print("usage: dulwich clone host:path [PATH]")
+            sys.exit(1)
 
-def cmd_commit_tree(args):
+        source = args.pop(0)
-    opts, args = getopt(args, "", ["message"])
+        if len(args) > 0:
-    if args == []:
+            target = args.pop(0)
-        print("usage: dulwich commit-tree tree")
+        else:
-        sys.exit(1)
+            target = None
-    opts = dict(opts)
-    porcelain.commit_tree(".", tree=args[0], message=opts["--message"])
 
+        porcelain.clone(source, target, bare=("--bare" in opts))
 
-def cmd_update_server_info(args):
-    porcelain.update_server_info(".")
 
+class cmd_commit(Command):
 
-def cmd_symbolic_ref(args):
+    def run(self, args):
-    opts, args = getopt(args, "", ["ref-name", "force"])
+        opts, args = getopt(args, "", ["message"])
-    if not args:
+        opts = dict(opts)
-        print("Usage: dulwich symbolic-ref REF_NAME [--force]")
+        porcelain.commit(".", message=opts["--message"])
-        sys.exit(1)
 
-    ref_name = args.pop(0)
-    porcelain.symbolic_ref(".", ref_name=ref_name, force='--force' in args)
 
+class cmd_commit_tree(Command):
 
-def cmd_show(args):
+    def run(self, args):
-    opts, args = getopt(args, "", [])
+        opts, args = getopt(args, "", ["message"])
-    porcelain.show(".", args)
+        if args == []:
+            print("usage: dulwich commit-tree tree")
+            sys.exit(1)
+        opts = dict(opts)
+        porcelain.commit_tree(".", tree=args[0], message=opts["--message"])
 
 
-def cmd_diff_tree(args):
+class cmd_update_server_info(Command):
-    opts, args = getopt(args, "", [])
-    if len(args) < 2:
-        print("Usage: dulwich diff-tree OLD-TREE NEW-TREE")
-        sys.exit(1)
-    porcelain.diff_tree(".", args[0], args[1])
 
+    def run(self, args):
+        porcelain.update_server_info(".")
 
-def cmd_rev_list(args):
-    opts, args = getopt(args, "", [])
-    if len(args) < 1:
-        print('Usage: dulwich rev-list COMMITID...')
-        sys.exit(1)
-    porcelain.rev_list('.', args)
 
+class cmd_symbolic_ref(Command):
 
-def cmd_tag(args):
+    def run(self, args):
-    opts, args = getopt(args, '', [])
+        opts, args = getopt(args, "", ["ref-name", "force"])
-    if len(args) < 2:
+        if not args:
-        print('Usage: dulwich tag NAME')
+            print("Usage: dulwich symbolic-ref REF_NAME [--force]")
-        sys.exit(1)
+            sys.exit(1)
-    porcelain.tag('.', args[0])
 
-
+        ref_name = args.pop(0)
-def cmd_repack(args):
+        porcelain.symbolic_ref(".", ref_name=ref_name, force='--force' in args)
-    opts, args = getopt(args, "", [])
-    opts = dict(opts)
-    porcelain.repack('.')
 
 
-def cmd_reset(args):
+class cmd_show(Command):
-    opts, args = getopt(args, "", ["hard", "soft", "mixed"])
+
-    opts = dict(opts)
+    def run(self, args):
-    mode = ""
+        opts, args = getopt(args, "", [])
-    if "--hard" in opts:
+        porcelain.show(".", args)
-        mode = "hard"
+
-    elif "--soft" in opts:
+
-        mode = "soft"
+class cmd_diff_tree(Command):
-    elif "--mixed" in opts:
+
-        mode = "mixed"
+    def run(self, args):
-    porcelain.reset('.', mode=mode, *args)
+        opts, args = getopt(args, "", [])
-
+        if len(args) < 2:
-
+            print("Usage: dulwich diff-tree OLD-TREE NEW-TREE")
-def cmd_daemon(args):
+            sys.exit(1)
-    from dulwich import log_utils
+        porcelain.diff_tree(".", args[0], args[1])
-    from dulwich.protocol import TCP_GIT_PORT
+
-    parser = optparse.OptionParser()
+
-    parser.add_option("-l", "--listen_address", dest="listen_address",
+class cmd_rev_list(Command):
-                      default="localhost",
+
-                      help="Binding IP address.")
+    def run(self, args):
-    parser.add_option("-p", "--port", dest="port", type=int,
+        opts, args = getopt(args, "", [])
-                      default=TCP_GIT_PORT,
+        if len(args) < 1:
-                      help="Binding TCP port.")
+            print('Usage: dulwich rev-list COMMITID...')
-    options, args = parser.parse_args(args)
+            sys.exit(1)
-
+        porcelain.rev_list('.', args)
-    log_utils.default_logging_config()
+
-    if len(args) >= 1:
+
-        gitdir = args[0]
+class cmd_tag(Command):
-    else:
+
-        gitdir = '.'
+    def run(self, args):
-    from dulwich import porcelain
+        opts, args = getopt(args, '', [])
-    porcelain.daemon(gitdir, address=options.listen_address,
+        if len(args) < 2:
-                     port=options.port)
+            print('Usage: dulwich tag NAME')
-
+            sys.exit(1)
-
+        porcelain.tag('.', args[0])
-def cmd_web_daemon(args):
+
-    from dulwich import log_utils
+
-    parser = optparse.OptionParser()
+class cmd_repack(Command):
-    parser.add_option("-l", "--listen_address", dest="listen_address",
+
-                      default="",
+    def run(self, args):
-                      help="Binding IP address.")
+        opts, args = getopt(args, "", [])
-    parser.add_option("-p", "--port", dest="port", type=int,
+        opts = dict(opts)
-                      default=8000,
+        porcelain.repack('.')
-                      help="Binding TCP port.")
+
-    options, args = parser.parse_args(args)
+
-
+class cmd_reset(Command):
-    log_utils.default_logging_config()
+
-    if len(args) >= 1:
+    def run(self, args):
-        gitdir = args[0]
+        opts, args = getopt(args, "", ["hard", "soft", "mixed"])
-    else:
+        opts = dict(opts)
-        gitdir = '.'
+        mode = ""
-    from dulwich import porcelain
+        if "--hard" in opts:
-    porcelain.web_daemon(gitdir, address=options.listen_address,
+            mode = "hard"
+        elif "--soft" in opts:
+            mode = "soft"
+        elif "--mixed" in opts:
+            mode = "mixed"
+        porcelain.reset('.', mode=mode, *args)
+
+
+class cmd_daemon(Command):
+
+    def run(self, args):
+        from dulwich import log_utils
+        from dulwich.protocol import TCP_GIT_PORT
+        parser = optparse.OptionParser()
+        parser.add_option("-l", "--listen_address", dest="listen_address",
+                          default="localhost",
+                          help="Binding IP address.")
+        parser.add_option("-p", "--port", dest="port", type=int,
+                          default=TCP_GIT_PORT,
+                          help="Binding TCP port.")
+        options, args = parser.parse_args(args)
+
+        log_utils.default_logging_config()
+        if len(args) >= 1:
+            gitdir = args[0]
+        else:
+            gitdir = '.'
+        from dulwich import porcelain
+        porcelain.daemon(gitdir, address=options.listen_address,
                          port=options.port)
 
 
-def cmd_receive_pack(args):
+class cmd_web_daemon(Command):
-    parser = optparse.OptionParser()
+
-    options, args = parser.parse_args(args)
+    def run(self, args):
-    if len(args) >= 1:
+        from dulwich import log_utils
-        gitdir = args[0]
+        parser = optparse.OptionParser()
-    else:
+        parser.add_option("-l", "--listen_address", dest="listen_address",
-        gitdir = '.'
+                          default="",
-    porcelain.receive_pack(gitdir)
+                          help="Binding IP address.")
-
+        parser.add_option("-p", "--port", dest="port", type=int,
-
+                          default=8000,
-def cmd_upload_pack(args):
+                          help="Binding TCP port.")
-    parser = optparse.OptionParser()
+        options, args = parser.parse_args(args)
-    options, args = parser.parse_args(args)
+
-    if len(args) >= 1:
+        log_utils.default_logging_config()
-        gitdir = args[0]
+        if len(args) >= 1:
-    else:
+            gitdir = args[0]
-        gitdir = '.'
+        else:
-    porcelain.upload_pack(gitdir)
+            gitdir = '.'
-
+        from dulwich import porcelain
-
+        porcelain.web_daemon(gitdir, address=options.listen_address,
-def cmd_status(args):
+                             port=options.port)
-    parser = optparse.OptionParser()
+
-    options, args = parser.parse_args(args)
+
-    if len(args) >= 1:
+class cmd_receive_pack(Command):
-        gitdir = args[0]
+
-    else:
+    def run(self, args):
-        gitdir = '.'
+        parser = optparse.OptionParser()
-    status = porcelain.status(gitdir)
+        options, args = parser.parse_args(args)
-    if status.staged:
+        if len(args) >= 1:
-        sys.stdout.write("Changes to be committed:\n\n")
+            gitdir = args[0]
-        for kind, names in status.staged.items():
+        else:
-            for name in names:
+            gitdir = '.'
-                sys.stdout.write("\t%s: %s\n" % (kind, name))
+        porcelain.receive_pack(gitdir)
-        sys.stdout.write("\n")
+
-    if status.unstaged:
+
-        sys.stdout.write("Changes not staged for commit:\n\n")
+class cmd_upload_pack(Command):
-        for name in status.unstaged:
+
-            sys.stdout.write("\t%s\n" %
+    def run(self, args):
-                    name.decode(sys.getfilesystemencoding()))
+        parser = optparse.OptionParser()
-        sys.stdout.write("\n")
+        options, args = parser.parse_args(args)
-    if status.untracked:
+        if len(args) >= 1:
-        sys.stdout.write("Untracked files:\n\n")
+            gitdir = args[0]
-        for name in status.untracked:
+        else:
-            sys.stdout.write("\t%s\n" % name)
+            gitdir = '.'
-        sys.stdout.write("\n")
+        porcelain.upload_pack(gitdir)
-
+
-
+
-def cmd_ls_remote(args):
+class cmd_status(Command):
-    opts, args = getopt(args, '', [])
+
-    if len(args) < 1:
+    def run(self, args):
-        print('Usage: dulwich ls-remote URL')
+        parser = optparse.OptionParser()
-        sys.exit(1)
+        options, args = parser.parse_args(args)
-    refs = porcelain.ls_remote(args[0])
+        if len(args) >= 1:
-    for ref in sorted(refs):
+            gitdir = args[0]
-        sys.stdout.write("%s\t%s\n" % (ref, refs[ref]))
+        else:
-
+            gitdir = '.'
-
+        status = porcelain.status(gitdir)
-def cmd_ls_tree(args):
+        if any(names for (kind, names) in status.staged.items()):
-    parser = optparse.OptionParser()
+            sys.stdout.write("Changes to be committed:\n\n")
-    parser.add_option("-r", "--recursive", action="store_true",
+            for kind, names in status.staged.items():
-                      help="Recusively list tree contents.")
+                for name in names:
-    parser.add_option("--name-only", action="store_true",
+                    sys.stdout.write("\t%s: %s\n" % (
-                      help="Only display name.")
+                        kind, name.decode(sys.getfilesystemencoding())))
-    options, args = parser.parse_args(args)
+            sys.stdout.write("\n")
-    try:
+        if status.unstaged:
-        treeish = args.pop(0)
+            sys.stdout.write("Changes not staged for commit:\n\n")
-    except IndexError:
+            for name in status.unstaged:
-        treeish = None
+                sys.stdout.write("\t%s\n" %
-    porcelain.ls_tree(
+                        name.decode(sys.getfilesystemencoding()))
-        '.', treeish, outstream=sys.stdout, recursive=options.recursive,
+            sys.stdout.write("\n")
-        name_only=options.name_only)
+        if status.untracked:
-
+            sys.stdout.write("Untracked files:\n\n")
-
+            for name in status.untracked:
-def cmd_pack_objects(args):
+                sys.stdout.write("\t%s\n" % name)
-    opts, args = getopt(args, '', ['stdout'])
+            sys.stdout.write("\n")
-    opts = dict(opts)
+
-    if len(args) < 1 and not '--stdout' in args:
+
-        print('Usage: dulwich pack-objects basename')
+class cmd_ls_remote(Command):
-        sys.exit(1)
+
-    object_ids = [l.strip() for l in sys.stdin.readlines()]
+    def run(self, args):
-    basename = args[0]
+        opts, args = getopt(args, '', [])
-    if '--stdout' in opts:
+        if len(args) < 1:
-        packf = getattr(sys.stdout, 'buffer', sys.stdout)
+            print('Usage: dulwich ls-remote URL')
-        idxf = None
+            sys.exit(1)
-        close = []
+        refs = porcelain.ls_remote(args[0])
-    else:
+        for ref in sorted(refs):
-        packf = open(basename + '.pack', 'w')
+            sys.stdout.write("%s\t%s\n" % (ref, refs[ref]))
-        idxf = open(basename + '.idx', 'w')
+
-        close = [packf, idxf]
+
-    porcelain.pack_objects('.', object_ids, packf, idxf)
+class cmd_ls_tree(Command):
-    for f in close:
+
-        f.close()
+    def run(self, args):
-
+        parser = optparse.OptionParser()
-
+        parser.add_option("-r", "--recursive", action="store_true",
-def cmd_help(args):
+                          help="Recusively list tree contents.")
-    parser = optparse.OptionParser()
+        parser.add_option("--name-only", action="store_true",
-    parser.add_option("-a", "--all", dest="all",
+                          help="Only display name.")
-                      action="store_true",
+        options, args = parser.parse_args(args)
-                      help="List all commands.")
+        try:
-    options, args = parser.parse_args(args)
+            treeish = args.pop(0)
-
+        except IndexError:
-    if options.all:
+            treeish = None
-        print('Available commands:')
+        porcelain.ls_tree(
-        for cmd in sorted(commands):
+            '.', treeish, outstream=sys.stdout, recursive=options.recursive,
-            print('  %s' % cmd)
+            name_only=options.name_only)
-    else:
+
-        print("""\
+
+class cmd_pack_objects(Command):
+
+    def run(self, args):
+        opts, args = getopt(args, '', ['stdout'])
+        opts = dict(opts)
+        if len(args) < 1 and not '--stdout' in args:
+            print('Usage: dulwich pack-objects basename')
+            sys.exit(1)
+        object_ids = [l.strip() for l in sys.stdin.readlines()]
+        basename = args[0]
+        if '--stdout' in opts:
+            packf = getattr(sys.stdout, 'buffer', sys.stdout)
+            idxf = None
+            close = []
+        else:
+            packf = open(basename + '.pack', 'w')
+            idxf = open(basename + '.idx', 'w')
+            close = [packf, idxf]
+        porcelain.pack_objects('.', object_ids, packf, idxf)
+        for f in close:
+            f.close()
+
+
+class cmd_pull(Command):
+
+    def run(self, args):
+        parser = optparse.OptionParser()
+        options, args = parser.parse_args(args)
+        try:
+            from_location = args[0]
+        except IndexError:
+            from_location = None
+        porcelain.pull('.', from_location)
+
+
+class cmd_remote_add(Command):
+
+    def run(self, args):
+        parser = optparse.OptionParser()
+        options, args = parser.parse_args(args)
+        porcelain.remote_add('.', args[0], args[1])
+
+
+class cmd_remote(Command):
+
+    subcommands = {
+        "add": cmd_remote_add,
+    }
+
+    def run(self, args):
+        if not args:
+            print("Supported subcommands: %s" % ', '.join(self.subcommands.keys()))
+            return False
+        cmd = args[0]
+        try:
+            cmd_kls = self.subcommands[cmd]
+        except KeyError:
+            print('No such subcommand: %s' % args[0])
+            return False
+        return cmd_kls(args[1:])
+
+
+class cmd_help(Command):
+
+    def run(self, args):
+        parser = optparse.OptionParser()
+        parser.add_option("-a", "--all", dest="all",
+                          action="store_true",
+                          help="List all commands.")
+        options, args = parser.parse_args(args)
+
+        if options.all:
+            print('Available commands:')
+            for cmd in sorted(commands):
+                print('  %s' % cmd)
+        else:
+            print("""\
 The dulwich command line tool is currently a very basic frontend for the
 Dulwich python module. For full functionality, please see the API reference.
 
@@ -440,7 +548,9 @@ commands = {
     "ls-remote": cmd_ls_remote,
     "ls-tree": cmd_ls_tree,
     "pack-objects": cmd_pack_objects,
+    "pull": cmd_pull,
     "receive-pack": cmd_receive_pack,
+    "remote": cmd_remote,
     "repack": cmd_repack,
     "reset": cmd_reset,
     "rev-list": cmd_rev_list,
@@ -459,7 +569,10 @@ if len(sys.argv) < 2:
     sys.exit(1)
 
 cmd = sys.argv[1]
-if not cmd in commands:
+try:
+    cmd_kls = commands[cmd]
+except KeyError:
     print("No such subcommand: %s" % cmd)
     sys.exit(1)
-commands[cmd](sys.argv[2:])
+# TODO(jelmer): Return non-0 on errors
+cmd_kls().run(sys.argv[2:])

dulwich.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: dulwich
-Version: 0.16.3
+Version: 0.17.1
 Summary: Python Git Library
 Home-page: https://www.dulwich.io/
 Author: Jelmer Vernooij

dulwich.egg-info/SOURCES.txt

@@ -1,11 +1,13 @@
 .testr.conf
 .travis.yml
 AUTHORS
+CONTRIBUTING.md
 COPYING
 MANIFEST.in
 Makefile
 NEWS
 README.md
+README.swift.md
 TODO
 appveyor.yml
 dulwich.cfg

dulwich/__init__.py

@@ -22,4 +22,4 @@
 
 """Python implementation of the Git file formats and protocols."""
 
-__version__ = (0, 16, 3)
+__version__ = (0, 17, 1)

dulwich/client.py

@@ -38,8 +38,6 @@ Known capabilities that are not supported:
  * include-tag
 """
 
-__docformat__ = 'restructuredText'
-
 from contextlib import closing
 from io import BytesIO, BufferedReader
 import dulwich
@@ -632,6 +630,7 @@ class TraditionalGitClient(GitClient):
         proto, _ = self._connect(b'upload-pack', path)
         with proto:
             refs, _ = read_pkt_refs(proto)
+            proto.write_pkt_line(None)
             return refs
 
     def archive(self, path, committish, write_data, progress=None,
@@ -679,9 +678,9 @@ class TCPGitClient(TraditionalGitClient):
         return urlparse.urlunsplit(("git", netloc, path, '', ''))
 
     def _connect(self, cmd, path):
-        if type(cmd) is not bytes:
+        if not isinstance(cmd, bytes):
             raise TypeError(cmd)
-        if type(path) is not bytes:
+        if not isinstance(path, bytes):
             path = path.encode(self._remote_path_encoding)
         sockaddrs = socket.getaddrinfo(
             self._host, self._port, socket.AF_UNSPEC, socket.SOCK_STREAM)
@@ -779,9 +778,9 @@ class SubprocessGitClient(TraditionalGitClient):
     git_command = None
 
     def _connect(self, service, path):
-        if type(service) is not bytes:
+        if not isinstance(service, bytes):
             raise TypeError(service)
-        if type(path) is not bytes:
+        if not isinstance(path, bytes):
             path = path.encode(self._remote_path_encoding)
         if self.git_command is None:
             git_command = find_git_command()
@@ -951,7 +950,7 @@ class SubprocessSSHVendor(SSHVendor):
         if username is not None:
             host = '%s@%s' % (username, host)
         args.append(host)
-        proc = subprocess.Popen(args + [command],
+        proc = subprocess.Popen(args + [command], bufsize=0,
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE)
         return SubprocessWrapper(proc)
@@ -1004,9 +1003,9 @@ class SSHGitClient(TraditionalGitClient):
         return cmd
 
     def _connect(self, cmd, path):
-        if type(cmd) is not bytes:
+        if not isinstance(cmd, bytes):
             raise TypeError(cmd)
-        if type(path) is not bytes:
+        if not isinstance(path, bytes):
             path = path.encode(self._remote_path_encoding)
         if path.startswith(b"/~"):
             path = path[1:]
@@ -1252,10 +1251,7 @@ def get_transport_and_path_from_url(url, config=None, **kwargs):
         return (TCPGitClient.from_parsedurl(parsed, **kwargs),
                 parsed.path)
     elif parsed.scheme in ('git+ssh', 'ssh'):
-        path = parsed.path
+        return SSHGitClient.from_parsedurl(parsed, **kwargs), parsed.path
-        if path.startswith('/'):
-            path = parsed.path[1:]
-        return SSHGitClient.from_parsedurl(parsed, **kwargs), path
     elif parsed.scheme in ('http', 'https'):
         return HttpGitClient.from_parsedurl(
             parsed, config=config, **kwargs), parsed.path

dulwich/config.py

@@ -95,6 +95,14 @@ class Config(object):
         """
         raise NotImplementedError(self.itersections)
 
+    def has_section(self, name):
+        """Check if a specified section exists.
+
+        :param name: Name of section to check for
+        :return: boolean indicating whether the section exists
+        """
+        return (name in self.itersections())
+
 
 class ConfigDict(Config, MutableMapping):
     """Git configuration stored in a dictionary."""
@@ -307,23 +315,20 @@ class ConfigFile(ConfigDict):
                 if not _check_variable_name(setting):
                     raise ValueError("invalid variable name %s" % setting)
                 if value.endswith(b"\\\n"):
-                    value = value[:-2]
+                    continuation = value[:-2]
-                    continuation = True
                 else:
-                    continuation = False
+                    continuation = None
-                value = _parse_string(value)
+                    value = _parse_string(value)
-                ret._values[section][setting] = value
+                    ret._values[section][setting] = value
-                if not continuation:
                     setting = None
             else:  # continuation line
                 if line.endswith(b"\\\n"):
-                    line = line[:-2]
+                    continuation += line[:-2]
-                    continuation = True
                 else:
-                    continuation = False
+                    continuation += line
-                value = _parse_string(line)
+                    value = _parse_string(continuation)
-                ret._values[section][setting] += value
+                    ret._values[section][setting] = value
-                if not continuation:
+                    continuation = None
                     setting = None
         return ret
 

dulwich/contrib/__init__.py

@@ -22,6 +22,7 @@
 def test_suite():
     import unittest
     names = [
+        'release_robot',
         'swift',
         ]
     module_names = ['dulwich.contrib.test_' + name for name in names]

dulwich/contrib/release_robot.py

@@ -23,74 +23,100 @@ Alternate to `Versioneer <https://pypi.python.org/pypi/versioneer/>`_ using
 `Dulwich <https://pypi.python.org/pypi/dulwich>`_ to sort tags by time from
 newest to oldest.
 
-Import this module into the package ``__init__.py`` and then set ``__version__``
+Copy the following into the package ``__init__.py`` module::
-as follows::
 
     from dulwich.contrib.release_robot import get_current_version
-
     __version__ = get_current_version()
-    # other dunder classes like __author__, etc.
 
 This example assumes the tags have a leading "v" like "v0.3", and that the
-``.git`` folder is in the project folder that containts the package folder.
+``.git`` folder is in a project folder that containts the package folder.
+
+EG::
+
+    * project
+    |
+    * .git
+    |
+    +-* package
+      |
+      * __init__.py  <-- put __version__ here
+
+
 """
 
-from dulwich.repo import Repo
-import time
 import datetime
-import os
 import re
 import sys
+import time
+
+from dulwich.repo import Repo
 
 # CONSTANTS
-DIRNAME = os.path.abspath(os.path.dirname(__file__))
+PROJDIR = '.'
-PROJDIR = os.path.dirname(DIRNAME)
+PATTERN = r'[ a-zA-Z_\-]*([\d\.]+[\-\w\.]*)'
-PATTERN = '[ a-zA-Z_\-]*([\d\.]+[\-\w\.]*)'
 
 
 def get_recent_tags(projdir=PROJDIR):
     """Get list of tags in order from newest to oldest and their datetimes.
 
     :param projdir: path to ``.git``
-    :returns: list of (tag, [datetime, commit, author]) sorted from new to old
+    :returns: list of tags sorted by commit time from newest to oldest
+
+    Each tag in the list contains the tag name, commit time, commit id, author
+    and any tag meta. If a tag isn't annotated, then its tag meta is ``None``.
+    Otherwise the tag meta is a tuple containing the tag time, tag id and tag
+    name. Time is in UTC.
     """
-    project = Repo(projdir)  # dulwich repository object
+    with Repo(projdir) as project:  # dulwich repository object
-    refs = project.get_refs()  # dictionary of refs and their SHA-1 values
+        refs = project.get_refs()  # dictionary of refs and their SHA-1 values
-    tags = {}  # empty dictionary to hold tags, commits and datetimes
+        tags = {}  # empty dictionary to hold tags, commits and datetimes
-    # iterate over refs in repository
+        # iterate over refs in repository
-    for key, value in refs.items():
+        for key, value in refs.items():
-        obj = project.get_object(value)  # dulwich object from SHA-1
+            key = key.decode('utf-8')  # compatible with Python-3
-        # check if object is tag
+            obj = project.get_object(value)  # dulwich object from SHA-1
-        if obj.type_name != 'tag':
+            # don't just check if object is "tag" b/c it could be a "commit"
-            # skip ref if not a tag
+            # instead check if "tags" is in the ref-name
-            continue
+            if u'tags' not in key:
-        # strip the leading text from "refs/tag/<tag name>" to get "tag name"
+                # skip ref if not a tag
-        _, tag = key.rsplit('/', 1)
+                continue
-        # check if tag object is commit, altho it should always be true
+            # strip the leading text from refs to get "tag name"
-        if obj.object[0].type_name == 'commit':
+            _, tag = key.rsplit(u'/', 1)
-            commit = project.get_object(obj.object[1])  # commit object
+            # check if tag object is "commit" or "tag" pointing to a "commit"
+            try:
+                commit = obj.object  # a tuple (commit class, commit id)
+            except AttributeError:
+                commit = obj
+                tag_meta = None
+            else:
+                tag_meta = (
+                    datetime.datetime(*time.gmtime(obj.tag_time)[:6]),
+                    obj.id.decode('utf-8'),
+                    obj.name.decode('utf-8')
+                )  # compatible with Python-3
+                commit = project.get_object(commit[1])  # commit object
             # get tag commit datetime, but dulwich returns seconds since
             # beginning of epoch, so use Python time module to convert it to
             # timetuple then convert to datetime
             tags[tag] = [
                 datetime.datetime(*time.gmtime(commit.commit_time)[:6]),
-                commit.id,
+                commit.id.decode('utf-8'),
-                commit.author
+                commit.author.decode('utf-8'),
-            ]
+                tag_meta
+            ]  # compatible with Python-3
 
     # return list of tags sorted by their datetimes from newest to oldest
     return sorted(tags.items(), key=lambda tag: tag[1][0], reverse=True)
 
 
-def get_current_version(pattern=PATTERN, projdir=PROJDIR, logger=None):
+def get_current_version(projdir=PROJDIR, pattern=PATTERN, logger=None):
     """Return the most recent tag, using an options regular expression pattern.
 
     The default pattern will strip any characters preceding the first semantic
     version. *EG*: "Release-0.2.1-rc.1" will be come "0.2.1-rc.1". If no match
     is found, then the most recent tag is return without modification.
 
-    :param pattern: regular expression pattern with group that matches version
     :param projdir: path to ``.git``
+    :param pattern: regular expression pattern with group that matches version
     :param logger: a Python logging instance to capture exception
     :returns: tag matching first group in regular expression pattern
     """
@@ -99,9 +125,9 @@ def get_current_version(pattern=PATTERN, projdir=PROJDIR, logger=None):
         tag = tags[0][0]
     except IndexError:
         return
-    m = re.match(pattern, tag)
+    matches = re.match(pattern, tag)
     try:
-        current_version = m.group(1)
+        current_version = matches.group(1)
     except (IndexError, AttributeError) as err:
         if logger:
             logger.exception(err)
@@ -109,21 +135,9 @@ def get_current_version(pattern=PATTERN, projdir=PROJDIR, logger=None):
     return current_version
 
 
-def test_tag_pattern():
-    test_cases = {
-        '0.3': '0.3', 'v0.3': '0.3', 'release0.3': '0.3', 'Release-0.3': '0.3',
-        'v0.3rc1': '0.3rc1', 'v0.3-rc1': '0.3-rc1', 'v0.3-rc.1': '0.3-rc.1',
-        'version 0.3': '0.3', 'version_0.3_rc_1': '0.3_rc_1', 'v1': '1',
-        '0.3rc1': '0.3rc1'
-    }
-    for tc, version in test_cases.iteritems():
-        m = re.match(PATTERN, tc)
-        assert m.group(1) == version
-
-
 if __name__ == '__main__':
     if len(sys.argv) > 1:
-        projdir = sys.argv[1]
+        _PROJDIR = sys.argv[1]
     else:
-        projdir = PROJDIR
+        _PROJDIR = PROJDIR
-    print(get_current_version(projdir=projdir))
+    print(get_current_version(projdir=_PROJDIR))

dulwich/contrib/test_release_robot.py

@@ -19,21 +19,109 @@
 
 """Tests for release_robot."""
 
+import datetime
+import os
 import re
+import shutil
+import tempfile
+import time
 import unittest
 
-from dulwich.contrib.release_robot import PATTERN
+from dulwich.contrib import release_robot
+from dulwich.repo import Repo
+from dulwich.tests.utils import make_commit, make_tag
+
+BASEDIR = os.path.abspath(os.path.dirname(__file__))  # this directory
+
+
+def gmtime_to_datetime(gmt):
+    return datetime.datetime(*time.gmtime(gmt)[:6])
 
 
 class TagPatternTests(unittest.TestCase):
+    """test tag patterns"""
 
     def test_tag_pattern(self):
+        """test tag patterns"""
         test_cases = {
-            '0.3': '0.3', 'v0.3': '0.3', 'release0.3': '0.3', 'Release-0.3': '0.3',
+            '0.3': '0.3', 'v0.3': '0.3', 'release0.3': '0.3',
-            'v0.3rc1': '0.3rc1', 'v0.3-rc1': '0.3-rc1', 'v0.3-rc.1': '0.3-rc.1',
+            'Release-0.3': '0.3', 'v0.3rc1': '0.3rc1', 'v0.3-rc1': '0.3-rc1',
-            'version 0.3': '0.3', 'version_0.3_rc_1': '0.3_rc_1', 'v1': '1',
+            'v0.3-rc.1': '0.3-rc.1', 'version 0.3': '0.3',
-            '0.3rc1': '0.3rc1'
+            'version_0.3_rc_1': '0.3_rc_1', 'v1': '1', '0.3rc1': '0.3rc1'
         }
-        for tc, version in test_cases.items():
+        for testcase, version in test_cases.items():
-            m = re.match(PATTERN, tc)
+            matches = re.match(release_robot.PATTERN, testcase)
-            self.assertEqual(m.group(1), version)
+            self.assertEqual(matches.group(1), version)
+
+
+class GetRecentTagsTest(unittest.TestCase):
+    """test get recent tags"""
+
+    # Git repo for dulwich project
+    test_repo = os.path.join(BASEDIR, 'dulwich_test_repo.zip')
+    committer = b"Mark Mikofski <mark.mikofski@sunpowercorp.com>"
+    test_tags = [b'v0.1a', b'v0.1']
+    tag_test_data = {
+        test_tags[0]: [1484788003, b'0' * 40, None],
+        test_tags[1]: [1484788314, b'1' * 40, (1484788401, b'2' * 40)]
+    }
+
+    @classmethod
+    def setUpClass(cls):
+        cls.projdir = tempfile.mkdtemp()  # temporary project directory
+        cls.repo = Repo.init(cls.projdir)  # test repo
+        obj_store = cls.repo.object_store  # test repo object store
+        # commit 1 ('2017-01-19T01:06:43')
+        cls.c1 = make_commit(
+            id=cls.tag_test_data[cls.test_tags[0]][1],
+            commit_time=cls.tag_test_data[cls.test_tags[0]][0],
+            message=b'unannotated tag',
+            author=cls.committer
+        )
+        obj_store.add_object(cls.c1)
+        # tag 1: unannotated
+        cls.t1 = cls.test_tags[0]
+        cls.repo[b'refs/tags/' + cls.t1] = cls.c1.id  # add unannotated tag
+        # commit 2 ('2017-01-19T01:11:54')
+        cls.c2 = make_commit(
+            id=cls.tag_test_data[cls.test_tags[1]][1],
+            commit_time=cls.tag_test_data[cls.test_tags[1]][0],
+            message=b'annotated tag',
+            parents=[cls.c1.id],
+            author=cls.committer
+        )
+        obj_store.add_object(cls.c2)
+        # tag 2: annotated ('2017-01-19T01:13:21')
+        cls.t2 = make_tag(
+            cls.c2,
+            id=cls.tag_test_data[cls.test_tags[1]][2][1],
+            name=cls.test_tags[1],
+            tag_time=cls.tag_test_data[cls.test_tags[1]][2][0]
+        )
+        obj_store.add_object(cls.t2)
+        cls.repo[b'refs/heads/master'] = cls.c2.id
+        cls.repo[b'refs/tags/' + cls.t2.name] = cls.t2.id  # add annotated tag
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.repo.close()
+        shutil.rmtree(cls.projdir)
+
+    def test_get_recent_tags(self):
+        """test get recent tags"""
+        tags = release_robot.get_recent_tags(self.projdir)  # get test tags
+        for tag, metadata in tags:
+            tag = tag.encode('utf-8')
+            test_data = self.tag_test_data[tag]  # test data tag
+            # test commit date, id and author name
+            self.assertEqual(metadata[0], gmtime_to_datetime(test_data[0]))
+            self.assertEqual(metadata[1].encode('utf-8'), test_data[1])
+            self.assertEqual(metadata[2].encode('utf-8'), self.committer)
+            # skip unannotated tags
+            tag_obj = test_data[2]
+            if not tag_obj:
+                continue
+            # tag date, id and name
+            self.assertEqual(metadata[3][0], gmtime_to_datetime(tag_obj[0]))
+            self.assertEqual(metadata[3][1].encode('utf-8'), tag_obj[1])
+            self.assertEqual(metadata[3][2].encode('utf-8'), tag)

dulwich/index.py

@@ -508,10 +508,12 @@ def build_index_from_tree(root_path, index_path, object_store, tree_id,
         if not os.path.exists(os.path.dirname(full_path)):
             os.makedirs(os.path.dirname(full_path))
 
-        # FIXME: Merge new index into working tree
+        # TODO(jelmer): Merge new index into working tree
         if S_ISGITLINK(entry.mode):
-            os.mkdir(full_path)
+            if not os.path.isdir(full_path):
+                os.mkdir(full_path)
             st = os.lstat(full_path)
+            # TODO(jelmer): record and return submodule paths
         else:
             obj = object_store[entry.sha]
             st = build_file_from_blob(obj, entry.mode, full_path,
@@ -560,6 +562,7 @@ def get_unstaged_changes(index, root_path):
 
     for tree_path, entry in index.iteritems():
         full_path = _tree_to_fs_path(root_path, tree_path)
+        # TODO(jelmer): handle S_ISGITLINK(entry.mode) here
         try:
             blob = blob_from_path_and_stat(full_path, os.lstat(full_path))
         except OSError as e:
@@ -568,6 +571,11 @@ def get_unstaged_changes(index, root_path):
             # The file was removed, so we assume that counts as
             # different from whatever file used to exist.
             yield tree_path
+        except IOError as e:
+            if e.errno != errno.EISDIR:
+                raise
+            # The file was changed to a directory, so consider it removed.
+            yield tree_path
         else:
             if blob.id != entry.sha:
                 yield tree_path

dulwich/objects.py

@@ -609,6 +609,12 @@ def _parse_message(chunks):
     v = ""
     eof = False
 
+    def _strip_last_newline(value):
+        """Strip the last newline from value"""
+        if value and value.endswith(b'\n'):
+            return value[:-1]
+        return value
+
     # Parse the headers
     #
     # Headers can contain newlines. The next line is indented with a space.
@@ -620,7 +626,7 @@ def _parse_message(chunks):
         else:
             if k is not None:
                 # We parsed a new header, return its value
-                yield (k, v.rstrip(b'\n'))
+                yield (k, _strip_last_newline(v))
             if l == b'\n':
                 # Empty line indicates end of headers
                 break
@@ -632,7 +638,7 @@ def _parse_message(chunks):
         # the text.
         eof = True
         if k is not None:
-            yield (k, v.rstrip(b'\n'))
+            yield (k, _strip_last_newline(v))
         yield (None, None)
 
     if not eof:
@@ -655,6 +661,9 @@ class Tag(ShaFile):
 
     def __init__(self):
         super(Tag, self).__init__()
+        self._tagger = None
+        self._tag_time = None
+        self._tag_timezone = None
         self._tag_timezone_neg_utc = False
 
     @classmethod
@@ -714,6 +723,9 @@ class Tag(ShaFile):
     def _deserialize(self, chunks):
         """Grab the metadata attached to the tag"""
         self._tagger = None
+        self._tag_time = None
+        self._tag_timezone = None
+        self._tag_timezone_neg_utc = False
         for field, value in _parse_message(chunks):
             if field == _OBJECT_HEADER:
                 self._object_sha = value
@@ -1204,7 +1216,8 @@ class Commit(ShaFile):
                 chunks.append(b' ' + chunk + b'\n')
 
             # No trailing empty line
-            chunks[-1] = chunks[-1].rstrip(b' \n')
+            if chunks[-1].endswith(b' \n'):
+                chunks[-1] = chunks[-1][:-2]
         for k, v in self.extra:
             if b'\n' in k or b'\n' in v:
                 raise AssertionError(

dulwich/pack.py

@@ -1158,8 +1158,7 @@ class PackData(object):
             object count
         :return: List of tuples with (sha, offset, crc32)
         """
-        ret = list(self.iterentries(progress=progress))
+        ret = sorted(self.iterentries(progress=progress))
-        ret.sort()
         return ret
 
     def create_index_v1(self, filename, progress=None):
@@ -1495,8 +1494,7 @@ def write_pack(filename, objects, deltify=None, delta_window_size=None):
     with GitFile(filename + '.pack', 'wb') as f:
         entries, data_sum = write_pack_objects(f, objects,
             delta_window_size=delta_window_size, deltify=deltify)
-    entries = [(k, v[0], v[1]) for (k, v) in entries.items()]
+    entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
-    entries.sort()
     with GitFile(filename + '.idx', 'wb') as f:
         return data_sum, write_pack_index_v2(f, entries, data_sum)
 

dulwich/patch.py

@@ -45,7 +45,7 @@ def write_commit_patch(f, commit, contents, progress, version=None, encoding=Non
     :return: tuple with filename and contents
     """
     encoding = encoding or getattr(f, "encoding", "ascii")
-    if type(contents) is str:
+    if isinstance(contents, str):
         contents = contents.encode(encoding)
     (num, total) = progress
     f.write(b"From " + commit.id + b" " + time.ctime(commit.commit_time).encode(encoding) + b"\n")
@@ -255,7 +255,7 @@ def git_am_patch_split(f, encoding=None):
     """
     encoding = encoding or getattr(f, "encoding", "ascii")
     contents = f.read()
-    if type(contents) is bytes and getattr(email.parser, "BytesParser", None):
+    if isinstance(contents, bytes) and getattr(email.parser, "BytesParser", None):
         parser = email.parser.BytesParser()
         msg = parser.parsebytes(contents)
     else:

dulwich/porcelain.py

@@ -36,6 +36,7 @@ Currently implemented:
  * pull
  * push
  * rm
+ * remote{_add}
  * receive-pack
  * reset
  * rev-list
@@ -49,8 +50,6 @@ These functions are meant to behave similarly to the git subcommands.
 Differences in behaviour are considered bugs.
 """
 
-__docformat__ = 'restructuredText'
-
 from collections import namedtuple
 from contextlib import (
     closing,
@@ -123,6 +122,10 @@ default_bytes_err_stream = getattr(sys.stderr, 'buffer', sys.stderr)
 DEFAULT_ENCODING = 'utf-8'
 
 
+class RemoteExists(Exception):
+    """Raised when the remote already exists."""
+
+
 def open_repo(path_or_repo):
     """Open an argument that can be a repository or a path for a repository."""
     if isinstance(path_or_repo, BaseRepo):
@@ -282,7 +285,10 @@ def clone(source, target=None, bare=False, checkout=None,
             {n[len(b'refs/tags/'):]: v for (n, v) in remote_refs.items()
                 if n.startswith(b'refs/tags/') and
                 not n.endswith(ANNOTATED_TAG_SUFFIX)})
-        r[b"HEAD"] = remote_refs[b"HEAD"]
+        if b"HEAD" in remote_refs and not bare:
+            # TODO(jelmer): Support symref capability,
+            # https://github.com/jelmer/dulwich/issues/485
+            r[b"HEAD"] = remote_refs[b"HEAD"]
         target_config = r.get_config()
         if not isinstance(source, bytes):
             source = source.encode(DEFAULT_ENCODING)
@@ -290,7 +296,7 @@ def clone(source, target=None, bare=False, checkout=None,
         target_config.set((b'remote', b'origin'), b'fetch',
             b'+refs/heads/*:refs/remotes/origin/*')
         target_config.write_to_path()
-        if checkout:
+        if checkout and b"HEAD" in r.refs:
             errstream.write(b'Checking out HEAD\n')
             r.reset_index()
     except:
@@ -306,7 +312,6 @@ def add(repo=".", paths=None):
     :param repo: Repository for the files
     :param paths: Paths to add.  No value passed stages all modified files.
     """
-    # FIXME: Support patterns, directories.
     with open_repo_closing(repo) as r:
         if not paths:
             # If nothing is specified, add all non-ignored files.
@@ -317,7 +322,18 @@ def add(repo=".", paths=None):
                     dirnames.remove('.git')
                 for filename in filenames:
                     paths.append(os.path.join(dirpath[len(r.path)+1:], filename))
-        r.stage(paths)
+        # TODO(jelmer): Possibly allow passing in absolute paths?
+        relpaths = []
+        if not isinstance(paths, list):
+            paths = [paths]
+        for p in paths:
+            # FIXME: Support patterns, directories.
+            if os.path.isabs(p) and p.startswith(repo.path):
+                relpath = os.path.relpath(p, repo.path)
+            else:
+                relpath = p
+            relpaths.append(relpath)
+        r.stage(relpaths)
 
 
 def rm(repo=".", paths=None):
@@ -440,7 +456,7 @@ def print_name_status(changes):
     for change in changes:
         if not change:
             continue
-        if type(change) is list:
+        if isinstance(change, list):
             change = change[0]
         if change.type == CHANGE_ADD:
             path1 = change.new.path
@@ -597,8 +613,7 @@ def tag_list(repo, outstream=sys.stdout):
     :param outstream: Stream to write tags to
     """
     with open_repo_closing(repo) as r:
-        tags = list(r.refs.as_dict(b"refs/tags"))
+        tags = sorted(r.refs.as_dict(b"refs/tags"))
-        tags.sort()
         return tags
 
 
@@ -677,7 +692,7 @@ def push(repo, remote_location, refspecs=None,
                             b"\n")
 
 
-def pull(repo, remote_location, refspecs=None,
+def pull(repo, remote_location=None, refspecs=None,
          outstream=default_bytes_out_stream, errstream=default_bytes_err_stream):
     """Pull from remote via dulwich.client
 
@@ -689,6 +704,10 @@ def pull(repo, remote_location, refspecs=None,
     """
     # Open the repo
     with open_repo_closing(repo) as r:
+        if remote_location is None:
+            # TODO(jelmer): Lookup 'remote' for current branch in config
+            raise NotImplementedError(
+                "looking up remote from branch config not supported yet")
         if refspecs is None:
             refspecs = [b"HEAD"]
         selected_refs = []
@@ -806,6 +825,7 @@ def upload_pack(path=".", inf=None, outf=None):
         outf = getattr(sys.stdout, 'buffer', sys.stdout)
     if inf is None:
         inf = getattr(sys.stdin, 'buffer', sys.stdin)
+    path = os.path.expanduser(path)
     backend = FileSystemBackend(path)
     def send_fn(data):
         outf.write(data)
@@ -828,6 +848,7 @@ def receive_pack(path=".", inf=None, outf=None):
         outf = getattr(sys.stdout, 'buffer', sys.stdout)
     if inf is None:
         inf = getattr(sys.stdin, 'buffer', sys.stdin)
+    path = os.path.expanduser(path)
     backend = FileSystemBackend(path)
     def send_fn(data):
         outf.write(data)
@@ -940,8 +961,7 @@ def pack_objects(repo, object_ids, packf, idxf, delta_window_size=None):
             r.object_store.iter_shas((oid, None) for oid in object_ids),
             delta_window_size=delta_window_size)
     if idxf is not None:
-        entries = [(k, v[0], v[1]) for (k, v) in entries.items()]
+        entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
-        entries.sort()
         write_pack_index(idxf, entries, data_sum)
 
 
@@ -971,3 +991,23 @@ def ls_tree(repo, tree_ish=None, outstream=sys.stdout, recursive=False,
         c = r[tree_ish]
         treeid = c.tree
         list_tree(r.object_store, treeid, "")
+
+
+def remote_add(repo, name, url):
+    """Add a remote.
+
+    :param repo: Path to the repository
+    :param name: Remote name
+    :param url: Remote URL
+    """
+    if not isinstance(name, bytes):
+        name = name.encode(DEFAULT_ENCODING)
+    if not isinstance(url, bytes):
+        url = url.encode(DEFAULT_ENCODING)
+    with open_repo_closing(repo) as r:
+        c = r.get_config()
+        section = (b'remote', name)
+        if c.has_section(section):
+            raise RemoteExists(section)
+        c.set(section, b"url", url)
+        c.write_to_path()

dulwich/repo.py

@@ -750,7 +750,7 @@ class Repo(BaseRepo):
     def commondir(self):
         """Return the path of the common directory.
 
-        For a main working tree, it is identical to `controldir()`.
+        For a main working tree, it is identical to controldir().
 
         For a linked working tree, it is the control directory of the
         main working tree."""
@@ -850,6 +850,10 @@ class Repo(BaseRepo):
         for fs_path in fs_paths:
             if not isinstance(fs_path, bytes):
                 fs_path = fs_path.encode(sys.getfilesystemencoding())
+            if os.path.isabs(fs_path):
+                raise ValueError(
+                    "path %r should be relative to "
+                    "repository root, not absolute" % fs_path)
             tree_path = _fs_to_tree_path(fs_path)
             full_path = os.path.join(root_path_bytes, fs_path)
             try:
@@ -1044,7 +1048,7 @@ class Repo(BaseRepo):
     def init_bare(cls, path):
         """Create a new bare repository.
 
-        ``path`` should already exist and be an emty directory.
+        ``path`` should already exist and be an empty directory.
 
         :param path: Path to create bare repository in
         :return: a `Repo` instance
@@ -1077,6 +1081,13 @@ class MemoryRepo(BaseRepo):
         self._named_files = {}
         self.bare = True
         self._config = ConfigFile()
+        self._description = None
+
+    def set_description(self, description):
+        self._description = description
+
+    def get_description(self):
+        return self._description
 
     def _determine_file_mode(self):
         """Probe the file-system to determine whether permissions can be trusted.
@@ -1122,13 +1133,6 @@ class MemoryRepo(BaseRepo):
         """
         return self._config
 
-    def get_description(self):
-        """Retrieve the repository description.
-
-        This defaults to None, for no description.
-        """
-        return None
-
     @classmethod
     def init_bare(cls, objects, refs):
         """Create a new bare repository in memory.

dulwich/tests/__init__.py

@@ -115,6 +115,7 @@ def self_test_suite():
         'refs',
         'repository',
         'server',
+        'utils',
         'walk',
         'web',
         ]

dulwich/tests/test_archive.py

@@ -21,6 +21,7 @@
 """Tests for archive support."""
 
 from io import BytesIO
+import sys
 import tarfile
 
 from dulwich.archive import tar_stream
@@ -52,7 +53,6 @@ class ArchiveTests(TestCase):
         self.assertEqual([], tf.getnames())
 
     def test_simple(self):
-        self.skipTest("known to fail on python2.6 and 3.4; needs debugging")
         store = MemoryObjectStore()
         b1 = Blob.from_string(b"somedata")
         store.add_object(b1)

dulwich/tests/test_client.py

@@ -370,7 +370,7 @@ class TestGetTransportAndPath(TestCase):
         self.assertEqual('foo.com', c.host)
         self.assertEqual(None, c.port)
         self.assertEqual(None, c.username)
-        self.assertEqual('bar/baz', path)
+        self.assertEqual('/bar/baz', path)
 
     def test_ssh_explicit(self):
         c, path = get_transport_and_path('ssh://foo.com/bar/baz')
@@ -378,7 +378,7 @@ class TestGetTransportAndPath(TestCase):
         self.assertEqual('foo.com', c.host)
         self.assertEqual(None, c.port)
         self.assertEqual(None, c.username)
-        self.assertEqual('bar/baz', path)
+        self.assertEqual('/bar/baz', path)
 
     def test_ssh_port_explicit(self):
         c, path = get_transport_and_path(
@@ -386,7 +386,7 @@ class TestGetTransportAndPath(TestCase):
         self.assertTrue(isinstance(c, SSHGitClient))
         self.assertEqual('foo.com', c.host)
         self.assertEqual(1234, c.port)
-        self.assertEqual('bar/baz', path)
+        self.assertEqual('/bar/baz', path)
 
     def test_username_and_port_explicit_unknown_scheme(self):
         c, path = get_transport_and_path(
@@ -402,19 +402,19 @@ class TestGetTransportAndPath(TestCase):
         self.assertEqual('git', c.username)
         self.assertEqual('server', c.host)
         self.assertEqual(7999, c.port)
-        self.assertEqual('dply/stuff.git', path)
+        self.assertEqual('/dply/stuff.git', path)
 
-    def test_ssh_abspath_explicit(self):
+    def test_ssh_abspath_doubleslash(self):
         c, path = get_transport_and_path('git+ssh://foo.com//bar/baz')
         self.assertTrue(isinstance(c, SSHGitClient))
         self.assertEqual('foo.com', c.host)
         self.assertEqual(None, c.port)
         self.assertEqual(None, c.username)
-        self.assertEqual('/bar/baz', path)
+        self.assertEqual('//bar/baz', path)
 
-    def test_ssh_port_abspath_explicit(self):
+    def test_ssh_port(self):
         c, path = get_transport_and_path(
-            'git+ssh://foo.com:1234//bar/baz')
+            'git+ssh://foo.com:1234/bar/baz')
         self.assertTrue(isinstance(c, SSHGitClient))
         self.assertEqual('foo.com', c.host)
         self.assertEqual(1234, c.port)
@@ -534,7 +534,7 @@ class TestGetTransportAndPathFromUrl(TestCase):
         self.assertEqual('foo.com', c.host)
         self.assertEqual(None, c.port)
         self.assertEqual(None, c.username)
-        self.assertEqual('bar/baz', path)
+        self.assertEqual('/bar/baz', path)
 
     def test_ssh_port_explicit(self):
         c, path = get_transport_and_path_from_url(
@@ -542,23 +542,23 @@ class TestGetTransportAndPathFromUrl(TestCase):
         self.assertTrue(isinstance(c, SSHGitClient))
         self.assertEqual('foo.com', c.host)
         self.assertEqual(1234, c.port)
-        self.assertEqual('bar/baz', path)
+        self.assertEqual('/bar/baz', path)
 
-    def test_ssh_abspath_explicit(self):
+    def test_ssh_homepath(self):
-        c, path = get_transport_and_path_from_url('git+ssh://foo.com//bar/baz')
+        c, path = get_transport_and_path_from_url('git+ssh://foo.com/~/bar/baz')
         self.assertTrue(isinstance(c, SSHGitClient))
         self.assertEqual('foo.com', c.host)
         self.assertEqual(None, c.port)
         self.assertEqual(None, c.username)
-        self.assertEqual('/bar/baz', path)
+        self.assertEqual('/~/bar/baz', path)
 
-    def test_ssh_port_abspath_explicit(self):
+    def test_ssh_port_homepath(self):
         c, path = get_transport_and_path_from_url(
-            'git+ssh://foo.com:1234//bar/baz')
+            'git+ssh://foo.com:1234/~/bar/baz')
         self.assertTrue(isinstance(c, SSHGitClient))
         self.assertEqual('foo.com', c.host)
         self.assertEqual(1234, c.port)
-        self.assertEqual('/bar/baz', path)
+        self.assertEqual('/~/bar/baz', path)
 
     def test_ssh_host_relpath(self):
         self.assertRaises(ValueError, get_transport_and_path_from_url,

dulwich/tests/test_config.py

@@ -155,7 +155,6 @@ class ConfigFileTests(TestCase):
         cf = self.from_file(b"[branch.foo] foo = bar\n")
         self.assertEqual(b"bar", cf.get((b"branch", b"foo"), b"foo"))
 
-    #@expectedFailure
     def test_quoted(self):
         cf = self.from_file(b"""[gui]
 	fontdiff = -family \\\"Ubuntu Mono\\\" -size 11 -weight normal -slant roman -underline 0 -overstrike 0
@@ -164,6 +163,16 @@ class ConfigFileTests(TestCase):
             b'fontdiff': b'-family "Ubuntu Mono" -size 11 -weight normal -slant roman -underline 0 -overstrike 0',
         }}), cf)
 
+    def test_quoted_multiline(self):
+        cf = self.from_file(b"""[alias]
+who = \"!who() {\\
+  git log --no-merges --pretty=format:'%an - %ae' $@ | sort | uniq -c | sort -rn;\\
+};\\
+who\"
+""")
+        self.assertEqual(ConfigFile({(b'alias', ): {
+            b'who': b"!who() {git log --no-merges --pretty=format:'%an - %ae' $@ | sort | uniq -c | sort -rn;};who"}}), cf)
+
 
 class ConfigDictTests(TestCase):
 

dulwich/tests/test_index.py

@@ -368,8 +368,8 @@ class BuildIndexTests(TestCase):
             self.assertEqual(['d'],
                 sorted(os.listdir(os.path.join(repo.path, 'c'))))
 
+    @skipIf(not getattr(os, 'sync', None), 'Requires sync support')
     def test_norewrite(self):
-        sync = getattr(os, 'sync', lambda: os.system('sync'))
         repo_dir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, repo_dir)
         with Repo.init(repo_dir) as repo:
@@ -386,25 +386,25 @@ class BuildIndexTests(TestCase):
             build_index_from_tree(repo.path, repo.index_path(),
                                   repo.object_store, tree.id)
             # Use sync as metadata can be cached on some FS
-            sync()
+            os.sync()
             mtime = os.stat(filea_path).st_mtime
 
             # Test Rewrite
             build_index_from_tree(repo.path, repo.index_path(),
                                   repo.object_store, tree.id)
-            sync()
+            os.sync()
             self.assertEqual(mtime, os.stat(filea_path).st_mtime)
 
             # Modify content
             with open(filea_path, 'wb') as fh:
                 fh.write(b'test a')
-            sync()
+            os.sync()
             mtime = os.stat(filea_path).st_mtime
 
             # Test rewrite
             build_index_from_tree(repo.path, repo.index_path(),
                                   repo.object_store, tree.id)
-            sync()
+            os.sync()
             with open(filea_path, 'rb') as fh:
                 self.assertEqual(b'file a', fh.read())
 
@@ -512,6 +512,47 @@ class BuildIndexTests(TestCase):
             self.assertEqual(index[b'c'][4], S_IFGITLINK)  # mode
             self.assertEqual(index[b'c'][8], c.id)  # sha
 
+    def test_git_submodule_exists(self):
+        repo_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, repo_dir)
+        with Repo.init(repo_dir) as repo:
+            filea = Blob.from_string(b'file alalala')
+
+            subtree = Tree()
+            subtree[b'a'] = (stat.S_IFREG | 0o644, filea.id)
+
+            c = Commit()
+            c.tree = subtree.id
+            c.committer = c.author = b'Somebody <somebody@example.com>'
+            c.commit_time = c.author_time = 42342
+            c.commit_timezone = c.author_timezone = 0
+            c.parents = []
+            c.message = b'Subcommit'
+
+            tree = Tree()
+            tree[b'c'] = (S_IFGITLINK, c.id)
+
+            os.mkdir(os.path.join(repo_dir, 'c'))
+            repo.object_store.add_objects(
+                [(o, None) for o in [tree]])
+
+            build_index_from_tree(repo.path, repo.index_path(),
+                    repo.object_store, tree.id)
+
+            # Verify index entries
+            index = repo.open_index()
+            self.assertEqual(len(index), 1)
+
+            # filea
+            apath = os.path.join(repo.path, 'c/a')
+            self.assertFalse(os.path.exists(apath))
+
+            # dir c
+            cpath = os.path.join(repo.path, 'c')
+            self.assertTrue(os.path.isdir(cpath))
+            self.assertEqual(index[b'c'][4], S_IFGITLINK)  # mode
+            self.assertEqual(index[b'c'][8], c.id)  # sha
+
 
 class GetUnstagedChangesTests(TestCase):
 

dulwich/tests/test_objects.py

@@ -667,6 +667,39 @@ fDeF1m4qYs+cUXKNUZ03
 =X6RT
 -----END PGP SIGNATURE-----""", c.gpgsig)
 
+    def test_parse_header_trailing_newline(self):
+        c = Commit.from_string(b'''\
+tree a7d6277f78d3ecd0230a1a5df6db00b1d9c521ac
+parent c09b6dec7a73760fbdb478383a3c926b18db8bbe
+author Neil Matatall <oreoshake@github.com> 1461964057 -1000
+committer Neil Matatall <oreoshake@github.com> 1461964057 -1000
+gpgsig -----BEGIN PGP SIGNATURE-----
+ 
+ wsBcBAABCAAQBQJXI80ZCRA6pcNDcVZ70gAAarcIABs72xRX3FWeox349nh6ucJK
+ CtwmBTusez2Zwmq895fQEbZK7jpaGO5TRO4OvjFxlRo0E08UFx3pxZHSpj6bsFeL
+ hHsDXnCaotphLkbgKKRdGZo7tDqM84wuEDlh4MwNe7qlFC7bYLDyysc81ZX5lpMm
+ 2MFF1TvjLAzSvkT7H1LPkuR3hSvfCYhikbPOUNnKOo0sYjeJeAJ/JdAVQ4mdJIM0
+ gl3REp9+A+qBEpNQI7z94Pg5Bc5xenwuDh3SJgHvJV6zBWupWcdB3fAkVd4TPnEZ
+ nHxksHfeNln9RKseIDcy4b2ATjhDNIJZARHNfr6oy4u3XPW4svRqtBsLoMiIeuI=
+ =ms6q
+ -----END PGP SIGNATURE-----
+ 
+
+3.3.0 version bump and docs
+''')
+        self.assertEqual([], c.extra)
+        self.assertEqual(b'''\
+-----BEGIN PGP SIGNATURE-----
+
+wsBcBAABCAAQBQJXI80ZCRA6pcNDcVZ70gAAarcIABs72xRX3FWeox349nh6ucJK
+CtwmBTusez2Zwmq895fQEbZK7jpaGO5TRO4OvjFxlRo0E08UFx3pxZHSpj6bsFeL
+hHsDXnCaotphLkbgKKRdGZo7tDqM84wuEDlh4MwNe7qlFC7bYLDyysc81ZX5lpMm
+2MFF1TvjLAzSvkT7H1LPkuR3hSvfCYhikbPOUNnKOo0sYjeJeAJ/JdAVQ4mdJIM0
+gl3REp9+A+qBEpNQI7z94Pg5Bc5xenwuDh3SJgHvJV6zBWupWcdB3fAkVd4TPnEZ
+nHxksHfeNln9RKseIDcy4b2ATjhDNIJZARHNfr6oy4u3XPW4svRqtBsLoMiIeuI=
+=ms6q
+-----END PGP SIGNATURE-----\n''', c.gpgsig)
+
 
 _TREE_ITEMS = {
     b'a.c': (0o100755, b'd80c186a03f423a81b39df39dc87fd269736ca86'),
@@ -918,6 +951,7 @@ class TagParseTests(ShaFileCheckTests):
         x.set_raw_string(self.make_tag_text(tagger=None))
         self.assertEqual(None, x.tagger)
         self.assertEqual(b'v2.6.22-rc7', x.name)
+        self.assertEqual(None, x.tag_time)
 
     def test_parse_no_message(self):
         x = Tag()

dulwich/tests/test_porcelain.py

@@ -172,7 +172,7 @@ class CloneTests(PorcelainTestCase):
         r = porcelain.clone(self.repo.path, target_path,
                             bare=True, errstream=errstream)
         self.assertEqual(r.path, target_path)
-        self.assertEqual(Repo(target_path).head(), c3.id)
+        self.assertRaises(KeyError, Repo(target_path).head)
         self.assertFalse(b'f1' in os.listdir(target_path))
         self.assertFalse(b'f2' in os.listdir(target_path))
 
@@ -183,12 +183,26 @@ class CloneTests(PorcelainTestCase):
 
         (c1, ) = build_commit_graph(self.repo.object_store, commit_spec, trees)
         self.repo.refs[b"refs/heads/master"] = c1.id
+        self.repo.refs[b"HEAD"] = c1.id
         target_path = tempfile.mkdtemp()
         errstream = BytesIO()
         self.addCleanup(shutil.rmtree, target_path)
         self.assertRaises(ValueError, porcelain.clone, self.repo.path,
             target_path, checkout=True, bare=True, errstream=errstream)
 
+    def test_no_head_no_checkout(self):
+        f1_1 = make_object(Blob, data=b'f1')
+        commit_spec = [[1]]
+        trees = {1: [(b'f1', f1_1), (b'f2', f1_1)]}
+
+        (c1, ) = build_commit_graph(self.repo.object_store, commit_spec, trees)
+        self.repo.refs[b"refs/heads/master"] = c1.id
+        target_path = tempfile.mkdtemp()
+        errstream = BytesIO()
+        self.addCleanup(shutil.rmtree, target_path)
+        porcelain.clone(self.repo.path, target_path, checkout=True,
+            errstream=errstream)
+
 
 class InitTests(TestCase):
 
@@ -230,6 +244,14 @@ class AddTests(PorcelainTestCase):
         with open(os.path.join(self.repo.path, 'foo'), 'w') as f:
             f.write("BAR")
         porcelain.add(self.repo.path, paths=["foo"])
+        self.assertIn(b"foo", self.repo.open_index())
+
+    def test_add_file_absolute_path(self):
+        # Absolute paths are (not yet) supported
+        with open(os.path.join(self.repo.path, 'foo'), 'w') as f:
+            f.write("BAR")
+        porcelain.add(self.repo, paths=[os.path.join(self.repo.path, "foo")])
+        self.assertIn(b"foo", self.repo.open_index())
 
 
 class RemoveTests(PorcelainTestCase):
@@ -894,3 +916,20 @@ class LsRemoteTests(PorcelainTestCase):
             b'refs/heads/master': cid,
             b'HEAD': cid},
             porcelain.ls_remote(self.repo.path))
+
+
+class RemoteAddTests(PorcelainTestCase):
+
+    def test_new(self):
+        porcelain.remote_add(
+            self.repo, 'jelmer', 'git://jelmer.uk/code/dulwich')
+        c = self.repo.get_config()
+        self.assertEqual(
+            c.get((b'remote', b'jelmer'), b'url'),
+            b'git://jelmer.uk/code/dulwich')
+
+    def test_exists(self):
+        porcelain.remote_add(
+            self.repo, 'jelmer', 'git://jelmer.uk/code/dulwich')
+        self.assertRaises(porcelain.RemoteExists, porcelain.remote_add,
+            self.repo, 'jelmer', 'git://jelmer.uk/code/dulwich')

dulwich/tests/test_repository.py

@@ -98,6 +98,15 @@ class CreateRepositoryTests(TestCase):
         self._check_repo_contents(repo, False)
 
 
+class MemoryRepoTests(TestCase):
+
+    def test_set_description(self):
+        r = MemoryRepo.init_bare([], {})
+        description = b"Some description"
+        r.set_description(description)
+        self.assertEqual(description, r.get_description())
+
+
 class RepositoryRootTests(TestCase):
 
     def mkdtemp(self):
@@ -804,6 +813,11 @@ class BuildRepoRootTests(TestCase):
         self.assertEqual([self._root_commit], r[commit_sha].parents)
         self.assertEqual(old_refs, r.get_refs())
 
+    def test_stage_absolute(self):
+        r = self._repo
+        os.remove(os.path.join(r.path, 'a'))
+        self.assertRaises(ValueError, r.stage, [os.path.join(r.path, 'a')])
+
     def test_stage_deleted(self):
         r = self._repo
         os.remove(os.path.join(r.path, 'a'))

dulwich/tests/test_walk.py

@@ -51,6 +51,7 @@ from dulwich.tests import TestCase
 from dulwich.tests.utils import (
     F,
     make_object,
+    make_tag,
     build_commit_graph,
     )
 
@@ -105,6 +106,12 @@ class WalkerTest(TestCase):
         actual = list(walker)
         self.assertEqual(expected, actual)
 
+    def test_tag(self):
+        c1, c2, c3 = self.make_linear_commits(3)
+        t2 = make_tag(target=c2)
+        self.store.add_object(t2)
+        self.assertWalkYields([c2, c1], [t2.id])
+
     def test_linear(self):
         c1, c2, c3 = self.make_linear_commits(3)
         self.assertWalkYields([c1], [c1.id])

dulwich/walk.py

@@ -36,6 +36,10 @@ from dulwich.diff_tree import (
 from dulwich.errors import (
     MissingCommitError,
     )
+from dulwich.objects import (
+    Commit,
+    Tag,
+    )
 
 ORDER_DATE = 'date'
 ORDER_TOPO = 'topo'
@@ -136,15 +140,20 @@ class _CommitTimeQueue(object):
         for commit_id in chain(walker.include, walker.excluded):
             self._push(commit_id)
 
-    def _push(self, commit_id):
+    def _push(self, object_id):
         try:
-            commit = self._store[commit_id]
+            obj = self._store[object_id]
         except KeyError:
-            raise MissingCommitError(commit_id)
+            raise MissingCommitError(object_id)
-        if commit_id not in self._pq_set and commit_id not in self._done:
+        if isinstance(obj, Tag):
+            self._push(obj.object[1])
+            return
+        # TODO(jelmer): What to do about non-Commit and non-Tag objects?
+        commit = obj
+        if commit.id not in self._pq_set and commit.id not in self._done:
             heapq.heappush(self._pq, (-commit.commit_time, commit))
-            self._pq_set.add(commit_id)
+            self._pq_set.add(commit.id)
-            self._seen.add(commit_id)
+            self._seen.add(commit.id)
 
     def _exclude_parents(self, commit):
         excluded = self._excluded
@@ -259,7 +268,9 @@ class Walker(object):
         if order not in ALL_ORDERS:
             raise ValueError('Unknown walk order %s' % order)
         self.store = store
-        if not isinstance(include, list):
+        if isinstance(include, bytes):
+            # TODO(jelmer): Really, this should require a single type.
+            # Print deprecation warning here?
             include = [include]
         self.include = include
         self.excluded = set(exclude or [])

setup.cfg

@@ -3,5 +3,4 @@
 [egg_info]
 tag_build = 
 tag_date = 0
-tag_svn_revision = 0
 

setup.py

@@ -9,7 +9,7 @@ except ImportError:
     from distutils.core import setup, Extension
 from distutils.core import Distribution
 
-dulwich_version_string = '0.16.3'
+dulwich_version_string = '0.17.1'
 
 include_dirs = []
 # Windows MSVC support