Work towards making Dulwich less dependent on the filesystem.

This change is mostly refactoring, and accomplishes the following:
 * Create an abstract base class for Repo that does not need to point
   to a repo on disk.
 * Move some methods in DiskRefsContainer dealing with packs to the
   base class. Packs are such a fundamental tool for optimization in
   git that they should be used by non-disk-backed implementations.
 * Change GitBackend to take in a repo rather than a directory as an
   argument.

Change-Id: Ib2f12855b34c60de48d2b777fbca32f9c2317c8c

bin/dul-daemon

@@ -18,6 +18,7 @@
 # MA  02110-1301, USA.
 
 import sys
+from dulwich.repo import Repo
 from dulwich.server import GitBackend, TCPGitServer
 
 if __name__ == "__main__":
@@ -25,6 +26,6 @@ if __name__ == "__main__":
     if len(sys.argv) > 1:
         gitdir = sys.argv[1]
 
-    backend = GitBackend(gitdir)
+    backend = GitBackend(Repo(gitdir))
     server = TCPGitServer(backend, 'localhost')
     server.serve_forever()

bin/dul-receive-pack

@@ -18,6 +18,7 @@
 # MA  02110-1301, USA.
 
 import sys
+from dulwich.repo import Repo
 from dulwich.server import GitBackend, ReceivePackHandler
 
 def send_fn(data):
@@ -29,6 +30,6 @@ if __name__ == "__main__":
     if len(sys.argv) > 1:
         gitdir = sys.argv[1]
 
-    backend = GitBackend(gitdir)
+    backend = GitBackend(Repo(gitdir))
     handler = ReceivePackHandler(backend, sys.stdin.read, send_fn)
     handler.handle()

bin/dul-upload-pack

@@ -18,6 +18,7 @@
 # MA  02110-1301, USA.
 
 import sys
+from dulwich.repo import Repo
 from dulwich.server import GitBackend, UploadPackHandler
 
 def send_fn(data):
@@ -29,6 +30,6 @@ if __name__ == "__main__":
     if len(sys.argv) > 1:
         gitdir = sys.argv[1]
 
-    backend = GitBackend(gitdir)
+    backend = GitBackend(Repo(gitdir))
     handler = UploadPackHandler(backend, sys.stdin.read, send_fn)
     handler.handle()

dulwich/object_store.py

@@ -66,9 +66,25 @@ class BaseObjectStore(object):
         """
         return ObjectStoreIterator(self, shas)
 
+    def contains_loose(self, sha):
+        """Check if a particular object is present by SHA1 and is loose."""
+        raise NotImplementedError(self.contains_loose)
+
+    def contains_packed(self, sha):
+        """Check if a particular object is present by SHA1 and is packed."""
+        raise NotImplementedError(self.contains_packed)
+
     def __contains__(self, sha):
-        """Check if a particular object is present by SHA1."""
-        raise NotImplementedError(self.__contains__)
+        """Check if a particular object is present by SHA1.
+
+        This method makes no distinction between loose and packed objects.
+        """
+        return self.contains_packed(sha) or self.contains_loose(sha)
+
+    @property
+    def packs(self):
+        """Iterable of pack objects."""
+        raise NotImplementedError
 
     def get_raw(self, name):
         """Obtain the raw text for an object.
@@ -233,14 +249,15 @@ class DiskObjectStore(BaseObjectStore):
         self._pack_cache = None
         self.pack_dir = os.path.join(self.path, PACKDIR)
 
-    def __contains__(self, sha):
-        """Check if a particular object is present by SHA1."""
+    def contains_loose(self, sha):
+        """Check if a particular object is present by SHA1 and is loose."""
+        return self._get_shafile(sha) is not None
+
+    def contains_packed(self, sha):
+        """Check if a particular object is present by SHA1 and is packed."""
         for pack in self.packs:
             if sha in pack:
                 return True
-        ret = self._get_shafile(sha)
-        if ret is not None:
-            return True
         return False
 
     def __iter__(self):
@@ -428,14 +445,23 @@ class MemoryObjectStore(BaseObjectStore):
         super(MemoryObjectStore, self).__init__()
         self._data = {}
 
-    def __contains__(self, sha):
-        """Check if the object with a particular SHA is present."""
+    def contains_loose(self, sha):
+        """Check if a particular object is present by SHA1 and is loose."""
         return sha in self._data
 
+    def contains_packed(self, sha):
+        """Check if a particular object is present by SHA1 and is packed."""
+        return False
+
     def __iter__(self):
         """Iterate over the SHAs that are present in this store."""
         return self._data.iterkeys()
 
+    @property
+    def packs(self):
+        """List with pack objects."""
+        return []
+
     def get_raw(self, name):
         """Obtain the raw text for an object.
         

dulwich/repo.py

@@ -203,45 +203,18 @@ def read_packed_refs(f):
         yield tuple(l.rstrip("\n").split(" ", 2))
 
 
-class Repo(object):
-    """A local git repository.
-    
-    :ivar refs: Dictionary with the refs in this repository
+
+class BaseRepo(object):
+    """Base class for a git repository.
+
     :ivar object_store: Dictionary-like object for accessing
         the objects
+    :ivar refs: Dictionary-like object with the refs in this repository
     """
 
-    def __init__(self, root):
-        if os.path.isdir(os.path.join(root, ".git", OBJECTDIR)):
-            self.bare = False
-            self._controldir = os.path.join(root, ".git")
-        elif (os.path.isdir(os.path.join(root, OBJECTDIR)) and
-              os.path.isdir(os.path.join(root, REFSDIR))):
-            self.bare = True
-            self._controldir = root
-        else:
-            raise NotGitRepository(root)
-        self.path = root
-        self.refs = DiskRefsContainer(self.controldir())
-        self.object_store = DiskObjectStore(
-            os.path.join(self.controldir(), OBJECTDIR))
-
-    def controldir(self):
-        """Return the path of the control directory."""
-        return self._controldir
-
-    def index_path(self):
-        """Return path to the index file."""
-        return os.path.join(self.controldir(), INDEX_FILENAME)
-
-    def open_index(self):
-        """Open the index for this repository."""
-        from dulwich.index import Index
-        return Index(self.index_path())
-
-    def has_index(self):
-        """Check if an index is present."""
-        return os.path.exists(self.index_path())
+    def __init__(self, object_store, refs):
+        self.object_store = object_store
+        self.refs = refs
 
     def fetch(self, target, determine_wants=None, progress=None):
         """Fetch objects into another repository.
@@ -282,42 +255,11 @@ class Repo(object):
 
     def ref(self, name):
         """Return the SHA1 a ref is pointing to."""
-        try:
-            return self.refs.follow(name)
-        except KeyError:
-            return self.get_packed_refs()[name]
+        raise NotImplementedError(self.refs)
 
     def get_refs(self):
         """Get dictionary with all refs."""
-        ret = {}
-        try:
-            if self.head():
-                ret['HEAD'] = self.head()
-        except KeyError:
-            pass
-        ret.update(self.refs.as_dict())
-        ret.update(self.get_packed_refs())
-        return ret
-
-    def get_packed_refs(self):
-        """Get contents of the packed-refs file.
-
-        :return: Dictionary mapping ref names to SHA1s
-
-        :note: Will return an empty dictionary when no packed-refs file is 
-            present.
-        """
-        path = os.path.join(self.controldir(), 'packed-refs')
-        if not os.path.exists(path):
-            return {}
-        ret = {}
-        f = GitFile(path, 'rb')
-        try:
-            for entry in read_packed_refs(f):
-                ret[entry[1]] = entry[0]
-            return ret
-        finally:
-            f.close()
+        raise NotImplementedError(self.get_refs)
 
     def head(self):
         """Return the SHA1 pointed at by HEAD."""
@@ -393,9 +335,6 @@ class Repo(object):
         history.reverse()
         return history
 
-    def __repr__(self):
-        return "<Repo at %r>" % self.path
-
     def __getitem__(self, name):
         if len(name) in (20, 40):
             return self.object_store[name]
@@ -416,6 +355,87 @@ class Repo(object):
             del self.refs[name]
         raise ValueError(name)
 
+
+class Repo(BaseRepo):
+    """A git repository backed by local disk."""
+
+    def __init__(self, root):
+        if os.path.isdir(os.path.join(root, ".git", OBJECTDIR)):
+            self.bare = False
+            self._controldir = os.path.join(root, ".git")
+        elif (os.path.isdir(os.path.join(root, OBJECTDIR)) and
+              os.path.isdir(os.path.join(root, REFSDIR))):
+            self.bare = True
+            self._controldir = root
+        else:
+            raise NotGitRepository(root)
+        self.path = root
+        object_store = DiskObjectStore(
+            os.path.join(self.controldir(), OBJECTDIR))
+        refs = DiskRefsContainer(self.controldir())
+        BaseRepo.__init__(self, object_store, refs)
+
+    def controldir(self):
+        """Return the path of the control directory."""
+        return self._controldir
+
+    def index_path(self):
+        """Return path to the index file."""
+        return os.path.join(self.controldir(), INDEX_FILENAME)
+
+    def open_index(self):
+        """Open the index for this repository."""
+        from dulwich.index import Index
+        return Index(self.index_path())
+
+    def has_index(self):
+        """Check if an index is present."""
+        return os.path.exists(self.index_path())
+
+    def ref(self, name):
+        """Return the SHA1 a ref is pointing to."""
+        try:
+            return self.refs.follow(name)
+        except KeyError:
+            return self.get_packed_refs()[name]
+
+    def get_refs(self):
+        """Get dictionary with all refs."""
+        # TODO: move to base class after merging RefsContainer changes
+        ret = {}
+        try:
+            if self.head():
+                ret['HEAD'] = self.head()
+        except KeyError:
+            pass
+        ret.update(self.refs.as_dict())
+        ret.update(self.get_packed_refs())
+        return ret
+
+    def get_packed_refs(self):
+        """Get contents of the packed-refs file.
+
+        :return: Dictionary mapping ref names to SHA1s
+
+        :note: Will return an empty dictionary when no packed-refs file is
+            present.
+        """
+        # TODO: move to base class after merging RefsContainer changes
+        path = os.path.join(self.controldir(), 'packed-refs')
+        if not os.path.exists(path):
+            return {}
+        ret = {}
+        f = open(path, 'rb')
+        try:
+            for entry in read_packed_refs(f):
+                ret[entry[1]] = entry[0]
+            return ret
+        finally:
+            f.close()
+
+    def __repr__(self):
+        return "<Repo at %r>" % self.path
+
     def do_commit(self, committer, message,
                   author=None, commit_timestamp=None,
                   commit_timezone=None, author_timestamp=None, 

dulwich/server.py

@@ -76,14 +76,10 @@ class Backend(object):
 
 class GitBackend(Backend):
 
-    def __init__(self, gitdir=None):
-        self.gitdir = gitdir
-
-        if not self.gitdir:
-            self.gitdir = tempfile.mkdtemp()
-            Repo.create(self.gitdir)
-
-        self.repo = Repo(self.gitdir)
+    def __init__(self, repo=None):
+        if repo is None:
+            repo = Repo(tmpfile.mkdtemp())
+        self.repo = repo
         self.object_store = self.repo.object_store
         self.fetch_objects = self.repo.fetch_objects
         self.get_refs = self.repo.get_refs