Fix handling of encoding for tags. #608

NEWS

@@ -5,6 +5,11 @@
   * Add really basic `dulwich.porcelain.fsck` implementation.
     (Jelmer Vernooij)
 
+
+ BUG FIXES
+
+  * Fix handling of encoding for tags. (Jelmer Vernooij, #608)
+
 0.19.2	2018-04-07
 
  BUG FIXES

dulwich/refs.py

@@ -485,12 +485,13 @@ class DiskRefsContainer(RefsContainer):
 
     def subkeys(self, base):
         subkeys = set()
-        path = self.refpath(base)
-        for root, dirs, files in os.walk(path):
-            dir = root[len(path):].strip(os.path.sep).replace(os.path.sep, "/")
+        path = self.refpath(base).encode(sys.getfilesystemencoding())
+        for root, unused_dirs, files in os.walk(path):
+            dir = root[len(path):].strip(b'/')
+            if os.path.sep != '/':
+                dir = dir.replace(os.path.sep, "/")
             for filename in files:
-                refname = (("%s/%s" % (dir, filename))
-                           .strip("/").encode(sys.getfilesystemencoding()))
+                refname = b"/".join(([dir] if dir else []) + [filename])
                 # check_ref_format requires at least one /, so we prepend the
                 # base before calling it.
                 if check_ref_format(base + b'/' + refname):
@@ -505,12 +506,13 @@ class DiskRefsContainer(RefsContainer):
         if os.path.exists(self.refpath(b'HEAD')):
             allkeys.add(b'HEAD')
         path = self.refpath(b'')
-        for root, dirs, files in os.walk(self.refpath(b'refs')):
-            dir = root[len(path):].strip(os.path.sep).replace(os.path.sep, "/")
+        refspath = self.refpath('refs').encode(sys.getfilesystemencoding())
+        for root, unused_dirs, files in os.walk(refspath):
+            dir = root[len(path):]
+            if os.path.sep != '/':
+                dir = dir.strip(os.path.sep).replace(os.path.sep, "/")
             for filename in files:
-                refname = (
-                    "%s/%s" % (dir, filename)).encode(
-                            sys.getfilesystemencoding())
+                refname = b"/".join([dir, filename])
                 if check_ref_format(refname):
                     allkeys.add(refname)
         allkeys.update(self.get_packed_refs())

dulwich/tests/test_refs.py

@@ -485,6 +485,24 @@ class DiskRefsContainerTests(RefsContainerTests, TestCase):
 
         self.assertEqual(expected_refs, self._repo.get_refs())
 
+    def test_cyrillic(self):
+        # reported in https://github.com/dulwich/dulwich/issues/608
+        name = b'\xcd\xee\xe2\xe0\xff\xe2\xe5\xf2\xea\xe01'
+        encoded_ref = b'refs/heads/' + name
+        with open(os.path.join(
+            self._repo.path.encode(
+                sys.getfilesystemencoding()), encoded_ref), 'w') as f:
+            f.write('00' * 20)
+
+        expected_refs = set(_TEST_REFS.keys())
+        expected_refs.add(encoded_ref)
+
+        self.assertEqual(expected_refs,
+                         set(self._repo.refs.allkeys()))
+        self.assertEqual({r[len(b'refs/'):] for r in expected_refs
+                          if r.startswith(b'refs/')},
+                         set(self._repo.refs.subkeys(b'refs/')))
+
 
 _TEST_REFS_SERIALIZED = (
     b'42d06bd4b77fed026b154d16493e5deab78f02ec\t'