Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit d49ecc1

Browse files
Merge pull request h5py#716 from aragilar/support-pathlib
Add support for pathlib/PEP519
2 parents 7973fb6 + d3e8c48 commit d49ecc1

File tree

7 files changed

+165
-28
lines changed

7 files changed

+165
-28
lines changed

h5py/_hl/base.py

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@
1515

1616
import posixpath
1717
import os
18-
import sys
1918
import six
2019
from collections import (Mapping, MutableMapping, KeysView,
2120
ValuesView, ItemsView)
2221

22+
from .compat import fspath
23+
from .compat import fsencode
24+
2325
from .. import h5d, h5i, h5r, h5p, h5f, h5t, h5s
2426

2527
# The high-level interface is serialized; every public API function & method
@@ -32,14 +34,10 @@
3234
def is_hdf5(fname):
3335
""" Determine if a file is valid HDF5 (False if it doesn't exist). """
3436
with phil:
35-
fname = os.path.abspath(fname)
37+
fname = os.path.abspath(fspath(fname))
3638

3739
if os.path.isfile(fname):
38-
try:
39-
fname = fname.encode(sys.getfilesystemencoding())
40-
except (UnicodeError, LookupError):
41-
pass
42-
return h5f.is_hdf5(fname)
40+
return h5f.is_hdf5(fsencode(fname))
4341
return False
4442

4543

h5py/_hl/compat.py

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
"""
2+
Compatibility module for high-level h5py
3+
"""
4+
import sys
5+
import six
6+
7+
8+
try:
9+
from os import fspath
10+
except ImportError:
11+
def fspath(path):
12+
"""
13+
Return the string representation of the path.
14+
If str or bytes is passed in, it is returned unchanged.
15+
This code comes from PEP 519, modified to support earlier versions of
16+
python.
17+
18+
This is required for python < 3.6.
19+
"""
20+
if isinstance(path, (six.text_type, six.binary_type)):
21+
return path
22+
23+
# Work from the object's type to match method resolution of other magic
24+
# methods.
25+
path_type = type(path)
26+
try:
27+
return path_type.__fspath__(path)
28+
except AttributeError:
29+
if hasattr(path_type, '__fspath__'):
30+
raise
31+
try:
32+
import pathlib
33+
except ImportError:
34+
pass
35+
else:
36+
if isinstance(path, pathlib.PurePath):
37+
return six.text_type(path)
38+
39+
raise TypeError("expected str, bytes or os.PathLike object, not "
40+
+ path_type.__name__)
41+
42+
# This is from python 3.5 stdlib (hence lacks PEP 519 changes)
43+
# This was introduced into python 3.2, so python < 3.2 does not have this
44+
# Effectively, this is only required for python 2.6 and 2.7, and can be removed
45+
# once support for them is dropped
46+
def _fscodec():
47+
encoding = sys.getfilesystemencoding()
48+
if encoding == 'mbcs':
49+
errors = 'strict'
50+
else:
51+
errors = 'surrogateescape'
52+
53+
def fsencode(filename):
54+
"""
55+
Encode filename to the filesystem encoding with 'surrogateescape' error
56+
handler, return bytes unchanged. On Windows, use 'strict' error handler if
57+
the file system encoding is 'mbcs' (which is the default encoding).
58+
"""
59+
if isinstance(filename, six.binary_type):
60+
return filename
61+
elif isinstance(filename, six.text_type):
62+
return filename.encode(encoding, errors)
63+
else:
64+
raise TypeError("expect bytes or str, not %s" % type(filename).__name__)
65+
66+
def fsdecode(filename):
67+
"""
68+
Decode filename from the filesystem encoding with 'surrogateescape' error
69+
handler, return str unchanged. On Windows, use 'strict' error handler if
70+
the file system encoding is 'mbcs' (which is the default encoding).
71+
"""
72+
if isinstance(filename, six.text_type):
73+
return filename
74+
elif isinstance(filename, six.binary_type):
75+
return filename.decode(encoding, errors)
76+
else:
77+
raise TypeError("expect bytes or str, not %s" % type(filename).__name__)
78+
79+
return fsencode, fsdecode
80+
81+
_fsencode, _fsdecode = _fscodec()
82+
del _fscodec
83+
84+
try:
85+
from os import fsencode
86+
except ImportError:
87+
fsencode = _fsencode
88+
89+
try:
90+
from os import fsdecode
91+
except ImportError:
92+
fsdecode = _fsdecode

h5py/_hl/files.py

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@
1616
import sys
1717
import os
1818

19+
from .compat import fspath
20+
from .compat import fsencode
21+
from .compat import fsdecode
22+
1923
import six
2024

2125
from .base import phil, with_phil
@@ -149,11 +153,7 @@ def attrs(self):
149153
@with_phil
150154
def filename(self):
151155
"""File name on disk"""
152-
name = h5f.get_name(self.fid)
153-
try:
154-
return name.decode(sys.getfilesystemencoding())
155-
except (UnicodeError, LookupError):
156-
return name
156+
return fsdecode(h5f.get_name(self.fid))
157157

158158
@property
159159
@with_phil
@@ -260,13 +260,7 @@ def __init__(self, name, mode=None, driver=None,
260260
if isinstance(name, _objects.ObjectID):
261261
fid = h5i.get_file_id(name)
262262
else:
263-
try:
264-
# If the byte string doesn't match the default
265-
# encoding, just pass it on as-is. Note Unicode
266-
# objects can always be encoded.
267-
name = name.encode(sys.getfilesystemencoding())
268-
except (UnicodeError, LookupError):
269-
pass
263+
name = fsencode(fspath(name))
270264

271265
fapl = make_fapl(driver, libver, **kwds)
272266
fid = make_fid(name, mode, userblock_size, fapl, swmr=swmr)

h5py/_hl/group.py

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@
1616
import posixpath as pp
1717
import six
1818
import numpy
19-
import sys
19+
20+
from .compat import fsdecode
21+
from .compat import fsencode
22+
from .compat import fspath
2023

2124
from .. import h5g, h5i, h5o, h5r, h5t, h5l, h5p
2225
from . import base
@@ -234,11 +237,7 @@ def get(self, name, default=None, getclass=False, getlink=False):
234237
if getclass:
235238
return ExternalLink
236239
filebytes, linkbytes = self.id.links.get_val(self._e(name))
237-
try:
238-
filetext = filebytes.decode(sys.getfilesystemencoding())
239-
except (UnicodeError, LookupError):
240-
filetext = filebytes
241-
return ExternalLink(filetext, self._d(linkbytes))
240+
return ExternalLink(fsdecode(filebytes), self._d(linkbytes))
242241

243242
elif typecode == h5l.TYPE_HARD:
244243
return HardLink if getclass else HardLink()
@@ -281,7 +280,7 @@ def __setitem__(self, name, obj):
281280
lcpl=lcpl, lapl=self._lapl)
282281

283282
elif isinstance(obj, ExternalLink):
284-
self.id.links.create_external(name, self._e(obj.filename),
283+
self.id.links.create_external(name, fsencode(obj.filename),
285284
self._e(obj.path), lcpl=lcpl, lapl=self._lapl)
286285

287286
elif isinstance(obj, numpy.dtype):
@@ -526,7 +525,7 @@ def filename(self):
526525
return self._filename
527526

528527
def __init__(self, filename, path):
529-
self._filename = str(filename)
528+
self._filename = fspath(filename)
530529
self._path = str(path)
531530

532531
def __repr__(self):

h5py/tests/old/test_file.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,20 @@
1616
from __future__ import absolute_import, with_statement
1717

1818
import os, stat
19+
import tempfile
1920

2021
import six
2122

2223
from .common import ut, TestCase, unicode_filenames
2324
from h5py.highlevel import File
2425
import h5py
2526

27+
try:
28+
import pathlib
29+
except ImportError:
30+
pathlib = None
31+
32+
2633
mpi = h5py.get_config().mpi
2734

2835
class TestFileOpen(TestCase):
@@ -542,3 +549,25 @@ def test_close(self):
542549
self.assertFalse(bool(f1.id))
543550
self.assertFalse(bool(g1.id))
544551

552+
@ut.skipIf(pathlib is None, "pathlib module not installed")
553+
class TestPathlibSupport(TestCase):
554+
555+
"""
556+
Check that h5py doesn't break on pathlib
557+
"""
558+
def test_pathlib_accepted_file(self):
559+
""" Check that pathlib is accepted by h5py.File """
560+
with tempfile.NamedTemporaryFile() as f:
561+
path = pathlib.Path(f.name)
562+
with File(path) as f2:
563+
self.assertTrue(True)
564+
565+
def test_pathlib_name_match(self):
566+
""" Check that using pathlib does not affect naming """
567+
with tempfile.NamedTemporaryFile() as f:
568+
path = pathlib.Path(f.name)
569+
with File(path) as h5f1:
570+
pathlib_name = h5f1.filename
571+
with File(f.name) as h5f2:
572+
normal_name = h5f2.filename
573+
self.assertEqual(pathlib_name, normal_name)

h5py/tests/old/test_group.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# -*- coding: utf-8 -*-
12
# This file is part of h5py, a Python interface to the HDF5 library.
23
#
34
# http://www.h5py.org
@@ -21,7 +22,9 @@
2122
import collections
2223
import numpy as np
2324
import os
25+
import os.path
2426
import sys
27+
from tempfile import mkdtemp
2528

2629
import six
2730

@@ -731,6 +734,28 @@ def test_close_file(self):
731734
f2.close()
732735
self.assertFalse(f2)
733736

737+
def test_unicode_encode(self):
738+
"""
739+
Check that external links encode unicode filenames properly
740+
Testing issue #732
741+
"""
742+
ext_filename = os.path.join(mkdtemp(), "α.hdf5")
743+
with File(ext_filename, "w") as ext_file:
744+
ext_file.create_group('external')
745+
self.f['ext'] = ExternalLink(ext_filename, '/external')
746+
747+
def test_unicode_decode(self):
748+
"""
749+
Check that external links decode unicode filenames properly
750+
Testing issue #732
751+
"""
752+
ext_filename = os.path.join(mkdtemp(), "α.hdf5")
753+
with File(ext_filename, "w") as ext_file:
754+
ext_file.create_group('external')
755+
ext_file["external"].attrs["ext_attr"] = "test"
756+
self.f['ext'] = ExternalLink(ext_filename, '/external')
757+
self.assertEqual(self.f["ext"].attrs["ext_attr"], "test")
758+
734759
class TestExtLinkBugs(TestCase):
735760

736761
"""

tox.ini

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[tox]
2-
envlist = {py26,py27,py33,py34}-{test}-{deps,mindeps}
2+
envlist = {py26,py27,py33,py34,py35}-{test}-{deps,mindeps}
33

44
[testenv]
55
deps =

0 commit comments

Comments
 (0)