From 052461d2a9ae2e04e23750fdcd38848e675e6e53 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 16 Jul 2019 14:55:28 -0700 Subject: [PATCH 01/10] ENH: Modify ``Directory`` and ``File`` traits to get along with pathlib Closes #2959 --- nipype/info.py | 1 + nipype/interfaces/base/specs.py | 72 +-- nipype/interfaces/base/support.py | 2 +- nipype/interfaces/base/tests/test_specs.py | 9 +- nipype/interfaces/base/traits_extension.py | 498 ++++++++++++--------- nipype/interfaces/io.py | 21 +- nipype/interfaces/spm/base.py | 36 +- requirements.txt | 1 + 8 files changed, 350 insertions(+), 290 deletions(-) diff --git a/nipype/info.py b/nipype/info.py index d4a5df9cac..75e9bae6d7 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -155,6 +155,7 @@ def get_nipype_gitversion(): 'packaging', 'futures; python_version == "2.7"', 'configparser; python_version <= "3.4"', + 'pathlib2; python_version <= "3.4"', ] TESTS_REQUIRES = [ diff --git a/nipype/interfaces/base/specs.py b/nipype/interfaces/base/specs.py index dbbc816dc9..47fec9ad1d 100644 --- a/nipype/interfaces/base/specs.py +++ b/nipype/interfaces/base/specs.py @@ -19,19 +19,26 @@ from builtins import str, bytes from packaging.version import Version +from traits.trait_errors import TraitError +from traits.trait_handlers import TraitDictObject, TraitListObject from ...utils.filemanip import md5, hash_infile, hash_timestamp, to_str from .traits_extension import ( traits, Undefined, isdefined, - TraitError, - TraitDictObject, - TraitListObject, has_metadata, ) from ... import config, __version__ + +USING_PATHLIB2 = False +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path # noqa + USING_PATHLIB2 = True + FLOAT_FORMAT = '{:.10f}'.format nipype_version = Version(__version__) @@ -314,6 +321,39 @@ def __all__(self): return self.copyable_trait_names() +def _deepcopypatch(self, memo): + """ + Replace the ``__deepcopy__`` member with a traits-friendly implementation. + + A bug in ``__deepcopy__`` for ``HasTraits`` results in weird cloning behaviors. + Occurs for all specs in Python<3 and only for DynamicTraitedSpec in Python>2. + + """ + id_self = id(self) + if id_self in memo: + return memo[id_self] + dup_dict = deepcopy(self.trait_get(), memo) + # access all keys + for key in self.copyable_trait_names(): + if key in self.__dict__.keys(): + _ = getattr(self, key) + # clone once + dup = self.clone_traits(memo=memo) + for key in self.copyable_trait_names(): + try: + _ = getattr(dup, key) + except: + pass + # clone twice + dup = self.clone_traits(memo=memo) + dup.trait_set(**dup_dict) + return dup + + +if USING_PATHLIB2: + BaseTraitedSpec.__deepcopy__ = _deepcopypatch + + class TraitedSpec(BaseTraitedSpec): """ Create a subclass with strict traits. @@ -333,29 +373,9 @@ class DynamicTraitedSpec(BaseTraitedSpec): functioning well together. """ - def __deepcopy__(self, memo): - """ bug in deepcopy for HasTraits results in weird cloning behavior for - added traits - """ - id_self = id(self) - if id_self in memo: - return memo[id_self] - dup_dict = deepcopy(self.trait_get(), memo) - # access all keys - for key in self.copyable_trait_names(): - if key in self.__dict__.keys(): - _ = getattr(self, key) - # clone once - dup = self.clone_traits(memo=memo) - for key in self.copyable_trait_names(): - try: - _ = getattr(dup, key) - except: - pass - # clone twice - dup = self.clone_traits(memo=memo) - dup.trait_set(**dup_dict) - return dup + +if not USING_PATHLIB2: + DynamicTraitedSpec.__deepcopy__ = _deepcopypatch class CommandLineInputSpec(BaseInterfaceInputSpec): diff --git a/nipype/interfaces/base/support.py b/nipype/interfaces/base/support.py index de9d46f61a..0fd1d27674 100644 --- a/nipype/interfaces/base/support.py +++ b/nipype/interfaces/base/support.py @@ -278,7 +278,7 @@ def _inputs_help(cls): >>> from nipype.interfaces.afni import GCOR >>> _inputs_help(GCOR) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE - ['Inputs::', '', '\t[Mandatory]', '\tin_file: (an existing file name)', ... + ['Inputs::', '', '\t[Mandatory]', '\tin_file: (a pathlike object or string... """ helpstr = ['Inputs::'] diff --git a/nipype/interfaces/base/tests/test_specs.py b/nipype/interfaces/base/tests/test_specs.py index bab112e96d..b27daea6a8 100644 --- a/nipype/interfaces/base/tests/test_specs.py +++ b/nipype/interfaces/base/tests/test_specs.py @@ -2,9 +2,9 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: from __future__ import print_function, unicode_literals -from future import standard_library import os import warnings +from future import standard_library import pytest @@ -420,7 +420,8 @@ def test_ImageFile(): # setup traits x.add_trait('nifti', nib.ImageFile(types=['nifti1', 'dicom'])) x.add_trait('anytype', nib.ImageFile()) - x.add_trait('newtype', nib.ImageFile(types=['nifti10'])) + with pytest.raises(ValueError): + x.add_trait('newtype', nib.ImageFile(types=['nifti10'])) x.add_trait('nocompress', nib.ImageFile(types=['mgh'], allow_compressed=False)) @@ -428,10 +429,8 @@ def test_ImageFile(): x.nifti = 'test.mgz' x.nifti = 'test.nii' x.anytype = 'test.xml' - with pytest.raises(AttributeError): - x.newtype = 'test.nii' with pytest.raises(nib.TraitError): - x.nocompress = 'test.nii.gz' + x.nocompress = 'test.mgz' x.nocompress = 'test.mgh' diff --git a/nipype/interfaces/base/traits_extension.py b/nipype/interfaces/base/traits_extension.py index 7a464cc557..3538c4e27e 100644 --- a/nipype/interfaces/base/traits_extension.py +++ b/nipype/interfaces/base/traits_extension.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """ @@ -24,28 +23,72 @@ absolute_import) from builtins import str, bytes -import os from collections import Sequence # perform all external trait imports here from traits import __version__ as traits_version import traits.api as traits -from traits.trait_handlers import TraitDictObject, TraitListObject -from traits.trait_errors import TraitError -from traits.trait_base import _Undefined, class_of +from traits.trait_handlers import TraitType, NoDefaultSpecified +from traits.trait_base import _Undefined -from traits.api import BaseUnicode from traits.api import Unicode from future import standard_library +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + if traits_version < '3.7.0': raise ImportError('Traits version 3.7.0 or higher must be installed') standard_library.install_aliases() +IMG_FORMATS = { + 'afni': ('.HEAD', '.BRIK'), + 'cifti2': ('.nii', '.nii.gz'), + 'dicom': ('.dcm', '.IMA', '.tar', '.tar.gz'), + 'gifti': ('.gii', '.gii.gz'), + 'mgh': ('.mgh', '.mgz', '.mgh.gz'), + 'nifti1': ('.nii', '.nii.gz', '.hdr', '.img', '.img.gz'), + 'nifti2': ('.nii', '.nii.gz'), + 'nrrd': ('.nrrd', '.nhdr'), +} +IMG_ZIP_FMT = set(['.nii.gz', 'tar.gz', '.gii.gz', '.mgz', '.mgh.gz', 'img.gz']) + +""" +The functions that pop-up the Traits GUIs, edit_traits and +configure_traits, were failing because all of our inputs default to +Undefined deep and down in traits/ui/wx/list_editor.py it checks for +the len() of the elements of the list. The _Undefined class in traits +does not define the __len__ method and would error. I tried defining +our own Undefined and even sublassing Undefined, but both of those +failed with a TraitError in our initializer when we assign the +Undefined to the inputs because of an incompatible type: + +TraitError: The 'vertical_gradient' trait of a BetInputSpec instance must be \ +a float, but a value of was specified. + +So... in order to keep the same type but add the missing method, I +monkey patched. +""" + + +def _length(self): + return 0 + + +########################################################################## +# Apply monkeypatch here +_Undefined.__len__ = _length +########################################################################## + +Undefined = _Undefined() + class Str(Unicode): - """Replacement for the default traits.Str based in bytes""" + """Replaces the default traits.Str based in bytes.""" # Monkeypatch Str and DictStrStr for Python 2 compatibility @@ -54,244 +97,254 @@ class Str(Unicode): traits.DictStrStr = DictStrStr -class File(BaseUnicode): - """ Defines a trait whose value must be the name of a file. - """ +class BasePath(TraitType): + """Defines a trait whose value must be a valid filesystem path.""" # A description of the type of value this trait accepts: - info_text = 'a file name' - - def __init__(self, - value='', - filter=None, - auto_set=False, - entries=0, - exists=False, - **metadata): - """ Creates a File trait. - - Parameters - ---------- - value : string - The default value for the trait - filter : string - A wildcard string to filter filenames in the file dialog box used by - the attribute trait editor. - auto_set : boolean - Indicates whether the file editor updates the trait value after - every key stroke. - exists : boolean - Indicates whether the trait value must be an existing file or - not. - - Default Value - ------------- - *value* or '' - """ - self.filter = filter - self.auto_set = auto_set - self.entries = entries + info_text = 'a pathlike object or string' + exists = False + pathlike = False + resolve = False + _is_file = False + _is_dir = False + + def __init__(self, value=Undefined, + exists=False, pathlike=False, resolve=False, **metadata): + """Create a BasePath trait.""" self.exists = exists + self.resolve = resolve + self.pathlike = pathlike + if any((exists, self._is_file, self._is_dir)): + self.info_text += ' representing a' + if exists: + self.info_text += 'n existing' + if self._is_file: + self.info_text += ' file' + elif self._is_dir: + self.info_text += ' directory' + else: + self.info_text += ' file or directory' - if exists: - self.info_text = 'an existing file name' + super(BasePath, self).__init__(value, **metadata) - super(File, self).__init__(value, **metadata) + def validate(self, objekt, name, value, return_pathlike=False): + """Validate a value change.""" + try: + value = Path('%s' % value) # Use pathlib's validation + except Exception: + self.error(objekt, name, str(value)) - def validate(self, object, name, value): - """ Validates that a specified value is valid for this trait.""" - validated_value = super(File, self).validate(object, name, value) - if not self.exists: - return validated_value - elif os.path.isfile(value): - return validated_value - else: - raise TraitError( - args='The trait \'{}\' of {} instance is {}, but the path ' - ' \'{}\' does not exist.'.format(name, class_of(object), - self.info_text, value)) + if self.exists: + if self.exists and not value.exists(): + self.error(objekt, name, str(value)) + + if self._is_file and not value.is_file(): + self.error(objekt, name, str(value)) + + if self._is_dir and not value.is_dir(): + self.error(objekt, name, str(value)) + + if self.resolve: + try: + value = value.resolve(strict=self.exists) + except TypeError: + if self.exists: + value = value.resolve() + elif not value.is_absolute(): + value = Path().resolve() / value + + if not return_pathlike and not self.pathlike: + value = str(value) + + return value + + # def get_value(self, objekt, name, trait=None): + # value = super(BasePath, self).get_value(objekt, name) + # if value is Undefined: + # return self.default_value + + # if self.pathlike: + # return value + # return str(value) + + +class Directory(BasePath): + """ + Defines a trait whose value must be a directory path. + + Examples:: + + >>> from nipype.interfaces.base import Directory, TraitedSpec, TraitError + >>> class A(TraitedSpec): + ... foo = Directory(exists=False) + >>> a = A() + >>> a.foo + + + >>> a.foo = '/some/made/out/path' + >>> a.foo + '/some/made/out/path' + + >>> class A(TraitedSpec): + ... foo = Directory(exists=False, resolve=True) + >>> a = A(foo='relative_dir') + >>> a.foo # doctest: +ELLIPSIS + '.../relative_dir' + + >>> class A(TraitedSpec): + ... foo = Directory(exists=True, resolve=True) + >>> a = A() + >>> a.foo = 'relative_dir' # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + TraitError: + + >>> from os import mkdir + >>> mkdir('relative_dir') + >>> a.foo = 'relative_dir' + >>> a.foo # doctest: +ELLIPSIS + '.../relative_dir' + + >>> class A(TraitedSpec): + ... foo = Directory(exists=True, resolve=False) + >>> a = A(foo='relative_dir') + >>> a.foo + 'relative_dir' - self.error(object, name, value) + >>> class A(TraitedSpec): + ... foo = Directory('tmpdir') + >>> a = A() + >>> a.foo # doctest: +ELLIPSIS + -# ------------------------------------------------------------------------------- -# 'Directory' trait -# ------------------------------------------------------------------------------- + >>> class A(TraitedSpec): + ... foo = Directory('tmpdir', usedefault=True) + >>> a = A() + >>> a.foo # doctest: +ELLIPSIS + 'tmpdir' -class Directory(BaseUnicode): """ - Defines a trait whose value must be the name of a directory. + + _is_dir = True + + +class File(BasePath): """ + Defines a trait whose value must be a file path. - # A description of the type of value this trait accepts: - info_text = 'a directory name' - - def __init__(self, - value='', - auto_set=False, - entries=0, - exists=False, - **metadata): - """ Creates a Directory trait. - - Parameters - ---------- - value : string - The default value for the trait - auto_set : boolean - Indicates whether the directory editor updates the trait value - after every key stroke. - exists : boolean - Indicates whether the trait value must be an existing directory or - not. - - Default Value - ------------- - *value* or '' - """ - self.entries = entries - self.auto_set = auto_set - self.exists = exists + >>> from nipype.interfaces.base import File, TraitedSpec, TraitError + >>> class A(TraitedSpec): + ... foo = File() + >>> a = A() + >>> a.foo + - if exists: - self.info_text = 'an existing directory name' + >>> a.foo = '/some/made/out/path/to/file' + >>> a.foo + '/some/made/out/path/to/file' - super(Directory, self).__init__(value, **metadata) + >>> class A(TraitedSpec): + ... foo = File(exists=False, resolve=True) + >>> a = A(foo='idontexist.txt') + >>> a.foo # doctest: +ELLIPSIS + '.../idontexist.txt' - def validate(self, object, name, value): - """ Validates that a specified value is valid for this trait.""" - if isinstance(value, (str, bytes)): - if not self.exists: - return value - if os.path.isdir(value): - return value - else: - raise TraitError( - args='The trait \'{}\' of {} instance is {}, but the path ' - ' \'{}\' does not exist.'.format(name, class_of(object), - self.info_text, value)) - - self.error(object, name, value) - - -# lists of tuples -# each element consists of : -# - uncompressed (tuple[0]) extension -# - compressed (tuple[1]) extension -img_fmt_types = { - 'nifti1': [('.nii', '.nii.gz'), (('.hdr', '.img'), ('.hdr', '.img.gz'))], - 'mgh': [('.mgh', '.mgz'), ('.mgh', '.mgh.gz')], - 'nifti2': [('.nii', '.nii.gz')], - 'cifti2': [('.nii', '.nii.gz')], - 'gifti': [('.gii', '.gii.gz')], - 'dicom': [('.dcm', '.dcm'), ('.IMA', '.IMA'), ('.tar', '.tar.gz')], - 'nrrd': [('.nrrd', 'nrrd'), ('nhdr', 'nhdr')], - 'afni': [('.HEAD', '.HEAD'), ('.BRIK', '.BRIK')] -} + >>> class A(TraitedSpec): + ... foo = File(exists=True, resolve=True) + >>> a = A() + >>> a.foo = 'idontexist.txt' # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + TraitError: + >>> open('idoexist.txt', 'w').close() + >>> a.foo = 'idoexist.txt' + >>> a.foo # doctest: +ELLIPSIS + '.../idoexist.txt' -class ImageFile(File): - """ Defines a trait of specific neuroimaging files """ - - def __init__(self, - value='', - filter=None, - auto_set=False, - entries=0, - exists=False, - types=[], - allow_compressed=True, - **metadata): - """ Trait handles neuroimaging files. - - Parameters - ---------- - types : list - Strings of file format types accepted - compressed : boolean - Indicates whether the file format can compressed - """ - self.types = types - self.allow_compressed = allow_compressed - super(ImageFile, self).__init__(value, filter, auto_set, entries, - exists, **metadata) - - def info(self): - existing = 'n existing' if self.exists else '' - comma = ',' if self.exists and not self.allow_compressed else '' - uncompressed = ' uncompressed' if not self.allow_compressed else '' - with_ext = ' (valid extensions: [{}])'.format( - ', '.join(self.grab_exts())) if self.types else '' - return 'a{existing}{comma}{uncompressed} file{with_ext}'.format( - existing=existing, comma=comma, uncompressed=uncompressed, - with_ext=with_ext) - - def grab_exts(self): - # TODO: file type validation - exts = [] - for fmt in self.types: - if fmt in img_fmt_types: - exts.extend( - sum([[u for u in y[0]] - if isinstance(y[0], tuple) else [y[0]] - for y in img_fmt_types[fmt]], [])) - if self.allow_compressed: - exts.extend( - sum([[u for u in y[-1]] - if isinstance(y[-1], tuple) else [y[-1]] - for y in img_fmt_types[fmt]], [])) - else: - raise AttributeError( - 'Information has not been added for format' - ' type {} yet. Supported formats include: ' - '{}'.format(fmt, ', '.join(img_fmt_types.keys()))) - return list(set(exts)) - - def validate(self, object, name, value): - """ Validates that a specified value is valid for this trait. - """ - validated_value = super(ImageFile, self).validate(object, name, value) - if validated_value and self.types: - _exts = self.grab_exts() - if not any(validated_value.endswith(x) for x in _exts): - raise TraitError( - args="{} is not included in allowed types: {}".format( - validated_value, ', '.join(_exts))) - return validated_value + >>> class A(TraitedSpec): + ... foo = File('idoexist.txt') + >>> a = A() + >>> a.foo + + >>> class A(TraitedSpec): + ... foo = File('idoexist.txt', usedefault=True) + >>> a = A() + >>> a.foo + 'idoexist.txt' -""" -The functions that pop-up the Traits GUIs, edit_traits and -configure_traits, were failing because all of our inputs default to -Undefined deep and down in traits/ui/wx/list_editor.py it checks for -the len() of the elements of the list. The _Undefined class in traits -does not define the __len__ method and would error. I tried defining -our own Undefined and even sublassing Undefined, but both of those -failed with a TraitError in our initializer when we assign the -Undefined to the inputs because of an incompatible type: + >>> class A(TraitedSpec): + ... foo = File(exists=True, resolve=True, extensions=['.txt', 'txt.gz']) + >>> a = A() + >>> a.foo = 'idoexist.badtxt' # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + TraitError: -TraitError: The 'vertical_gradient' trait of a BetInputSpec instance must be a float, but a value of was specified. + >>> a.foo = 'idoexist.txt' + >>> a.foo # doctest: +ELLIPSIS + '.../idoexist.txt' -So... in order to keep the same type but add the missing method, I -monkey patched. -""" + """ + _is_file = True + _exts = None -def length(self): - return 0 + def __init__(self, value=NoDefaultSpecified, exists=False, pathlike=False, + resolve=False, allow_compressed=True, extensions=None, **metadata): + """Create a File trait.""" + if extensions is not None: + if isinstance(extensions, (bytes, str)): + extensions = [extensions] + if allow_compressed is False: + extensions = list(set(extensions) - IMG_ZIP_FMT) -########################################################################## -# Apply monkeypatch here -_Undefined.__len__ = length -########################################################################## + self._exts = sorted(set(['.%s' % ext if not ext.startswith('.') else ext + for ext in extensions])) -Undefined = _Undefined() + super(File, self).__init__(value=value, exists=exists, + pathlike=pathlike, resolve=resolve, **metadata) + + def validate(self, objekt, name, value, return_pathlike=False): + """Validate a value change.""" + value = super(File, self).validate(objekt, name, value, return_pathlike=True) + if self._exts: + ext = ''.join(value.suffixes) + if ext not in self._exts: + self.error(objekt, name, str(value)) + + if not return_pathlike and not self.pathlike: + value = str(value) + + return value -def isdefined(object): - return not isinstance(object, _Undefined) +class ImageFile(File): + """Defines a trait whose value must be a known neuroimaging file.""" + + def __init__(self, value=NoDefaultSpecified, exists=False, + pathlike=False, resolve=False, types=None, **metadata): + """Create an ImageFile trait.""" + extensions = None + if types is not None: + if isinstance(types, (bytes, str)): + types = [types] + + if set(types) - set(IMG_FORMATS.keys()): + invalid = set(types) - set(IMG_FORMATS.keys()) + raise ValueError("""\ +Unknown value(s) %s for metadata type of an ImageFile input.\ +""" % ', '.join(['"%s"' % t for t in invalid])) + extensions = [ext for t in types for ext in IMG_FORMATS[t]] + + super(ImageFile, self).__init__( + value=value, exists=exists, extensions=extensions, + pathlike=pathlike, resolve=resolve, **metadata) + + +def isdefined(objekt): + return not isinstance(objekt, _Undefined) def has_metadata(trait, metadata, value=None, recursive=True): @@ -319,7 +372,7 @@ class MultiObject(traits.List): """ Abstract class - shared functionality of input and output MultiObject """ - def validate(self, object, name, value): + def validate(self, objekt, name, value): # want to treat range and other sequences (except str) as list if not isinstance(value, (str, bytes)) and isinstance( @@ -338,12 +391,12 @@ def validate(self, object, name, value): not isinstance(inner_trait.trait_type, InputMultiObject) and not isinstance(value[0], list)): newvalue = [value] - value = super(MultiObject, self).validate(object, name, newvalue) + value = super(MultiObject, self).validate(objekt, name, newvalue) if value: return value - self.error(object, name, value) + self.error(objekt, name, value) class OutputMultiObject(MultiObject): @@ -379,8 +432,8 @@ class OutputMultiObject(MultiObject): """ - def get(self, object, name): - value = self.get_value(object, name) + def get(self, objekt, name): + value = self.get_value(objekt, name) if len(value) == 0: return Undefined elif len(value) == 1: @@ -388,8 +441,8 @@ def get(self, object, name): else: return value - def set(self, object, name, value): - self.set_value(object, name, value) + def set(self, objekt, name, value): + self.set_value(objekt, name, value) class InputMultiObject(MultiObject): @@ -425,5 +478,6 @@ class InputMultiObject(MultiObject): """ pass + InputMultiPath = InputMultiObject OutputMultiPath = OutputMultiObject diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 60e8b6fafc..4b9afd1ce1 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -1024,15 +1024,18 @@ def _list_outputs(self): def s3tolocal(self, s3path, bkt): import boto # path formatting - if not os.path.split(self.inputs.local_directory)[1] == '': - self.inputs.local_directory += '/' - if not os.path.split(self.inputs.bucket_path)[1] == '': - self.inputs.bucket_path += '/' - if self.inputs.template[0] == '/': - self.inputs.template = self.inputs.template[1:] - - localpath = s3path.replace(self.inputs.bucket_path, - self.inputs.local_directory) + local_directory = str(self.inputs.local_directory) + bucket_path = str(self.inputs.bucket_path) + template = str(self.inputs.template) + if not os.path.split(local_directory)[1] == '': + local_directory += '/' + if not os.path.split(bucket_path)[1] == '': + bucket_path += '/' + if template[0] == '/': + template = template[1:] + + localpath = s3path.replace(bucket_path, + local_directory) localdir = os.path.split(localpath)[0] if not os.path.exists(localdir): os.makedirs(localdir) diff --git a/nipype/interfaces/spm/base.py b/nipype/interfaces/spm/base.py index fd93dfc522..bbabbf42ba 100644 --- a/nipype/interfaces/spm/base.py +++ b/nipype/interfaces/spm/base.py @@ -32,6 +32,7 @@ from ..base import (BaseInterface, traits, isdefined, InputMultiPath, BaseInterfaceInputSpec, Directory, Undefined, ImageFile, PackageInfo) +from ..base.traits_extension import NoDefaultSpecified from ..matlab import MatlabCommand from ...external.due import due, Doi, BibTeX @@ -597,30 +598,11 @@ def _make_matlab_command(self, contents, postscript=None): class ImageFileSPM(ImageFile): - """ - Defines an ImageFile trait specific to SPM interfaces. - """ - - def __init__(self, - value='', - filter=None, - auto_set=False, - entries=0, - exists=False, - types=['nifti1', 'nifti2'], - allow_compressed=False, - **metadata): - """ Trait handles neuroimaging files. - - Parameters - ---------- - types : list - Strings of file format types accepted - compressed : boolean - Indicates whether the file format can compressed - """ - self.types = types - self.allow_compressed = allow_compressed - super(ImageFileSPM, - self).__init__(value, filter, auto_set, entries, exists, types, - allow_compressed, **metadata) + """Defines a trait whose value must be a NIfTI file.""" + + def __init__(self, value=NoDefaultSpecified, exists=False, + pathlike=False, resolve=False, **metadata): + """Create an ImageFileSPM trait.""" + super(ImageFileSPM, self).__init__( + value=value, exists=exists, types=['nifti1', 'nifti2'], + pathlike=pathlike, resolve=resolve, **metadata) diff --git a/requirements.txt b/requirements.txt index 0d951f49c0..a5cd2d8cd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ neurdflib click>=6.6.0 funcsigs configparser +pathlib2 pydotplus pydot>=1.2.3 packaging From b76860dd068466c4a0ede3bf75df68f37591d8e9 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 16 Jul 2019 15:06:16 -0700 Subject: [PATCH 02/10] sty: better placement of imports from enthought.traits --- nipype/interfaces/base/__init__.py | 5 ++++- nipype/interfaces/base/core.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/nipype/interfaces/base/__init__.py b/nipype/interfaces/base/__init__.py index 2284c1763a..30d44db56a 100644 --- a/nipype/interfaces/base/__init__.py +++ b/nipype/interfaces/base/__init__.py @@ -8,6 +8,9 @@ This module defines the API of all nipype interfaces. """ +from traits.trait_handlers import TraitDictObject, TraitListObject +from traits.trait_errors import TraitError + from .core import (Interface, BaseInterface, SimpleInterface, CommandLine, StdOutCommandLine, MpiCommandLine, SEMLikeCommandLine, LibraryBaseInterface, PackageInfo) @@ -17,7 +20,7 @@ StdOutCommandLineInputSpec) from .traits_extension import ( - traits, Undefined, TraitDictObject, TraitListObject, TraitError, isdefined, + traits, Undefined, isdefined, File, Directory, Str, DictStrStr, has_metadata, ImageFile, OutputMultiObject, InputMultiObject, OutputMultiPath, InputMultiPath) diff --git a/nipype/interfaces/base/core.py b/nipype/interfaces/base/core.py index 77ab6cf398..0011c925dd 100644 --- a/nipype/interfaces/base/core.py +++ b/nipype/interfaces/base/core.py @@ -27,6 +27,7 @@ import simplejson as json from dateutil.parser import parse as parseutc from future import standard_library +from traits.trait_errors import TraitError from ... import config, logging, LooseVersion from ...utils.provenance import write_provenance @@ -37,7 +38,7 @@ from ...external.due import due -from .traits_extension import traits, isdefined, TraitError +from .traits_extension import traits, isdefined from .specs import (BaseInterfaceInputSpec, CommandLineInputSpec, StdOutCommandLineInputSpec, MpiCommandLineInputSpec, get_filecopy_info) From e993a7f1ad4d518a4a3b1391882aaf2a5de90c52 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 16 Jul 2019 17:47:15 -0700 Subject: [PATCH 03/10] enh(filemanip): Solidify a patched version of ``pathlib.Path`` for internal consumption. --- nipype/interfaces/base/specs.py | 10 +---- nipype/interfaces/base/traits_extension.py | 24 +---------- nipype/utils/filemanip.py | 47 +++++++++++++++++++++- nipype/utils/tests/test_filemanip.py | 19 ++++++++- 4 files changed, 67 insertions(+), 33 deletions(-) diff --git a/nipype/interfaces/base/specs.py b/nipype/interfaces/base/specs.py index 47fec9ad1d..fb0f3295f0 100644 --- a/nipype/interfaces/base/specs.py +++ b/nipype/interfaces/base/specs.py @@ -21,7 +21,8 @@ from traits.trait_errors import TraitError from traits.trait_handlers import TraitDictObject, TraitListObject -from ...utils.filemanip import md5, hash_infile, hash_timestamp, to_str +from ...utils.filemanip import ( + md5, hash_infile, hash_timestamp, to_str, USING_PATHLIB2) from .traits_extension import ( traits, Undefined, @@ -32,13 +33,6 @@ from ... import config, __version__ -USING_PATHLIB2 = False -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path # noqa - USING_PATHLIB2 = True - FLOAT_FORMAT = '{:.10f}'.format nipype_version = Version(__version__) diff --git a/nipype/interfaces/base/traits_extension.py b/nipype/interfaces/base/traits_extension.py index 3538c4e27e..ce4c43e4cc 100644 --- a/nipype/interfaces/base/traits_extension.py +++ b/nipype/interfaces/base/traits_extension.py @@ -34,10 +34,7 @@ from traits.api import Unicode from future import standard_library -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from ...utils.filemanip import Path if traits_version < '3.7.0': @@ -145,35 +142,18 @@ def validate(self, objekt, name, value, return_pathlike=False): self.error(objekt, name, str(value)) if self.resolve: - try: - value = value.resolve(strict=self.exists) - except TypeError: - if self.exists: - value = value.resolve() - elif not value.is_absolute(): - value = Path().resolve() / value + value = value.resolve(strict=self.exists) if not return_pathlike and not self.pathlike: value = str(value) return value - # def get_value(self, objekt, name, trait=None): - # value = super(BasePath, self).get_value(objekt, name) - # if value is Undefined: - # return self.default_value - - # if self.pathlike: - # return value - # return str(value) - class Directory(BasePath): """ Defines a trait whose value must be a directory path. - Examples:: - >>> from nipype.interfaces.base import Directory, TraitedSpec, TraitError >>> class A(TraitedSpec): ... foo = Directory(exists=False) diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index d8a65a6712..3012fd6e56 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -21,7 +21,6 @@ import contextlib import posixpath import simplejson as json -import numpy as np from builtins import str, bytes, open @@ -40,8 +39,52 @@ PY3 = sys.version_info[0] >= 3 -class FileNotFoundError(Exception): + +class FileNotFoundError(OSError): + """Defines the expception for Python 2.""" + + def __init__(self, path): + """Initialize the exception.""" + super(FileNotFoundError, self).__init__( + 2, 'No such file or directory', '%s' % path) + + +USING_PATHLIB2 = False +USING_PATHLIB_PATCHED = False +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + USING_PATHLIB2 = True + +try: + Path('/invented/file/path').resolve(strict=True) +except TypeError: + def _patch_resolve(self, strict=False): + """Add the argument strict to signature in Python>3,<3.6.""" + resolved = Path().old_resolve() / self + + if strict and not resolved.exists(): + raise FileNotFoundError(resolved) + return resolved + + Path.old_resolve = Path.resolve + Path.resolve = _patch_resolve + USING_PATHLIB_PATCHED = True +except FileNotFoundError: pass +except OSError: + def _patch_resolve(self, strict=False): + """Add the argument strict to signature for pathlib2.""" + try: + resolved = self.old_resolve(strict=strict) + except OSError: + raise FileNotFoundError(self.old_resolve()) + + return resolved + + Path.old_resolve = Path.resolve + Path.resolve = _patch_resolve def split_filename(fname): diff --git a/nipype/utils/tests/test_filemanip.py b/nipype/utils/tests/test_filemanip.py index b9a413d557..5d064a3817 100644 --- a/nipype/utils/tests/test_filemanip.py +++ b/nipype/utils/tests/test_filemanip.py @@ -16,7 +16,7 @@ check_forhash, _parse_mount_table, _cifs_table, on_cifs, copyfile, copyfiles, ensure_list, simplify_list, check_depends, split_filename, get_related_files, indirectory, - loadpkl, loadcrash, savepkl) + loadpkl, loadcrash, savepkl, FileNotFoundError, Path) def _ignore_atime(stat): @@ -570,3 +570,20 @@ def test_unversioned_pklization(tmpdir): with pytest.raises(Exception): with mock.patch('nipype.utils.tests.test_filemanip.Pickled', PickledBreaker): loadpkl('./pickled.pkz', versioning=True) + + +def test_Path_strict_resolve(tmpdir): + """Check the monkeypatch to test strict resolution of Path.""" + tmpdir.chdir() + + # Default strict=False should work out out of the box + testfile = Path('somefile.txt') + assert '%s/somefile.txt' % tmpdir == '%s' % testfile.resolve() + + # Switching to strict=True must raise FileNotFoundError (also in Python2) + with pytest.raises(FileNotFoundError): + testfile.resolve(strict=True) + + # If the file is created, it should not raise + testfile.write_text('') + assert '%s/somefile.txt' % tmpdir == '%s' % testfile.resolve(strict=True) From 06ba99f6722f5a224e8ffaa71ac8ae3b2957ef1c Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 16 Jul 2019 18:34:52 -0700 Subject: [PATCH 04/10] fix(py2/34): address compatibility issues --- nipype/interfaces/base/traits_extension.py | 9 ++++++--- nipype/utils/filemanip.py | 4 ++-- nipype/utils/tests/test_filemanip.py | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/nipype/interfaces/base/traits_extension.py b/nipype/interfaces/base/traits_extension.py index ce4c43e4cc..652ca5f772 100644 --- a/nipype/interfaces/base/traits_extension.py +++ b/nipype/interfaces/base/traits_extension.py @@ -33,9 +33,10 @@ from traits.api import Unicode from future import standard_library +from ...utils.filemanip import Path, USING_PATHLIB2 -from ...utils.filemanip import Path - +if USING_PATHLIB2: + from future.types.newstr import newstr if traits_version < '3.7.0': raise ImportError('Traits version 3.7.0 or higher must be installed') @@ -127,7 +128,9 @@ def __init__(self, value=Undefined, def validate(self, objekt, name, value, return_pathlike=False): """Validate a value change.""" try: - value = Path('%s' % value) # Use pathlib's validation + if USING_PATHLIB2 and isinstance(value, newstr): + value = '%s' % value # pathlib2 doesn't like newstr + value = Path(value) # Use pathlib's validation except Exception: self.error(objekt, name, str(value)) diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index 3012fd6e56..7403671a48 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -41,7 +41,7 @@ class FileNotFoundError(OSError): - """Defines the expception for Python 2.""" + """Defines the exception for Python 2.""" def __init__(self, path): """Initialize the exception.""" @@ -75,7 +75,7 @@ def _patch_resolve(self, strict=False): pass except OSError: def _patch_resolve(self, strict=False): - """Add the argument strict to signature for pathlib2.""" + """Raise FileNotFoundError instead of OSError with pathlib2.""" try: resolved = self.old_resolve(strict=strict) except OSError: diff --git a/nipype/utils/tests/test_filemanip.py b/nipype/utils/tests/test_filemanip.py index 5d064a3817..7eaa8b9c86 100644 --- a/nipype/utils/tests/test_filemanip.py +++ b/nipype/utils/tests/test_filemanip.py @@ -585,5 +585,5 @@ def test_Path_strict_resolve(tmpdir): testfile.resolve(strict=True) # If the file is created, it should not raise - testfile.write_text('') + open('somefile.txt', 'w').close() assert '%s/somefile.txt' % tmpdir == '%s' % testfile.resolve(strict=True) From 5e69e7a552c34ff5f0ba1e608c4af2cdd897baae Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 17 Jul 2019 06:50:33 -0700 Subject: [PATCH 05/10] Update nipype/interfaces/spm/base.py [skip ci] Co-Authored-By: Satrajit Ghosh --- nipype/interfaces/spm/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/interfaces/spm/base.py b/nipype/interfaces/spm/base.py index bbabbf42ba..98d53b761b 100644 --- a/nipype/interfaces/spm/base.py +++ b/nipype/interfaces/spm/base.py @@ -604,5 +604,5 @@ def __init__(self, value=NoDefaultSpecified, exists=False, pathlike=False, resolve=False, **metadata): """Create an ImageFileSPM trait.""" super(ImageFileSPM, self).__init__( - value=value, exists=exists, types=['nifti1', 'nifti2'], + value=value, exists=exists, types=['nifti1', 'nifti2'], allow_compressed=False, pathlike=pathlike, resolve=resolve, **metadata) From 523a21e5dbfc4bcff60f6f8a2e43dc3649857ace Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 17 Jul 2019 06:54:31 -0700 Subject: [PATCH 06/10] Update nipype/interfaces/base/traits_extension.py [skip ci] Co-Authored-By: Chris Markiewicz --- nipype/interfaces/base/traits_extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/interfaces/base/traits_extension.py b/nipype/interfaces/base/traits_extension.py index 652ca5f772..cf1b3353c2 100644 --- a/nipype/interfaces/base/traits_extension.py +++ b/nipype/interfaces/base/traits_extension.py @@ -135,7 +135,7 @@ def validate(self, objekt, name, value, return_pathlike=False): self.error(objekt, name, str(value)) if self.exists: - if self.exists and not value.exists(): + if not value.exists(): self.error(objekt, name, str(value)) if self._is_file and not value.is_file(): From c1d1151bd20d1ab109bda7965b35c72e20054f37 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 08:02:15 -0700 Subject: [PATCH 07/10] fix: make info_text of ``BasePath``s a property (@effigies' suggestion) --- nipype/interfaces/base/traits_extension.py | 28 ++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/nipype/interfaces/base/traits_extension.py b/nipype/interfaces/base/traits_extension.py index cf1b3353c2..4d3538fd41 100644 --- a/nipype/interfaces/base/traits_extension.py +++ b/nipype/interfaces/base/traits_extension.py @@ -99,30 +99,34 @@ class BasePath(TraitType): """Defines a trait whose value must be a valid filesystem path.""" # A description of the type of value this trait accepts: - info_text = 'a pathlike object or string' exists = False pathlike = False resolve = False _is_file = False _is_dir = False + @property + def info_text(self): + """Create the trait's general description.""" + info_text = 'a pathlike object or string' + if any((self.exists, self._is_file, self._is_dir)): + info_text += ' representing a' + if self.exists: + info_text += 'n existing' + if self._is_file: + info_text += ' file' + elif self._is_dir: + info_text += ' directory' + else: + info_text += ' file or directory' + return info_text + def __init__(self, value=Undefined, exists=False, pathlike=False, resolve=False, **metadata): """Create a BasePath trait.""" self.exists = exists self.resolve = resolve self.pathlike = pathlike - if any((exists, self._is_file, self._is_dir)): - self.info_text += ' representing a' - if exists: - self.info_text += 'n existing' - if self._is_file: - self.info_text += ' file' - elif self._is_dir: - self.info_text += ' directory' - else: - self.info_text += ' file or directory' - super(BasePath, self).__init__(value, **metadata) def validate(self, objekt, name, value, return_pathlike=False): From 6c3858ab81c6d375a53050cb89c896f94c86b916 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 08:07:35 -0700 Subject: [PATCH 08/10] fix: remove pathlike metadata as it wasn't useful --- nipype/interfaces/base/traits_extension.py | 21 +++++++++------------ nipype/interfaces/spm/base.py | 7 +++---- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/nipype/interfaces/base/traits_extension.py b/nipype/interfaces/base/traits_extension.py index 4d3538fd41..de215beb96 100644 --- a/nipype/interfaces/base/traits_extension.py +++ b/nipype/interfaces/base/traits_extension.py @@ -100,7 +100,6 @@ class BasePath(TraitType): # A description of the type of value this trait accepts: exists = False - pathlike = False resolve = False _is_file = False _is_dir = False @@ -121,12 +120,10 @@ def info_text(self): info_text += ' file or directory' return info_text - def __init__(self, value=Undefined, - exists=False, pathlike=False, resolve=False, **metadata): + def __init__(self, value=Undefined, exists=False, resolve=False, **metadata): """Create a BasePath trait.""" self.exists = exists self.resolve = resolve - self.pathlike = pathlike super(BasePath, self).__init__(value, **metadata) def validate(self, objekt, name, value, return_pathlike=False): @@ -151,7 +148,7 @@ def validate(self, objekt, name, value, return_pathlike=False): if self.resolve: value = value.resolve(strict=self.exists) - if not return_pathlike and not self.pathlike: + if not return_pathlike: value = str(value) return value @@ -277,8 +274,8 @@ class File(BasePath): _is_file = True _exts = None - def __init__(self, value=NoDefaultSpecified, exists=False, pathlike=False, - resolve=False, allow_compressed=True, extensions=None, **metadata): + def __init__(self, value=NoDefaultSpecified, exists=False, resolve=False, + allow_compressed=True, extensions=None, **metadata): """Create a File trait.""" if extensions is not None: if isinstance(extensions, (bytes, str)): @@ -290,8 +287,8 @@ def __init__(self, value=NoDefaultSpecified, exists=False, pathlike=False, self._exts = sorted(set(['.%s' % ext if not ext.startswith('.') else ext for ext in extensions])) - super(File, self).__init__(value=value, exists=exists, - pathlike=pathlike, resolve=resolve, **metadata) + super(File, self).__init__(value=value, exists=exists, resolve=resolve, + extensions=self._exts, **metadata) def validate(self, objekt, name, value, return_pathlike=False): """Validate a value change.""" @@ -301,7 +298,7 @@ def validate(self, objekt, name, value, return_pathlike=False): if ext not in self._exts: self.error(objekt, name, str(value)) - if not return_pathlike and not self.pathlike: + if not return_pathlike: value = str(value) return value @@ -311,7 +308,7 @@ class ImageFile(File): """Defines a trait whose value must be a known neuroimaging file.""" def __init__(self, value=NoDefaultSpecified, exists=False, - pathlike=False, resolve=False, types=None, **metadata): + resolve=False, types=None, **metadata): """Create an ImageFile trait.""" extensions = None if types is not None: @@ -327,7 +324,7 @@ def __init__(self, value=NoDefaultSpecified, exists=False, super(ImageFile, self).__init__( value=value, exists=exists, extensions=extensions, - pathlike=pathlike, resolve=resolve, **metadata) + resolve=resolve, **metadata) def isdefined(objekt): diff --git a/nipype/interfaces/spm/base.py b/nipype/interfaces/spm/base.py index 98d53b761b..fda02d40f1 100644 --- a/nipype/interfaces/spm/base.py +++ b/nipype/interfaces/spm/base.py @@ -600,9 +600,8 @@ def _make_matlab_command(self, contents, postscript=None): class ImageFileSPM(ImageFile): """Defines a trait whose value must be a NIfTI file.""" - def __init__(self, value=NoDefaultSpecified, exists=False, - pathlike=False, resolve=False, **metadata): + def __init__(self, value=NoDefaultSpecified, exists=False, resolve=False, **metadata): """Create an ImageFileSPM trait.""" super(ImageFileSPM, self).__init__( - value=value, exists=exists, types=['nifti1', 'nifti2'], allow_compressed=False, - pathlike=pathlike, resolve=resolve, **metadata) + value=value, exists=exists, types=['nifti1', 'nifti2'], + allow_compressed=False, resolve=resolve, **metadata) From 66c0f9b1ae83f0646ae71b913ef2991938ea21bb Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 08:09:06 -0700 Subject: [PATCH 09/10] fix: remove unused ``USING_PATHLIB_PATCHED`` --- nipype/utils/filemanip.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index 7403671a48..edf23532f5 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -50,7 +50,6 @@ def __init__(self, path): USING_PATHLIB2 = False -USING_PATHLIB_PATCHED = False try: from pathlib import Path except ImportError: @@ -70,7 +69,6 @@ def _patch_resolve(self, strict=False): Path.old_resolve = Path.resolve Path.resolve = _patch_resolve - USING_PATHLIB_PATCHED = True except FileNotFoundError: pass except OSError: From d42db563bd11dadc2a38b8550f43677c0d22fb28 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 08:10:08 -0700 Subject: [PATCH 10/10] sty: make codacy happy (Redefining built-in 'FileNotFoundError') --- nipype/utils/filemanip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index edf23532f5..e19563ed54 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -40,7 +40,7 @@ PY3 = sys.version_info[0] >= 3 -class FileNotFoundError(OSError): +class FileNotFoundError(OSError): # noqa """Defines the exception for Python 2.""" def __init__(self, path):