diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index 0c73c505f9..ce61883288 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -34,7 +34,7 @@ import _frozen_importlib_external as _bootstrap_external except ImportError: from . import _bootstrap_external - _bootstrap_external._setup(_bootstrap) + _bootstrap_external._set_bootstrap_module(_bootstrap) _bootstrap._bootstrap_external = _bootstrap_external else: _bootstrap_external.__name__ = 'importlib._bootstrap_external' @@ -54,7 +54,6 @@ # Fully bootstrapped at this point, import whatever you like, circular # dependencies and startup overhead minimisation permitting :) -import types import warnings @@ -79,8 +78,8 @@ def find_loader(name, path=None): This function is deprecated in favor of importlib.util.find_spec(). """ - warnings.warn('Deprecated since Python 3.4. ' - 'Use importlib.util.find_spec() instead.', + warnings.warn('Deprecated since Python 3.4 and slated for removal in ' + 'Python 3.12; use importlib.util.find_spec() instead', DeprecationWarning, stacklevel=2) try: loader = sys.modules[name].__loader__ @@ -136,12 +135,13 @@ def reload(module): The module must have been successfully imported before. """ - if not module or not isinstance(module, types.ModuleType): - raise TypeError("reload() argument must be a module") try: name = module.__spec__.name except AttributeError: - name = module.__name__ + try: + name = module.__name__ + except AttributeError: + raise TypeError("reload() argument must be a module") if sys.modules.get(name) is not module: msg = "module {} not in sys.modules" diff --git a/Lib/importlib/_abc.py b/Lib/importlib/_abc.py new file mode 100644 index 0000000000..f80348fc7f --- /dev/null +++ b/Lib/importlib/_abc.py @@ -0,0 +1,54 @@ +"""Subset of importlib.abc used to reduce importlib.util imports.""" +from . import _bootstrap +import abc +import warnings + + +class Loader(metaclass=abc.ABCMeta): + + """Abstract base class for import loaders.""" + + def create_module(self, spec): + """Return a module to initialize and into which to load. + + This method should raise ImportError if anything prevents it + from creating a new module. It may return None to indicate + that the spec should create the new module. + """ + # By default, defer to default semantics for the new module. + return None + + # We don't define exec_module() here since that would break + # hasattr checks we do to support backward compatibility. + + def load_module(self, fullname): + """Return the loaded module. + + The module must be added to sys.modules and have import-related + attributes set properly. The fullname is a str. + + ImportError is raised on failure. + + This method is deprecated in favor of loader.exec_module(). If + exec_module() exists then it is used to provide a backwards-compatible + functionality for this method. + + """ + if not hasattr(self, 'exec_module'): + raise ImportError + # Warning implemented in _load_module_shim(). + return _bootstrap._load_module_shim(self, fullname) + + def module_repr(self, module): + """Return a module's repr. + + Used by the module type when the method does not raise + NotImplementedError. + + This method is deprecated. + + """ + warnings.warn("importlib.abc.Loader.module_repr() is deprecated and " + "slated for removal in Python 3.12", DeprecationWarning) + # The exception will cause ModuleType.__repr__ to ignore this method. + raise NotImplementedError diff --git a/Lib/importlib/_adapters.py b/Lib/importlib/_adapters.py new file mode 100644 index 0000000000..e72edd1070 --- /dev/null +++ b/Lib/importlib/_adapters.py @@ -0,0 +1,83 @@ +from contextlib import suppress + +from . import abc + + +class SpecLoaderAdapter: + """ + Adapt a package spec to adapt the underlying loader. + """ + + def __init__(self, spec, adapter=lambda spec: spec.loader): + self.spec = spec + self.loader = adapter(spec) + + def __getattr__(self, name): + return getattr(self.spec, name) + + +class TraversableResourcesLoader: + """ + Adapt a loader to provide TraversableResources. + """ + + def __init__(self, spec): + self.spec = spec + + def get_resource_reader(self, name): + return DegenerateFiles(self.spec)._native() + + +class DegenerateFiles: + """ + Adapter for an existing or non-existant resource reader + to provide a degenerate .files(). + """ + + class Path(abc.Traversable): + def iterdir(self): + return iter(()) + + def is_dir(self): + return False + + is_file = exists = is_dir # type: ignore + + def joinpath(self, other): + return DegenerateFiles.Path() + + @property + def name(self): + return '' + + def open(self, mode='rb', *args, **kwargs): + raise ValueError() + + def __init__(self, spec): + self.spec = spec + + @property + def _reader(self): + with suppress(AttributeError): + return self.spec.loader.get_resource_reader(self.spec.name) + + def _native(self): + """ + Return the native reader if it supports files(). + """ + reader = self._reader + return reader if hasattr(reader, 'files') else self + + def __getattr__(self, attr): + return getattr(self._reader, attr) + + def files(self): + return DegenerateFiles.Path() + + +def wrap_spec(package): + """ + Construct a package spec with traversable compatibility + on the spec/loader/reader. + """ + return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 0474c0a070..a9500d6dc5 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -20,10 +20,23 @@ # reference any injected objects! This includes not only global code but also # anything specified at the class level. +def _object_name(obj): + try: + return obj.__qualname__ + except AttributeError: + return type(obj).__qualname__ + # Bootstrap-related code ###################################################### +# Modules injected manually by _setup() +_thread = None +_warnings = None +_weakref = None + +# Import done by _install_external_importers() _bootstrap_external = None + def _wrap(new, old): """Simple substitute for functools.update_wrapper.""" for replace in ['__module__', '__name__', '__qualname__', '__doc__']: @@ -67,6 +80,7 @@ def has_deadlock(self): # Deadlock avoidance for concurrent circular imports. me = _thread.get_ident() tid = self.owner + seen = set() while True: lock = _blocking_on.get(tid) if lock is None: @@ -74,6 +88,14 @@ def has_deadlock(self): tid = lock.owner if tid == me: return True + if tid in seen: + # bpo 38091: the chain of tid's we encounter here + # eventually leads to a fixpoint or a cycle, but + # does not reach 'me'. This means we would not + # actually deadlock. This can happen if other + # threads are at the beginning of acquire() below. + return False + seen.add(tid) def acquire(self): """ @@ -254,9 +276,12 @@ def _requires_frozen_wrapper(self, fullname): def _load_module_shim(self, fullname): """Load the specified module into sys.modules and return it. - This method is deprecated. Use loader.exec_module instead. + This method is deprecated. Use loader.exec_module() instead. """ + msg = ("the load_module() method is deprecated and slated for removal in " + "Python 3.12; use exec_module() instead") + _warnings.warn(msg, DeprecationWarning) spec = spec_from_loader(fullname, self) if fullname in sys.modules: module = sys.modules[fullname] @@ -268,26 +293,16 @@ def _load_module_shim(self, fullname): # Module specifications ####################################################### def _module_repr(module): - # The implementation of ModuleType.__repr__(). + """The implementation of ModuleType.__repr__().""" loader = getattr(module, '__loader__', None) - if hasattr(loader, 'module_repr'): - # As soon as BuiltinImporter, FrozenImporter, and NamespaceLoader - # drop their implementations for module_repr. we can add a - # deprecation warning here. + if spec := getattr(module, "__spec__", None): + return _module_repr_from_spec(spec) + elif hasattr(loader, 'module_repr'): try: return loader.module_repr(module) except Exception: pass - try: - spec = module.__spec__ - except AttributeError: - pass - else: - if spec is not None: - return _module_repr_from_spec(spec) - - # We could use module.__class__.__name__ instead of 'module' in the - # various repr permutations. + # Fall through to a catch-all which always succeeds. try: name = module.__name__ except AttributeError: @@ -372,7 +387,7 @@ def __eq__(self, other): self.cached == other.cached and self.has_location == other.has_location) except AttributeError: - return False + return NotImplemented @property def cached(self): @@ -597,9 +612,9 @@ def _exec(spec, module): else: _init_module_attrs(spec, module, override=True) if not hasattr(spec.loader, 'exec_module'): - # (issue19713) Once BuiltinImporter and ExtensionFileLoader - # have exec_module() implemented, we can add a deprecation - # warning here. + msg = (f"{_object_name(spec.loader)}.exec_module() not found; " + "falling back to load_module()") + _warnings.warn(msg, ImportWarning) spec.loader.load_module(name) else: spec.loader.exec_module(module) @@ -612,9 +627,8 @@ def _exec(spec, module): def _load_backward_compatible(spec): - # (issue19713) Once BuiltinImporter and ExtensionFileLoader - # have exec_module() implemented, we can add a deprecation - # warning here. + # It is assumed that all callers have been warned about using load_module() + # appropriately before calling this function. try: spec.loader.load_module(spec.name) except: @@ -653,6 +667,9 @@ def _load_unlocked(spec): if spec.loader is not None: # Not a namespace package. if not hasattr(spec.loader, 'exec_module'): + msg = (f"{_object_name(spec.loader)}.exec_module() not found; " + "falling back to load_module()") + _warnings.warn(msg, ImportWarning) return _load_backward_compatible(spec) module = module_from_spec(spec) @@ -714,6 +731,8 @@ class BuiltinImporter: """ + _ORIGIN = "built-in" + @staticmethod def module_repr(module): """Return repr for the module. @@ -721,14 +740,16 @@ def module_repr(module): The method is deprecated. The import machinery does the job itself. """ - return ''.format(module.__name__) + _warnings.warn("BuiltinImporter.module_repr() is deprecated and " + "slated for removal in Python 3.12", DeprecationWarning) + return f'' @classmethod def find_spec(cls, fullname, path=None, target=None): if path is not None: return None if _imp.is_builtin(fullname): - return spec_from_loader(fullname, cls, origin='built-in') + return spec_from_loader(fullname, cls, origin=cls._ORIGIN) else: return None @@ -741,19 +762,22 @@ def find_module(cls, fullname, path=None): This method is deprecated. Use find_spec() instead. """ + _warnings.warn("BuiltinImporter.find_module() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) spec = cls.find_spec(fullname, path) return spec.loader if spec is not None else None - @classmethod - def create_module(self, spec): + @staticmethod + def create_module(spec): """Create a built-in module""" if spec.name not in sys.builtin_module_names: raise ImportError('{!r} is not a built-in module'.format(spec.name), name=spec.name) return _call_with_frames_removed(_imp.create_builtin, spec) - @classmethod - def exec_module(self, module): + @staticmethod + def exec_module(module): """Exec a built-in module""" _call_with_frames_removed(_imp.exec_builtin, module) @@ -796,6 +820,8 @@ def module_repr(m): The method is deprecated. The import machinery does the job itself. """ + _warnings.warn("FrozenImporter.module_repr() is deprecated and " + "slated for removal in Python 3.12", DeprecationWarning) return ''.format(m.__name__, FrozenImporter._ORIGIN) @classmethod @@ -812,10 +838,13 @@ def find_module(cls, fullname, path=None): This method is deprecated. Use find_spec() instead. """ + _warnings.warn("FrozenImporter.find_module() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) return cls if _imp.is_frozen(fullname) else None - @classmethod - def create_module(cls, spec): + @staticmethod + def create_module(spec): """Use default semantics for module creation.""" @staticmethod @@ -834,6 +863,7 @@ def load_module(cls, fullname): This method is deprecated. Use exec_module() instead. """ + # Warning about deprecation implemented in _load_module_shim(). return _load_module_shim(cls, fullname) @classmethod @@ -874,14 +904,15 @@ def _resolve_name(name, package, level): """Resolve a relative module name to an absolute one.""" bits = package.rsplit('.', level - 1) if len(bits) < level: - raise ValueError('attempted relative import beyond top-level package') + raise ImportError('attempted relative import beyond top-level package') base = bits[0] return '{}.{}'.format(base, name) if name else base def _find_spec_legacy(finder, name, path): - # This would be a good place for a DeprecationWarning if - # we ended up going that route. + msg = (f"{_object_name(finder)}.find_spec() not found; " + "falling back to find_module()") + _warnings.warn(msg, ImportWarning) loader = finder.find_module(name, path) if loader is None: return None @@ -977,7 +1008,12 @@ def _find_and_load_unlocked(name, import_): if parent: # Set the module as an attribute on its parent. parent_module = sys.modules[parent] - setattr(parent_module, name.rpartition('.')[2], module) + child = name.rpartition('.')[2] + try: + setattr(parent_module, child, module) + except AttributeError: + msg = f"Cannot set an attribute on {parent!r} for child module {child!r}" + _warnings.warn(msg, ImportWarning) return module @@ -1150,21 +1186,12 @@ def _setup(sys_module, _imp_module): # Directly load built-in modules needed during bootstrap. self_module = sys.modules[__name__] - for builtin_name in ('_warnings', '_weakref'): + for builtin_name in ('_thread', '_warnings', '_weakref'): if builtin_name not in sys.modules: builtin_module = _builtin_from_name(builtin_name) else: builtin_module = sys.modules[builtin_name] setattr(self_module, builtin_name, builtin_module) - # _thread was part of the above loop, but other parts of the code allow for it - # to be None, so we handle it separately here - builtin_name = '_thread' - if builtin_name in sys.modules: - builtin_module = sys.modules[builtin_name] - else: - builtin_spec = BuiltinImporter.find_spec(builtin_name) - builtin_module = builtin_spec and _load_unlocked(builtin_spec) - setattr(self_module, builtin_name, builtin_module) def _install(sys_module, _imp_module): diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index ccac7eab0a..49bcaea78d 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -19,6 +19,37 @@ # reference any injected objects! This includes not only global code but also # anything specified at the class level. +# Module injected manually by _set_bootstrap_module() +_bootstrap = None + +# Import builtin modules +import _imp +import _io +import sys +import _warnings +import marshal + + +_MS_WINDOWS = (sys.platform == 'win32') +if _MS_WINDOWS: + import nt as _os + import winreg +else: + import posix as _os + + +if _MS_WINDOWS: + path_separators = ['\\', '/'] +else: + path_separators = ['/'] +# Assumption made in _path_join() +assert all(len(sep) == 1 for sep in path_separators) +path_sep = path_separators[0] +path_sep_tuple = tuple(path_separators) +path_separators = ''.join(path_separators) +_pathseps_with_colon = {f':{s}' for s in path_separators} + + # Bootstrap-related code ###################################################### _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', _CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' @@ -34,14 +65,16 @@ def _make_relax_case(): key = b'PYTHONCASEOK' def _relax_case(): - """True if filenames must be checked case-insensitively.""" - return key in _os.environ + """True if filenames must be checked case-insensitively and ignore environment flags are not set.""" + return not sys.flags.ignore_environment and key in _os.environ else: def _relax_case(): """True if filenames must be checked case-insensitively.""" return False return _relax_case +_relax_case = _make_relax_case() + def _pack_uint32(x): """Convert a 32-bit integer to little-endian.""" @@ -59,22 +92,49 @@ def _unpack_uint16(data): return int.from_bytes(data, 'little') -def _path_join(*path_parts): - """Replacement for os.path.join().""" - return path_sep.join([part.rstrip(path_separators) - for part in path_parts if part]) +if _MS_WINDOWS: + def _path_join(*path_parts): + """Replacement for os.path.join().""" + if not path_parts: + return "" + if len(path_parts) == 1: + return path_parts[0] + root = "" + path = [] + for new_root, tail in map(_os._path_splitroot, path_parts): + if new_root.startswith(path_sep_tuple) or new_root.endswith(path_sep_tuple): + root = new_root.rstrip(path_separators) or root + path = [path_sep + tail] + elif new_root.endswith(':'): + if root.casefold() != new_root.casefold(): + # Drive relative paths have to be resolved by the OS, so we reset the + # tail but do not add a path_sep prefix. + root = new_root + path = [tail] + else: + path.append(tail) + else: + root = new_root or root + path.append(tail) + path = [p.rstrip(path_separators) for p in path if p] + if len(path) == 1 and not path[0]: + # Avoid losing the root's trailing separator when joining with nothing + return root + path_sep + return root + path_sep.join(path) + +else: + def _path_join(*path_parts): + """Replacement for os.path.join().""" + return path_sep.join([part.rstrip(path_separators) + for part in path_parts if part]) def _path_split(path): """Replacement for os.path.split().""" - if len(path_separators) == 1: - front, _, tail = path.rpartition(path_sep) - return front, tail - for x in reversed(path): - if x in path_separators: - front, tail = path.rsplit(x, maxsplit=1) - return front, tail - return '', path + i = max(path.rfind(p) for p in path_separators) + if i < 0: + return '', path + return path[:i], path[i + 1:] def _path_stat(path): @@ -108,13 +168,18 @@ def _path_isdir(path): return _path_is_mode_type(path, 0o040000) -def _path_isabs(path): - """Replacement for os.path.isabs. +if _MS_WINDOWS: + def _path_isabs(path): + """Replacement for os.path.isabs.""" + if not path: + return False + root = _os._path_splitroot(path)[0].replace('/', '\\') + return len(root) > 1 and (root.startswith('\\\\') or root.endswith('\\')) - Considers a Windows drive-relative path (no drive, but starts with slash) to - still be "absolute". - """ - return path.startswith(path_separators) or path[1:3] in _pathseps_with_colon +else: + def _path_isabs(path): + """Replacement for os.path.isabs.""" + return path.startswith(path_separators) def _write_atomic(path, data, mode=0o666): @@ -271,6 +336,23 @@ def _write_atomic(path, data, mode=0o666): # Python 3.8b2 3412 (Swap the position of positional args and positional # only args in ast.arguments #37593) # Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) +# Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) +# Python 3.9a0 3421 (simplified bytecode for with blocks #32949) +# Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) +# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) +# Python 3.9a2 3424 (simplify bytecodes for *value unpacking) +# Python 3.9a2 3425 (simplify bytecodes for **value unpacking) +# Python 3.10a1 3430 (Make 'annotations' future by default) +# Python 3.10a1 3431 (New line number table format -- PEP 626) +# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) +# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) +# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) +# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). +# Python 3.10b1 3436 (Add GEN_START bytecode #43683) +# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) +# Python 3.10b1 3438 Safer line number table handling. +# Python 3.10b1 3439 (Add ROT_N) + # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually @@ -279,13 +361,17 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3410).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3439).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' _OPT = 'opt-' -SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed. +SOURCE_SUFFIXES = ['.py'] +if _MS_WINDOWS: + SOURCE_SUFFIXES.append('.pyw') + +EXTENSION_SUFFIXES = _imp.extension_suffixes() BYTECODE_SUFFIXES = ['.pyc'] # Deprecated. @@ -460,15 +546,18 @@ def _check_name_wrapper(self, name=None, *args, **kwargs): raise ImportError('loader for %s cannot handle %s' % (self.name, name), name=name) return method(self, name, *args, **kwargs) - try: + + # FIXME: @_check_name is used to define class methods before the + # _bootstrap module is set by _set_bootstrap_module(). + if _bootstrap is not None: _wrap = _bootstrap._wrap - except NameError: - # XXX yuck + else: def _wrap(new, old): for replace in ['__module__', '__name__', '__qualname__', '__doc__']: if hasattr(old, replace): setattr(new, replace, getattr(old, replace)) new.__dict__.update(old.__dict__) + _wrap(_check_name_wrapper, method) return _check_name_wrapper @@ -480,6 +569,9 @@ def _find_module_shim(self, fullname): This method is deprecated in favor of finder.find_spec(). """ + _warnings.warn("find_module() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) # Call find_loader(). If it returns a string (indicating this # is a namespace package portion), generate a warning and # return None. @@ -651,6 +743,11 @@ def spec_from_file_location(name, location=None, *, loader=None, pass else: location = _os.fspath(location) + if not _path_isabs(location): + try: + location = _path_join(_os.getcwd(), location) + except OSError: + pass # If the location is on the filesystem, but doesn't actually exist, # we could return None here, indicating that the location is not @@ -704,14 +801,14 @@ class WindowsRegistryFinder: REGISTRY_KEY_DEBUG = ( 'Software\\Python\\PythonCore\\{sys_version}' '\\Modules\\{fullname}\\Debug') - DEBUG_BUILD = False # Changed in _setup() + DEBUG_BUILD = (_MS_WINDOWS and '_d.pyd' in EXTENSION_SUFFIXES) - @classmethod - def _open_registry(cls, key): + @staticmethod + def _open_registry(key): try: - return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key) + return winreg.OpenKey(winreg.HKEY_CURRENT_USER, key) except OSError: - return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key) + return winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key) @classmethod def _search_registry(cls, fullname): @@ -723,7 +820,7 @@ def _search_registry(cls, fullname): sys_version='%d.%d' % sys.version_info[:2]) try: with cls._open_registry(key) as hkey: - filepath = _winreg.QueryValue(hkey, '') + filepath = winreg.QueryValue(hkey, '') except OSError: return None return filepath @@ -748,9 +845,12 @@ def find_spec(cls, fullname, path=None, target=None): def find_module(cls, fullname, path=None): """Find module named in the registry. - This method is deprecated. Use exec_module() instead. + This method is deprecated. Use find_spec() instead. """ + _warnings.warn("WindowsRegistryFinder.find_module() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) spec = cls.find_spec(fullname, path) if spec is not None: return spec.loader @@ -783,7 +883,8 @@ def exec_module(self, module): _bootstrap._call_with_frames_removed(exec, code, module.__dict__) def load_module(self, fullname): - """This module is deprecated.""" + """This method is deprecated.""" + # Warning implemented in _load_module_shim(). return _bootstrap._load_module_shim(self, fullname) @@ -958,7 +1059,7 @@ def load_module(self, fullname): """ # The only reason for this method is for the name check. # Issue #14857: Avoid the zero-argument form of super so the implementation - # of that form can be updated without breaking the frozen module + # of that form can be updated without breaking the frozen module. return super(FileLoader, self).load_module(fullname) @_check_name @@ -975,32 +1076,10 @@ def get_data(self, path): with _io.FileIO(path, 'r') as file: return file.read() - # ResourceReader ABC API. - @_check_name def get_resource_reader(self, module): - if self.is_package(module): - return self - return None - - def open_resource(self, resource): - path = _path_join(_path_split(self.path)[0], resource) - return _io.FileIO(path, 'r') - - def resource_path(self, resource): - if not self.is_resource(resource): - raise FileNotFoundError - path = _path_join(_path_split(self.path)[0], resource) - return path - - def is_resource(self, name): - if path_sep in name: - return False - path = _path_join(_path_split(self.path)[0], name) - return _path_isfile(path) - - def contents(self): - return iter(_os.listdir(_path_split(self.path)[0])) + from importlib.readers import FileReader + return FileReader(self) class SourceFileLoader(FileLoader, SourceLoader): @@ -1073,10 +1152,6 @@ def get_source(self, fullname): return None -# Filled in by _setup(). -EXTENSION_SUFFIXES = [] - - class ExtensionFileLoader(FileLoader, _LoaderBasics): """Loader for extension modules. @@ -1097,7 +1172,7 @@ def __hash__(self): return hash(self.name) ^ hash(self.path) def create_module(self, spec): - """Create an uninitialized extension module""" + """Create an unitialized extension module""" module = _bootstrap._call_with_frames_removed( _imp.create_dynamic, spec) _bootstrap._verbose_message('extension module {!r} loaded from {!r}', @@ -1137,10 +1212,15 @@ class _NamespacePath: using path_finder. For top-level modules, the parent module's path is sys.path.""" + # When invalidate_caches() is called, this epoch is incremented + # https://bugs.python.org/issue45703 + _epoch = 0 + def __init__(self, name, path, path_finder): self._name = name self._path = path self._last_parent_path = tuple(self._get_parent_path()) + self._last_epoch = self._epoch self._path_finder = path_finder def _find_parent_path_names(self): @@ -1160,7 +1240,7 @@ def _get_parent_path(self): def _recalculate(self): # If the parent's path has changed, recalculate _path parent_path = tuple(self._get_parent_path()) # Make a copy - if parent_path != self._last_parent_path: + if parent_path != self._last_parent_path or self._epoch != self._last_epoch: spec = self._path_finder(self._name, parent_path) # Note that no changes are made if a loader is returned, but we # do remember the new parent path @@ -1168,6 +1248,7 @@ def _recalculate(self): if spec.submodule_search_locations: self._path = spec.submodule_search_locations self._last_parent_path = parent_path # Save the copy + self._last_epoch = self._epoch return self._path def __iter__(self): @@ -1197,13 +1278,15 @@ class _NamespaceLoader: def __init__(self, name, path, path_finder): self._path = _NamespacePath(name, path, path_finder) - @classmethod - def module_repr(cls, module): + @staticmethod + def module_repr(module): """Return repr for the module. The method is deprecated. The import machinery does the job itself. """ + _warnings.warn("_NamespaceLoader.module_repr() is deprecated and " + "slated for removal in Python 3.12", DeprecationWarning) return ''.format(module.__name__) def is_package(self, fullname): @@ -1230,8 +1313,13 @@ def load_module(self, fullname): # The import system never calls this method. _bootstrap._verbose_message('namespace module loaded with path {!r}', self._path) + # Warning implemented in _load_module_shim(). return _bootstrap._load_module_shim(self, fullname) + def get_resource_reader(self, module): + from importlib.readers import NamespaceReader + return NamespaceReader(self._path) + # Finders ##################################################################### @@ -1239,8 +1327,8 @@ class PathFinder: """Meta path finder for sys.path and package __path__ attributes.""" - @classmethod - def invalidate_caches(cls): + @staticmethod + def invalidate_caches(): """Call the invalidate_caches() method on all path entry finders stored in sys.path_importer_caches (where implemented).""" for name, finder in list(sys.path_importer_cache.items()): @@ -1248,9 +1336,12 @@ def invalidate_caches(cls): del sys.path_importer_cache[name] elif hasattr(finder, 'invalidate_caches'): finder.invalidate_caches() + # Also invalidate the caches of _NamespacePaths + # https://bugs.python.org/issue45703 + _NamespacePath._epoch += 1 - @classmethod - def _path_hooks(cls, path): + @staticmethod + def _path_hooks(path): """Search sys.path_hooks for a finder for 'path'.""" if sys.path_hooks is not None and not sys.path_hooks: _warnings.warn('sys.path_hooks is empty', ImportWarning) @@ -1289,8 +1380,14 @@ def _legacy_get_spec(cls, fullname, finder): # This would be a good place for a DeprecationWarning if # we ended up going that route. if hasattr(finder, 'find_loader'): + msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; " + "falling back to find_loader()") + _warnings.warn(msg, ImportWarning) loader, portions = finder.find_loader(fullname) else: + msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; " + "falling back to find_module()") + _warnings.warn(msg, ImportWarning) loader = finder.find_module(fullname) portions = [] if loader is not None: @@ -1363,13 +1460,16 @@ def find_module(cls, fullname, path=None): This method is deprecated. Use find_spec() instead. """ + _warnings.warn("PathFinder.find_module() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) spec = cls.find_spec(fullname, path) if spec is None: return None return spec.loader - @classmethod - def find_distributions(cls, *args, **kwargs): + @staticmethod + def find_distributions(*args, **kwargs): """ Find distributions. @@ -1401,6 +1501,8 @@ def __init__(self, path, *loader_details): self._loaders = loaders # Base (directory) path self.path = path or '.' + if not _path_isabs(self.path): + self.path = _path_join(_os.getcwd(), self.path) self._path_mtime = -1 self._path_cache = set() self._relaxed_path_cache = set() @@ -1418,6 +1520,9 @@ def find_loader(self, fullname): This method is deprecated. Use find_spec() instead. """ + _warnings.warn("FileFinder.find_loader() is deprecated and " + "slated for removal in Python 3.12; use find_spec() instead", + DeprecationWarning) spec = self.find_spec(fullname) if spec is None: return None, [] @@ -1463,7 +1568,10 @@ def find_spec(self, fullname, target=None): is_namespace = _path_isdir(base_path) # Check for a file w/ a proper suffix exists. for suffix, loader_class in self._loaders: - full_path = _path_join(self.path, tail_module + suffix) + try: + full_path = _path_join(self.path, tail_module + suffix) + except ValueError: + return None _bootstrap._verbose_message('trying {}', full_path, verbosity=2) if cache_module + suffix in cache: if _path_isfile(full_path): @@ -1565,77 +1673,14 @@ def _get_supported_file_loaders(): return [extensions, source, bytecode] -def _setup(_bootstrap_module): - """Setup the path-based importers for importlib by importing needed - built-in modules and injecting them into the global namespace. - - Other components are extracted from the core bootstrap module. - - """ - global sys, _imp, _bootstrap +def _set_bootstrap_module(_bootstrap_module): + global _bootstrap _bootstrap = _bootstrap_module - sys = _bootstrap.sys - _imp = _bootstrap._imp - - # Directly load built-in modules needed during bootstrap. - self_module = sys.modules[__name__] - for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'): - if builtin_name not in sys.modules: - builtin_module = _bootstrap._builtin_from_name(builtin_name) - else: - builtin_module = sys.modules[builtin_name] - setattr(self_module, builtin_name, builtin_module) - - # Directly load the os module (needed during bootstrap). - os_details = ('posix', ['/']), ('nt', ['\\', '/']) - for builtin_os, path_separators in os_details: - # Assumption made in _path_join() - assert all(len(sep) == 1 for sep in path_separators) - path_sep = path_separators[0] - if builtin_os in sys.modules: - os_module = sys.modules[builtin_os] - break - else: - try: - os_module = _bootstrap._builtin_from_name(builtin_os) - break - except ImportError: - continue - else: - raise ImportError('importlib requires posix or nt') - setattr(self_module, '_os', os_module) - setattr(self_module, 'path_sep', path_sep) - setattr(self_module, 'path_separators', ''.join(path_separators)) - setattr(self_module, '_pathseps_with_colon', {f':{s}' for s in path_separators}) - - # Directly load the _thread module (needed during bootstrap). - try: - thread_module = _bootstrap._builtin_from_name('_thread') - except ImportError: - thread_module = None - setattr(self_module, '_thread', thread_module) - - # Directly load the _weakref module (needed during bootstrap). - weakref_module = _bootstrap._builtin_from_name('_weakref') - setattr(self_module, '_weakref', weakref_module) - - # Directly load the winreg module (needed during bootstrap). - if builtin_os == 'nt': - winreg_module = _bootstrap._builtin_from_name('winreg') - setattr(self_module, '_winreg', winreg_module) - - # Constants - setattr(self_module, '_relax_case', _make_relax_case()) - EXTENSION_SUFFIXES.extend(_imp.extension_suffixes()) - if builtin_os == 'nt': - SOURCE_SUFFIXES.append('.pyw') - if '_d.pyd' in EXTENSION_SUFFIXES: - WindowsRegistryFinder.DEBUG_BUILD = True def _install(_bootstrap_module): """Install the path-based import components.""" - _setup(_bootstrap_module) + _set_bootstrap_module(_bootstrap_module) supported_loaders = _get_supported_file_loaders() sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) sys.meta_path.append(PathFinder) diff --git a/Lib/importlib/_common.py b/Lib/importlib/_common.py new file mode 100644 index 0000000000..84144c038c --- /dev/null +++ b/Lib/importlib/_common.py @@ -0,0 +1,118 @@ +import os +import pathlib +import tempfile +import functools +import contextlib +import types +import importlib + +from typing import Union, Any, Optional +from .abc import ResourceReader, Traversable + +from ._adapters import wrap_spec + +Package = Union[types.ModuleType, str] + + +def files(package): + # type: (Package) -> Traversable + """ + Get a Traversable resource from a package + """ + return from_package(get_package(package)) + + +def normalize_path(path): + # type: (Any) -> str + """Normalize a path by ensuring it is a string. + + If the resulting string contains path separators, an exception is raised. + """ + str_path = str(path) + parent, file_name = os.path.split(str_path) + if parent: + raise ValueError(f'{path!r} must be only a file name') + return file_name + + +def get_resource_reader(package): + # type: (types.ModuleType) -> Optional[ResourceReader] + """ + Return the package's loader if it's a ResourceReader. + """ + # We can't use + # a issubclass() check here because apparently abc.'s __subclasscheck__() + # hook wants to create a weak reference to the object, but + # zipimport.zipimporter does not support weak references, resulting in a + # TypeError. That seems terrible. + spec = package.__spec__ + reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore + if reader is None: + return None + return reader(spec.name) # type: ignore + + +def resolve(cand): + # type: (Package) -> types.ModuleType + return cand if isinstance(cand, types.ModuleType) else importlib.import_module(cand) + + +def get_package(package): + # type: (Package) -> types.ModuleType + """Take a package name or module object and return the module. + + Raise an exception if the resolved module is not a package. + """ + resolved = resolve(package) + if wrap_spec(resolved).submodule_search_locations is None: + raise TypeError(f'{package!r} is not a package') + return resolved + + +def from_package(package): + """ + Return a Traversable object for the given package. + + """ + spec = wrap_spec(package) + reader = spec.loader.get_resource_reader(spec.name) + return reader.files() + + +@contextlib.contextmanager +def _tempfile(reader, suffix='', + # gh-93353: Keep a reference to call os.remove() in late Python + # finalization. + *, _os_remove=os.remove): + # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' + # blocks due to the need to close the temporary file to work on Windows + # properly. + fd, raw_path = tempfile.mkstemp(suffix=suffix) + try: + os.write(fd, reader()) + os.close(fd) + del reader + yield pathlib.Path(raw_path) + finally: + try: + _os_remove(raw_path) + except FileNotFoundError: + pass + + +@functools.singledispatch +def as_file(path): + """ + Given a Traversable object, return that object as a + path on the local file system in a context manager. + """ + return _tempfile(path.read_bytes, suffix=path.name) + + +@as_file.register(pathlib.Path) +@contextlib.contextmanager +def _(path): + """ + Degenerate behavior for pathlib.Path objects. + """ + yield path diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 4b2d3de6d9..0b4a3f8071 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -1,5 +1,4 @@ """Abstract base classes related to import.""" -from . import _bootstrap from . import _bootstrap_external from . import machinery try: @@ -10,10 +9,13 @@ _frozen_importlib = None try: import _frozen_importlib_external -except ImportError as exc: +except ImportError: _frozen_importlib_external = _bootstrap_external +from ._abc import Loader import abc import warnings +from typing import BinaryIO, Iterable, Text +from typing import Protocol, runtime_checkable def _register(abstract_cls, *classes): @@ -39,15 +41,27 @@ class Finder(metaclass=abc.ABCMeta): Deprecated since Python 3.3 """ + def __init__(self): + warnings.warn("the Finder ABC is deprecated and " + "slated for removal in Python 3.12; use MetaPathFinder " + "or PathEntryFinder instead", + DeprecationWarning) + @abc.abstractmethod def find_module(self, fullname, path=None): """An abstract method that should find a module. The fullname is a str and the optional path is a str or None. Returns a Loader object or None. """ + warnings.warn("importlib.abc.Finder along with its find_module() " + "method are deprecated and " + "slated for removal in Python 3.12; use " + "MetaPathFinder.find_spec() or " + "PathEntryFinder.find_spec() instead", + DeprecationWarning) -class MetaPathFinder(Finder): +class MetaPathFinder(metaclass=abc.ABCMeta): """Abstract base class for import finders on sys.meta_path.""" @@ -66,8 +80,8 @@ def find_module(self, fullname, path): """ warnings.warn("MetaPathFinder.find_module() is deprecated since Python " - "3.4 in favor of MetaPathFinder.find_spec() " - "(available since 3.4)", + "3.4 in favor of MetaPathFinder.find_spec() and is " + "slated for removal in Python 3.12", DeprecationWarning, stacklevel=2) if not hasattr(self, 'find_spec'): @@ -84,7 +98,7 @@ def invalidate_caches(self): machinery.PathFinder, machinery.WindowsRegistryFinder) -class PathEntryFinder(Finder): +class PathEntryFinder(metaclass=abc.ABCMeta): """Abstract base class for path entry finders used by PathFinder.""" @@ -133,53 +147,6 @@ def invalidate_caches(self): _register(PathEntryFinder, machinery.FileFinder) -class Loader(metaclass=abc.ABCMeta): - - """Abstract base class for import loaders.""" - - def create_module(self, spec): - """Return a module to initialize and into which to load. - - This method should raise ImportError if anything prevents it - from creating a new module. It may return None to indicate - that the spec should create the new module. - """ - # By default, defer to default semantics for the new module. - return None - - # We don't define exec_module() here since that would break - # hasattr checks we do to support backward compatibility. - - def load_module(self, fullname): - """Return the loaded module. - - The module must be added to sys.modules and have import-related - attributes set properly. The fullname is a str. - - ImportError is raised on failure. - - This method is deprecated in favor of loader.exec_module(). If - exec_module() exists then it is used to provide a backwards-compatible - functionality for this method. - - """ - if not hasattr(self, 'exec_module'): - raise ImportError - return _bootstrap._load_module_shim(self, fullname) - - def module_repr(self, module): - """Return a module's repr. - - Used by the module type when the method does not raise - NotImplementedError. - - This method is deprecated. - - """ - # The exception will cause ModuleType.__repr__ to ignore this method. - raise NotImplementedError - - class ResourceLoader(Loader): """Abstract base class for loaders which can return data from their @@ -343,46 +310,133 @@ def set_data(self, path, data): class ResourceReader(metaclass=abc.ABCMeta): - - """Abstract base class to provide resource-reading support. - - Loaders that support resource reading are expected to implement - the ``get_resource_reader(fullname)`` method and have it either return None - or an object compatible with this ABC. - """ + """Abstract base class for loaders to provide resource reading support.""" @abc.abstractmethod - def open_resource(self, resource): + def open_resource(self, resource: Text) -> BinaryIO: """Return an opened, file-like object for binary reading. - The 'resource' argument is expected to represent only a file name - and thus not contain any subdirectory components. - + The 'resource' argument is expected to represent only a file name. If the resource cannot be found, FileNotFoundError is raised. """ + # This deliberately raises FileNotFoundError instead of + # NotImplementedError so that if this method is accidentally called, + # it'll still do the right thing. raise FileNotFoundError @abc.abstractmethod - def resource_path(self, resource): + def resource_path(self, resource: Text) -> Text: """Return the file system path to the specified resource. - The 'resource' argument is expected to represent only a file name - and thus not contain any subdirectory components. - + The 'resource' argument is expected to represent only a file name. If the resource does not exist on the file system, raise FileNotFoundError. """ + # This deliberately raises FileNotFoundError instead of + # NotImplementedError so that if this method is accidentally called, + # it'll still do the right thing. raise FileNotFoundError @abc.abstractmethod - def is_resource(self, name): - """Return True if the named 'name' is consider a resource.""" + def is_resource(self, path: Text) -> bool: + """Return True if the named 'path' is a resource. + + Files are resources, directories are not. + """ raise FileNotFoundError @abc.abstractmethod - def contents(self): - """Return an iterable of strings over the contents of the package.""" - return [] + def contents(self) -> Iterable[str]: + """Return an iterable of entries in `package`.""" + raise FileNotFoundError + + +@runtime_checkable +class Traversable(Protocol): + """ + An object with a subset of pathlib.Path methods suitable for + traversing directories and opening files. + """ + @abc.abstractmethod + def iterdir(self): + """ + Yield Traversable objects in self + """ -_register(ResourceReader, machinery.SourceFileLoader) + def read_bytes(self): + """ + Read contents of self as bytes + """ + with self.open('rb') as strm: + return strm.read() + + def read_text(self, encoding=None): + """ + Read contents of self as text + """ + with self.open(encoding=encoding) as strm: + return strm.read() + + @abc.abstractmethod + def is_dir(self) -> bool: + """ + Return True if self is a dir + """ + + @abc.abstractmethod + def is_file(self) -> bool: + """ + Return True if self is a file + """ + + @abc.abstractmethod + def joinpath(self, child): + """ + Return Traversable child in self + """ + + def __truediv__(self, child): + """ + Return Traversable child in self + """ + return self.joinpath(child) + + @abc.abstractmethod + def open(self, mode='r', *args, **kwargs): + """ + mode may be 'r' or 'rb' to open as text or binary. Return a handle + suitable for reading (same as pathlib.Path.open). + + When opening as text, accepts encoding parameters such as those + accepted by io.TextIOWrapper. + """ + + @abc.abstractproperty + def name(self) -> str: + """ + The base name of this object without any parent references. + """ + + +class TraversableResources(ResourceReader): + """ + The required interface for providing traversable + resources. + """ + + @abc.abstractmethod + def files(self): + """Return a Traversable object for the loaded package.""" + + def open_resource(self, resource): + return self.files().joinpath(resource).open('rb') + + def resource_path(self, resource): + raise FileNotFoundError(resource) + + def is_resource(self, path): + return self.files().joinpath(path).is_file() + + def contents(self): + return (item.name for item in self.files().iterdir()) diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index 1b2b5c9b4f..9a7757fb6e 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -1,7 +1,5 @@ """The machinery of importlib: finders, loaders, hooks, etc.""" -import _imp - from ._bootstrap import ModuleSpec from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py deleted file mode 100644 index 831f593277..0000000000 --- a/Lib/importlib/metadata.py +++ /dev/null @@ -1,566 +0,0 @@ -import io -import os -import re -import abc -import csv -import sys -import email -import pathlib -import zipfile -import operator -import functools -import itertools -import posixpath -import collections - -from configparser import ConfigParser -from contextlib import suppress -from importlib import import_module -from importlib.abc import MetaPathFinder -from itertools import starmap - - -__all__ = [ - 'Distribution', - 'DistributionFinder', - 'PackageNotFoundError', - 'distribution', - 'distributions', - 'entry_points', - 'files', - 'metadata', - 'requires', - 'version', - ] - - -class PackageNotFoundError(ModuleNotFoundError): - """The package was not found.""" - - -class EntryPoint( - collections.namedtuple('EntryPointBase', 'name value group')): - """An entry point as defined by Python packaging conventions. - - See `the packaging docs on entry points - `_ - for more information. - """ - - pattern = re.compile( - r'(?P[\w.]+)\s*' - r'(:\s*(?P[\w.]+))?\s*' - r'(?P\[.*\])?\s*$' - ) - """ - A regular expression describing the syntax for an entry point, - which might look like: - - - module - - package.module - - package.module:attribute - - package.module:object.attribute - - package.module:attr [extra1, extra2] - - Other combinations are possible as well. - - The expression is lenient about whitespace around the ':', - following the attr, and following any extras. - """ - - def load(self): - """Load the entry point from its definition. If only a module - is indicated by the value, return that module. Otherwise, - return the named object. - """ - match = self.pattern.match(self.value) - module = import_module(match.group('module')) - attrs = filter(None, (match.group('attr') or '').split('.')) - return functools.reduce(getattr, attrs, module) - - @property - def extras(self): - match = self.pattern.match(self.value) - return list(re.finditer(r'\w+', match.group('extras') or '')) - - @classmethod - def _from_config(cls, config): - return [ - cls(name, value, group) - for group in config.sections() - for name, value in config.items(group) - ] - - @classmethod - def _from_text(cls, text): - config = ConfigParser(delimiters='=') - # case sensitive: https://stackoverflow.com/q/1611799/812183 - config.optionxform = str - try: - config.read_string(text) - except AttributeError: # pragma: nocover - # Python 2 has no read_string - config.readfp(io.StringIO(text)) - return EntryPoint._from_config(config) - - def __iter__(self): - """ - Supply iter so one may construct dicts of EntryPoints easily. - """ - return iter((self.name, self)) - - def __reduce__(self): - return ( - self.__class__, - (self.name, self.value, self.group), - ) - - -class PackagePath(pathlib.PurePosixPath): - """A reference to a path in a package""" - - def read_text(self, encoding='utf-8'): - with self.locate().open(encoding=encoding) as stream: - return stream.read() - - def read_binary(self): - with self.locate().open('rb') as stream: - return stream.read() - - def locate(self): - """Return a path-like object for this path""" - return self.dist.locate_file(self) - - -class FileHash: - def __init__(self, spec): - self.mode, _, self.value = spec.partition('=') - - def __repr__(self): - return ''.format(self.mode, self.value) - - -class Distribution: - """A Python distribution package.""" - - @abc.abstractmethod - def read_text(self, filename): - """Attempt to load metadata file given by the name. - - :param filename: The name of the file in the distribution info. - :return: The text if found, otherwise None. - """ - - @abc.abstractmethod - def locate_file(self, path): - """ - Given a path to a file in this distribution, return a path - to it. - """ - - @classmethod - def from_name(cls, name): - """Return the Distribution for the given package name. - - :param name: The name of the distribution package to search for. - :return: The Distribution instance (or subclass thereof) for the named - package, if found. - :raises PackageNotFoundError: When the named package's distribution - metadata cannot be found. - """ - for resolver in cls._discover_resolvers(): - dists = resolver(DistributionFinder.Context(name=name)) - dist = next(dists, None) - if dist is not None: - return dist - else: - raise PackageNotFoundError(name) - - @classmethod - def discover(cls, **kwargs): - """Return an iterable of Distribution objects for all packages. - - Pass a ``context`` or pass keyword arguments for constructing - a context. - - :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. - """ - context = kwargs.pop('context', None) - if context and kwargs: - raise ValueError("cannot accept context and kwargs") - context = context or DistributionFinder.Context(**kwargs) - return itertools.chain.from_iterable( - resolver(context) - for resolver in cls._discover_resolvers() - ) - - @staticmethod - def at(path): - """Return a Distribution for the indicated metadata path - - :param path: a string or path-like object - :return: a concrete Distribution instance for the path - """ - return PathDistribution(pathlib.Path(path)) - - @staticmethod - def _discover_resolvers(): - """Search the meta_path for resolvers.""" - declared = ( - getattr(finder, 'find_distributions', None) - for finder in sys.meta_path - ) - return filter(None, declared) - - @property - def metadata(self): - """Return the parsed metadata for this Distribution. - - The returned object will have keys that name the various bits of - metadata. See PEP 566 for details. - """ - text = ( - self.read_text('METADATA') - or self.read_text('PKG-INFO') - # This last clause is here to support old egg-info files. Its - # effect is to just end up using the PathDistribution's self._path - # (which points to the egg-info file) attribute unchanged. - or self.read_text('') - ) - return email.message_from_string(text) - - @property - def version(self): - """Return the 'Version' metadata for the distribution package.""" - return self.metadata['Version'] - - @property - def entry_points(self): - return EntryPoint._from_text(self.read_text('entry_points.txt')) - - @property - def files(self): - """Files in this distribution. - - :return: List of PackagePath for this distribution or None - - Result is `None` if the metadata file that enumerates files - (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is - missing. - Result may be empty if the metadata exists but is empty. - """ - file_lines = self._read_files_distinfo() or self._read_files_egginfo() - - def make_file(name, hash=None, size_str=None): - result = PackagePath(name) - result.hash = FileHash(hash) if hash else None - result.size = int(size_str) if size_str else None - result.dist = self - return result - - return file_lines and list(starmap(make_file, csv.reader(file_lines))) - - def _read_files_distinfo(self): - """ - Read the lines of RECORD - """ - text = self.read_text('RECORD') - return text and text.splitlines() - - def _read_files_egginfo(self): - """ - SOURCES.txt might contain literal commas, so wrap each line - in quotes. - """ - text = self.read_text('SOURCES.txt') - return text and map('"{}"'.format, text.splitlines()) - - @property - def requires(self): - """Generated requirements specified for this Distribution""" - reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() - return reqs and list(reqs) - - def _read_dist_info_reqs(self): - return self.metadata.get_all('Requires-Dist') - - def _read_egg_info_reqs(self): - source = self.read_text('requires.txt') - return source and self._deps_from_requires_text(source) - - @classmethod - def _deps_from_requires_text(cls, source): - section_pairs = cls._read_sections(source.splitlines()) - sections = { - section: list(map(operator.itemgetter('line'), results)) - for section, results in - itertools.groupby(section_pairs, operator.itemgetter('section')) - } - return cls._convert_egg_info_reqs_to_simple_reqs(sections) - - @staticmethod - def _read_sections(lines): - section = None - for line in filter(None, lines): - section_match = re.match(r'\[(.*)\]$', line) - if section_match: - section = section_match.group(1) - continue - yield locals() - - @staticmethod - def _convert_egg_info_reqs_to_simple_reqs(sections): - """ - Historically, setuptools would solicit and store 'extra' - requirements, including those with environment markers, - in separate sections. More modern tools expect each - dependency to be defined separately, with any relevant - extras and environment markers attached directly to that - requirement. This method converts the former to the - latter. See _test_deps_from_requires_text for an example. - """ - def make_condition(name): - return name and 'extra == "{name}"'.format(name=name) - - def parse_condition(section): - section = section or '' - extra, sep, markers = section.partition(':') - if extra and markers: - markers = '({markers})'.format(markers=markers) - conditions = list(filter(None, [markers, make_condition(extra)])) - return '; ' + ' and '.join(conditions) if conditions else '' - - for section, deps in sections.items(): - for dep in deps: - yield dep + parse_condition(section) - - -class DistributionFinder(MetaPathFinder): - """ - A MetaPathFinder capable of discovering installed distributions. - """ - - class Context: - """ - Keyword arguments presented by the caller to - ``distributions()`` or ``Distribution.discover()`` - to narrow the scope of a search for distributions - in all DistributionFinders. - - Each DistributionFinder may expect any parameters - and should attempt to honor the canonical - parameters defined below when appropriate. - """ - - name = None - """ - Specific name for which a distribution finder should match. - A name of ``None`` matches all distributions. - """ - - def __init__(self, **kwargs): - vars(self).update(kwargs) - - @property - def path(self): - """ - The path that a distribution finder should search. - - Typically refers to Python package paths and defaults - to ``sys.path``. - """ - return vars(self).get('path', sys.path) - - @abc.abstractmethod - def find_distributions(self, context=Context()): - """ - Find distributions. - - Return an iterable of all Distribution instances capable of - loading the metadata for packages matching the ``context``, - a DistributionFinder.Context instance. - """ - - -class FastPath: - """ - Micro-optimized class for searching a path for - children. - """ - - def __init__(self, root): - self.root = root - self.base = os.path.basename(root).lower() - - def joinpath(self, child): - return pathlib.Path(self.root, child) - - def children(self): - with suppress(Exception): - return os.listdir(self.root or '') - with suppress(Exception): - return self.zip_children() - return [] - - def zip_children(self): - zip_path = zipfile.Path(self.root) - names = zip_path.root.namelist() - self.joinpath = zip_path.joinpath - - return ( - posixpath.split(child)[0] - for child in names - ) - - def is_egg(self, search): - base = self.base - return ( - base == search.versionless_egg_name - or base.startswith(search.prefix) - and base.endswith('.egg')) - - def search(self, name): - for child in self.children(): - n_low = child.lower() - if (n_low in name.exact_matches - or n_low.startswith(name.prefix) - and n_low.endswith(name.suffixes) - # legacy case: - or self.is_egg(name) and n_low == 'egg-info'): - yield self.joinpath(child) - - -class Prepared: - """ - A prepared search for metadata on a possibly-named package. - """ - normalized = '' - prefix = '' - suffixes = '.dist-info', '.egg-info' - exact_matches = [''][:0] - versionless_egg_name = '' - - def __init__(self, name): - self.name = name - if name is None: - return - self.normalized = name.lower().replace('-', '_') - self.prefix = self.normalized + '-' - self.exact_matches = [ - self.normalized + suffix for suffix in self.suffixes] - self.versionless_egg_name = self.normalized + '.egg' - - -class MetadataPathFinder(DistributionFinder): - @classmethod - def find_distributions(cls, context=DistributionFinder.Context()): - """ - Find distributions. - - Return an iterable of all Distribution instances capable of - loading the metadata for packages matching ``context.name`` - (or all names if ``None`` indicated) along the paths in the list - of directories ``context.path``. - """ - found = cls._search_paths(context.name, context.path) - return map(PathDistribution, found) - - @classmethod - def _search_paths(cls, name, paths): - """Find metadata directories in paths heuristically.""" - return itertools.chain.from_iterable( - path.search(Prepared(name)) - for path in map(FastPath, paths) - ) - - - -class PathDistribution(Distribution): - def __init__(self, path): - """Construct a distribution from a path to the metadata directory. - - :param path: A pathlib.Path or similar object supporting - .joinpath(), __div__, .parent, and .read_text(). - """ - self._path = path - - def read_text(self, filename): - with suppress(FileNotFoundError, IsADirectoryError, KeyError, - NotADirectoryError, PermissionError): - return self._path.joinpath(filename).read_text(encoding='utf-8') - read_text.__doc__ = Distribution.read_text.__doc__ - - def locate_file(self, path): - return self._path.parent / path - - -def distribution(distribution_name): - """Get the ``Distribution`` instance for the named package. - - :param distribution_name: The name of the distribution package as a string. - :return: A ``Distribution`` instance (or subclass thereof). - """ - return Distribution.from_name(distribution_name) - - -def distributions(**kwargs): - """Get all ``Distribution`` instances in the current environment. - - :return: An iterable of ``Distribution`` instances. - """ - return Distribution.discover(**kwargs) - - -def metadata(distribution_name): - """Get the metadata for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: An email.Message containing the parsed metadata. - """ - return Distribution.from_name(distribution_name).metadata - - -def version(distribution_name): - """Get the version string for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: The version string for the package as defined in the package's - "Version" metadata key. - """ - return distribution(distribution_name).version - - -def entry_points(): - """Return EntryPoint objects for all installed packages. - - :return: EntryPoint objects for all installed packages. - """ - eps = itertools.chain.from_iterable( - dist.entry_points for dist in distributions()) - by_group = operator.attrgetter('group') - ordered = sorted(eps, key=by_group) - grouped = itertools.groupby(ordered, by_group) - return { - group: tuple(eps) - for group, eps in grouped - } - - -def files(distribution_name): - """Return a list of files for the named package. - - :param distribution_name: The name of the distribution package to query. - :return: List of files composing the distribution. - """ - return distribution(distribution_name).files - - -def requires(distribution_name): - """ - Return a list of requirements for the named package. - - :return: An iterator of requirements, suitable for - packaging.requirement.Requirement. - """ - return distribution(distribution_name).requires diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py new file mode 100644 index 0000000000..7181ed8757 --- /dev/null +++ b/Lib/importlib/metadata/__init__.py @@ -0,0 +1,1051 @@ +import os +import re +import abc +import csv +import sys +import email +import pathlib +import zipfile +import operator +import textwrap +import warnings +import functools +import itertools +import posixpath +import collections + +from . import _adapters, _meta +from ._meta import PackageMetadata +from ._collections import FreezableDefaultDict, Pair +from ._functools import method_cache +from ._itertools import unique_everseen +from ._meta import PackageMetadata, SimplePath + +from contextlib import suppress +from importlib import import_module +from importlib.abc import MetaPathFinder +from itertools import starmap +from typing import List, Mapping, Optional, Union + + +__all__ = [ + 'Distribution', + 'DistributionFinder', + 'PackageMetadata', + 'PackageNotFoundError', + 'distribution', + 'distributions', + 'entry_points', + 'files', + 'metadata', + 'packages_distributions', + 'requires', + 'version', +] + + +class PackageNotFoundError(ModuleNotFoundError): + """The package was not found.""" + + def __str__(self): + return f"No package metadata was found for {self.name}" + + @property + def name(self): + (name,) = self.args + return name + + # TODO: RUSTPYTHON; the entire setter is added to avoid errors + @name.setter + def name(self, value): + import sys + sys.stderr.write("set value to PackageNotFoundError ignored\n") + + +class Sectioned: + """ + A simple entry point config parser for performance + + >>> for item in Sectioned.read(Sectioned._sample): + ... print(item) + Pair(name='sec1', value='# comments ignored') + Pair(name='sec1', value='a = 1') + Pair(name='sec1', value='b = 2') + Pair(name='sec2', value='a = 2') + + >>> res = Sectioned.section_pairs(Sectioned._sample) + >>> item = next(res) + >>> item.name + 'sec1' + >>> item.value + Pair(name='a', value='1') + >>> item = next(res) + >>> item.value + Pair(name='b', value='2') + >>> item = next(res) + >>> item.name + 'sec2' + >>> item.value + Pair(name='a', value='2') + >>> list(res) + [] + """ + + _sample = textwrap.dedent( + """ + [sec1] + # comments ignored + a = 1 + b = 2 + + [sec2] + a = 2 + """ + ).lstrip() + + @classmethod + def section_pairs(cls, text): + return ( + section._replace(value=Pair.parse(section.value)) + for section in cls.read(text, filter_=cls.valid) + if section.name is not None + ) + + @staticmethod + def read(text, filter_=None): + lines = filter(filter_, map(str.strip, text.splitlines())) + name = None + for value in lines: + section_match = value.startswith('[') and value.endswith(']') + if section_match: + name = value.strip('[]') + continue + yield Pair(name, value) + + @staticmethod + def valid(line): + return line and not line.startswith('#') + + +class EntryPoint( + collections.namedtuple('EntryPointBase', 'name value group')): + """An entry point as defined by Python packaging conventions. + + See `the packaging docs on entry points + `_ + for more information. + + >>> ep = EntryPoint( + ... name=None, group=None, value='package.module:attr [extra1, extra2]') + >>> ep.module + 'package.module' + >>> ep.attr + 'attr' + >>> ep.extras + ['extra1', 'extra2'] + """ + + pattern = re.compile( + r'(?P[\w.]+)\s*' + r'(:\s*(?P[\w.]+)\s*)?' + r'((?P\[.*\])\s*)?$' + ) + """ + A regular expression describing the syntax for an entry point, + which might look like: + + - module + - package.module + - package.module:attribute + - package.module:object.attribute + - package.module:attr [extra1, extra2] + + Other combinations are possible as well. + + The expression is lenient about whitespace around the ':', + following the attr, and following any extras. + """ + + dist: Optional['Distribution'] = None + + def load(self): + """Load the entry point from its definition. If only a module + is indicated by the value, return that module. Otherwise, + return the named object. + """ + match = self.pattern.match(self.value) + module = import_module(match.group('module')) + attrs = filter(None, (match.group('attr') or '').split('.')) + return functools.reduce(getattr, attrs, module) + + @property + def module(self): + match = self.pattern.match(self.value) + return match.group('module') + + @property + def attr(self): + match = self.pattern.match(self.value) + return match.group('attr') + + @property + def extras(self): + match = self.pattern.match(self.value) + return re.findall(r'\w+', match.group('extras') or '') + + def _for(self, dist): + self.dist = dist + return self + + def __iter__(self): + """ + Supply iter so one may construct dicts of EntryPoints by name. + """ + msg = ( + "Construction of dict of EntryPoints is deprecated in " + "favor of EntryPoints." + ) + warnings.warn(msg, DeprecationWarning) + return iter((self.name, self)) + + def __reduce__(self): + return ( + self.__class__, + (self.name, self.value, self.group), + ) + + def matches(self, **params): + """ + EntryPoint matches the given parameters. + + >>> ep = EntryPoint(group='foo', name='bar', value='bing:bong [extra1, extra2]') + >>> ep.matches(group='foo') + True + >>> ep.matches(name='bar', value='bing:bong [extra1, extra2]') + True + >>> ep.matches(group='foo', name='other') + False + >>> ep.matches() + True + >>> ep.matches(extras=['extra1', 'extra2']) + True + >>> ep.matches(module='bing') + True + >>> ep.matches(attr='bong') + True + """ + attrs = (getattr(self, param) for param in params) + return all(map(operator.eq, params.values(), attrs)) + + +class DeprecatedList(list): + """ + Allow an otherwise immutable object to implement mutability + for compatibility. + + >>> recwarn = getfixture('recwarn') + >>> dl = DeprecatedList(range(3)) + >>> dl[0] = 1 + >>> dl.append(3) + >>> del dl[3] + >>> dl.reverse() + >>> dl.sort() + >>> dl.extend([4]) + >>> dl.pop(-1) + 4 + >>> dl.remove(1) + >>> dl += [5] + >>> dl + [6] + [1, 2, 5, 6] + >>> dl + (6,) + [1, 2, 5, 6] + >>> dl.insert(0, 0) + >>> dl + [0, 1, 2, 5] + >>> dl == [0, 1, 2, 5] + True + >>> dl == (0, 1, 2, 5) + True + >>> len(recwarn) + 1 + """ + + __slots__ = () + + _warn = functools.partial( + warnings.warn, + "EntryPoints list interface is deprecated. Cast to list if needed.", + DeprecationWarning, + stacklevel=2, + ) + + def __setitem__(self, *args, **kwargs): + self._warn() + return super().__setitem__(*args, **kwargs) + + def __delitem__(self, *args, **kwargs): + self._warn() + return super().__delitem__(*args, **kwargs) + + def append(self, *args, **kwargs): + self._warn() + return super().append(*args, **kwargs) + + def reverse(self, *args, **kwargs): + self._warn() + return super().reverse(*args, **kwargs) + + def extend(self, *args, **kwargs): + self._warn() + return super().extend(*args, **kwargs) + + def pop(self, *args, **kwargs): + self._warn() + return super().pop(*args, **kwargs) + + def remove(self, *args, **kwargs): + self._warn() + return super().remove(*args, **kwargs) + + def __iadd__(self, *args, **kwargs): + self._warn() + return super().__iadd__(*args, **kwargs) + + def __add__(self, other): + if not isinstance(other, tuple): + self._warn() + other = tuple(other) + return self.__class__(tuple(self) + other) + + def insert(self, *args, **kwargs): + self._warn() + return super().insert(*args, **kwargs) + + def sort(self, *args, **kwargs): + self._warn() + return super().sort(*args, **kwargs) + + def __eq__(self, other): + if not isinstance(other, tuple): + self._warn() + other = tuple(other) + + return tuple(self).__eq__(other) + + +class EntryPoints(DeprecatedList): + """ + An immutable collection of selectable EntryPoint objects. + """ + + __slots__ = () + + def __getitem__(self, name): # -> EntryPoint: + """ + Get the EntryPoint in self matching name. + """ + if isinstance(name, int): + warnings.warn( + "Accessing entry points by index is deprecated. " + "Cast to tuple if needed.", + DeprecationWarning, + stacklevel=2, + ) + return super().__getitem__(name) + try: + return next(iter(self.select(name=name))) + except StopIteration: + raise KeyError(name) + + def select(self, **params): + """ + Select entry points from self that match the + given parameters (typically group and/or name). + """ + return EntryPoints(ep for ep in self if ep.matches(**params)) + + @property + def names(self): + """ + Return the set of all names of all entry points. + """ + return set(ep.name for ep in self) + + @property + def groups(self): + """ + Return the set of all groups of all entry points. + + For coverage while SelectableGroups is present. + >>> EntryPoints().groups + set() + """ + return set(ep.group for ep in self) + + @classmethod + def _from_text_for(cls, text, dist): + return cls(ep._for(dist) for ep in cls._from_text(text)) + + @classmethod + def _from_text(cls, text): + return itertools.starmap(EntryPoint, cls._parse_groups(text or '')) + + @staticmethod + def _parse_groups(text): + return ( + (item.value.name, item.value.value, item.name) + for item in Sectioned.section_pairs(text) + ) + + +class Deprecated: + """ + Compatibility add-in for mapping to indicate that + mapping behavior is deprecated. + + >>> recwarn = getfixture('recwarn') + >>> class DeprecatedDict(Deprecated, dict): pass + >>> dd = DeprecatedDict(foo='bar') + >>> dd.get('baz', None) + >>> dd['foo'] + 'bar' + >>> list(dd) + ['foo'] + >>> list(dd.keys()) + ['foo'] + >>> 'foo' in dd + True + >>> list(dd.values()) + ['bar'] + >>> len(recwarn) + 1 + """ + + _warn = functools.partial( + warnings.warn, + "SelectableGroups dict interface is deprecated. Use select.", + DeprecationWarning, + stacklevel=2, + ) + + def __getitem__(self, name): + self._warn() + return super().__getitem__(name) + + def get(self, name, default=None): + self._warn() + return super().get(name, default) + + def __iter__(self): + self._warn() + return super().__iter__() + + def __contains__(self, *args): + self._warn() + return super().__contains__(*args) + + def keys(self): + self._warn() + return super().keys() + + def values(self): + self._warn() + return super().values() + + +class SelectableGroups(Deprecated, dict): + """ + A backward- and forward-compatible result from + entry_points that fully implements the dict interface. + """ + + @classmethod + def load(cls, eps): + by_group = operator.attrgetter('group') + ordered = sorted(eps, key=by_group) + grouped = itertools.groupby(ordered, by_group) + return cls((group, EntryPoints(eps)) for group, eps in grouped) + + @property + def _all(self): + """ + Reconstruct a list of all entrypoints from the groups. + """ + groups = super(Deprecated, self).values() + return EntryPoints(itertools.chain.from_iterable(groups)) + + @property + def groups(self): + return self._all.groups + + @property + def names(self): + """ + for coverage: + >>> SelectableGroups().names + set() + """ + return self._all.names + + def select(self, **params): + if not params: + return self + return self._all.select(**params) + + +class PackagePath(pathlib.PurePosixPath): + """A reference to a path in a package""" + + def read_text(self, encoding='utf-8'): + with self.locate().open(encoding=encoding) as stream: + return stream.read() + + def read_binary(self): + with self.locate().open('rb') as stream: + return stream.read() + + def locate(self): + """Return a path-like object for this path""" + return self.dist.locate_file(self) + + +class FileHash: + def __init__(self, spec): + self.mode, _, self.value = spec.partition('=') + + def __repr__(self): + return f'' + + +class Distribution: + """A Python distribution package.""" + + @abc.abstractmethod + def read_text(self, filename): + """Attempt to load metadata file given by the name. + + :param filename: The name of the file in the distribution info. + :return: The text if found, otherwise None. + """ + + @abc.abstractmethod + def locate_file(self, path): + """ + Given a path to a file in this distribution, return a path + to it. + """ + + @classmethod + def from_name(cls, name): + """Return the Distribution for the given package name. + + :param name: The name of the distribution package to search for. + :return: The Distribution instance (or subclass thereof) for the named + package, if found. + :raises PackageNotFoundError: When the named package's distribution + metadata cannot be found. + """ + for resolver in cls._discover_resolvers(): + dists = resolver(DistributionFinder.Context(name=name)) + dist = next(iter(dists), None) + if dist is not None: + return dist + else: + raise PackageNotFoundError(name) + + @classmethod + def discover(cls, **kwargs): + """Return an iterable of Distribution objects for all packages. + + Pass a ``context`` or pass keyword arguments for constructing + a context. + + :context: A ``DistributionFinder.Context`` object. + :return: Iterable of Distribution objects for all packages. + """ + context = kwargs.pop('context', None) + if context and kwargs: + raise ValueError("cannot accept context and kwargs") + context = context or DistributionFinder.Context(**kwargs) + return itertools.chain.from_iterable( + resolver(context) for resolver in cls._discover_resolvers() + ) + + @staticmethod + def at(path): + """Return a Distribution for the indicated metadata path + + :param path: a string or path-like object + :return: a concrete Distribution instance for the path + """ + return PathDistribution(pathlib.Path(path)) + + @staticmethod + def _discover_resolvers(): + """Search the meta_path for resolvers.""" + declared = ( + getattr(finder, 'find_distributions', None) for finder in sys.meta_path + ) + return filter(None, declared) + + @classmethod + def _local(cls, root='.'): + from pep517 import build, meta + + system = build.compat_system(root) + builder = functools.partial( + meta.build, + source_dir=root, + system=system, + ) + return PathDistribution(zipfile.Path(meta.build_as_zip(builder))) + + @property + def metadata(self) -> _meta.PackageMetadata: + """Return the parsed metadata for this Distribution. + + The returned object will have keys that name the various bits of + metadata. See PEP 566 for details. + """ + text = ( + self.read_text('METADATA') + or self.read_text('PKG-INFO') + # This last clause is here to support old egg-info files. Its + # effect is to just end up using the PathDistribution's self._path + # (which points to the egg-info file) attribute unchanged. + or self.read_text('') + ) + return _adapters.Message(email.message_from_string(text)) + + @property + def name(self): + """Return the 'Name' metadata for the distribution package.""" + return self.metadata['Name'] + + @property + def _normalized_name(self): + """Return a normalized version of the name.""" + return Prepared.normalize(self.name) + + @property + def version(self): + """Return the 'Version' metadata for the distribution package.""" + return self.metadata['Version'] + + @property + def entry_points(self): + return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) + + @property + def files(self): + """Files in this distribution. + + :return: List of PackagePath for this distribution or None + + Result is `None` if the metadata file that enumerates files + (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is + missing. + Result may be empty if the metadata exists but is empty. + """ + file_lines = self._read_files_distinfo() or self._read_files_egginfo() + + def make_file(name, hash=None, size_str=None): + result = PackagePath(name) + result.hash = FileHash(hash) if hash else None + result.size = int(size_str) if size_str else None + result.dist = self + return result + + return file_lines and list(starmap(make_file, csv.reader(file_lines))) + + def _read_files_distinfo(self): + """ + Read the lines of RECORD + """ + text = self.read_text('RECORD') + return text and text.splitlines() + + def _read_files_egginfo(self): + """ + SOURCES.txt might contain literal commas, so wrap each line + in quotes. + """ + text = self.read_text('SOURCES.txt') + return text and map('"{}"'.format, text.splitlines()) + + @property + def requires(self): + """Generated requirements specified for this Distribution""" + reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() + return reqs and list(reqs) + + def _read_dist_info_reqs(self): + return self.metadata.get_all('Requires-Dist') + + def _read_egg_info_reqs(self): + source = self.read_text('requires.txt') + return None if source is None else self._deps_from_requires_text(source) + + @classmethod + def _deps_from_requires_text(cls, source): + return cls._convert_egg_info_reqs_to_simple_reqs(Sectioned.read(source)) + + @staticmethod + def _convert_egg_info_reqs_to_simple_reqs(sections): + """ + Historically, setuptools would solicit and store 'extra' + requirements, including those with environment markers, + in separate sections. More modern tools expect each + dependency to be defined separately, with any relevant + extras and environment markers attached directly to that + requirement. This method converts the former to the + latter. See _test_deps_from_requires_text for an example. + """ + + def make_condition(name): + return name and f'extra == "{name}"' + + def quoted_marker(section): + section = section or '' + extra, sep, markers = section.partition(':') + if extra and markers: + markers = f'({markers})' + conditions = list(filter(None, [markers, make_condition(extra)])) + return '; ' + ' and '.join(conditions) if conditions else '' + + def url_req_space(req): + """ + PEP 508 requires a space between the url_spec and the quoted_marker. + Ref python/importlib_metadata#357. + """ + # '@' is uniquely indicative of a url_req. + return ' ' * ('@' in req) + + for section in sections: + space = url_req_space(section.value) + yield section.value + space + quoted_marker(section.name) + + +class DistributionFinder(MetaPathFinder): + """ + A MetaPathFinder capable of discovering installed distributions. + """ + + class Context: + """ + Keyword arguments presented by the caller to + ``distributions()`` or ``Distribution.discover()`` + to narrow the scope of a search for distributions + in all DistributionFinders. + + Each DistributionFinder may expect any parameters + and should attempt to honor the canonical + parameters defined below when appropriate. + """ + + name = None + """ + Specific name for which a distribution finder should match. + A name of ``None`` matches all distributions. + """ + + def __init__(self, **kwargs): + vars(self).update(kwargs) + + @property + def path(self): + """ + The sequence of directory path that a distribution finder + should search. + + Typically refers to Python installed package paths such as + "site-packages" directories and defaults to ``sys.path``. + """ + return vars(self).get('path', sys.path) + + @abc.abstractmethod + def find_distributions(self, context=Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the ``context``, + a DistributionFinder.Context instance. + """ + + +class FastPath: + """ + Micro-optimized class for searching a path for + children. + """ + + @functools.lru_cache() # type: ignore + def __new__(cls, root): + return super().__new__(cls) + + def __init__(self, root): + self.root = root + + def joinpath(self, child): + return pathlib.Path(self.root, child) + + def children(self): + with suppress(Exception): + return os.listdir(self.root or '.') + with suppress(Exception): + return self.zip_children() + return [] + + def zip_children(self): + zip_path = zipfile.Path(self.root) + names = zip_path.root.namelist() + self.joinpath = zip_path.joinpath + + return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) + + def search(self, name): + return self.lookup(self.mtime).search(name) + + @property + def mtime(self): + with suppress(OSError): + return os.stat(self.root).st_mtime + self.lookup.cache_clear() + + @method_cache + def lookup(self, mtime): + return Lookup(self) + + +class Lookup: + def __init__(self, path: FastPath): + base = os.path.basename(path.root).lower() + base_is_egg = base.endswith(".egg") + self.infos = FreezableDefaultDict(list) + self.eggs = FreezableDefaultDict(list) + + for child in path.children(): + low = child.lower() + if low.endswith((".dist-info", ".egg-info")): + # rpartition is faster than splitext and suitable for this purpose. + name = low.rpartition(".")[0].partition("-")[0] + normalized = Prepared.normalize(name) + self.infos[normalized].append(path.joinpath(child)) + elif base_is_egg and low == "egg-info": + name = base.rpartition(".")[0].partition("-")[0] + legacy_normalized = Prepared.legacy_normalize(name) + self.eggs[legacy_normalized].append(path.joinpath(child)) + + self.infos.freeze() + self.eggs.freeze() + + def search(self, prepared): + infos = ( + self.infos[prepared.normalized] + if prepared + else itertools.chain.from_iterable(self.infos.values()) + ) + eggs = ( + self.eggs[prepared.legacy_normalized] + if prepared + else itertools.chain.from_iterable(self.eggs.values()) + ) + return itertools.chain(infos, eggs) + + +class Prepared: + """ + A prepared search for metadata on a possibly-named package. + """ + + normalized = None + legacy_normalized = None + + def __init__(self, name): + self.name = name + if name is None: + return + self.normalized = self.normalize(name) + self.legacy_normalized = self.legacy_normalize(name) + + @staticmethod + def normalize(name): + """ + PEP 503 normalization plus dashes as underscores. + """ + return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_') + + @staticmethod + def legacy_normalize(name): + """ + Normalize the package name as found in the convention in + older packaging tools versions and specs. + """ + return name.lower().replace('-', '_') + + def __bool__(self): + return bool(self.name) + + +class MetadataPathFinder(DistributionFinder): + @classmethod + def find_distributions(cls, context=DistributionFinder.Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching ``context.name`` + (or all names if ``None`` indicated) along the paths in the list + of directories ``context.path``. + """ + found = cls._search_paths(context.name, context.path) + return map(PathDistribution, found) + + @classmethod + def _search_paths(cls, name, paths): + """Find metadata directories in paths heuristically.""" + prepared = Prepared(name) + return itertools.chain.from_iterable( + path.search(prepared) for path in map(FastPath, paths) + ) + + def invalidate_caches(cls): + FastPath.__new__.cache_clear() + + +class PathDistribution(Distribution): + def __init__(self, path: SimplePath): + """Construct a distribution. + + :param path: SimplePath indicating the metadata directory. + """ + self._path = path + + def read_text(self, filename): + with suppress( + FileNotFoundError, + IsADirectoryError, + KeyError, + NotADirectoryError, + PermissionError, + ): + return self._path.joinpath(filename).read_text(encoding='utf-8') + + read_text.__doc__ = Distribution.read_text.__doc__ + + def locate_file(self, path): + return self._path.parent / path + + @property + def _normalized_name(self): + """ + Performance optimization: where possible, resolve the + normalized name from the file system path. + """ + stem = os.path.basename(str(self._path)) + return self._name_from_stem(stem) or super()._normalized_name + + def _name_from_stem(self, stem): + name, ext = os.path.splitext(stem) + if ext not in ('.dist-info', '.egg-info'): + return + name, sep, rest = stem.partition('-') + return name + + +def distribution(distribution_name): + """Get the ``Distribution`` instance for the named package. + + :param distribution_name: The name of the distribution package as a string. + :return: A ``Distribution`` instance (or subclass thereof). + """ + return Distribution.from_name(distribution_name) + + +def distributions(**kwargs): + """Get all ``Distribution`` instances in the current environment. + + :return: An iterable of ``Distribution`` instances. + """ + return Distribution.discover(**kwargs) + + +def metadata(distribution_name) -> _meta.PackageMetadata: + """Get the metadata for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: A PackageMetadata containing the parsed metadata. + """ + return Distribution.from_name(distribution_name).metadata + + +def version(distribution_name): + """Get the version string for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: The version string for the package as defined in the package's + "Version" metadata key. + """ + return distribution(distribution_name).version + + +def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: + """Return EntryPoint objects for all installed packages. + + Pass selection parameters (group or name) to filter the + result to entry points matching those properties (see + EntryPoints.select()). + + For compatibility, returns ``SelectableGroups`` object unless + selection parameters are supplied. In the future, this function + will return ``EntryPoints`` instead of ``SelectableGroups`` + even when no selection parameters are supplied. + + For maximum future compatibility, pass selection parameters + or invoke ``.select`` with parameters on the result. + + :return: EntryPoints or SelectableGroups for all installed packages. + """ + norm_name = operator.attrgetter('_normalized_name') + unique = functools.partial(unique_everseen, key=norm_name) + eps = itertools.chain.from_iterable( + dist.entry_points for dist in unique(distributions()) + ) + return SelectableGroups.load(eps).select(**params) + + +def files(distribution_name): + """Return a list of files for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: List of files composing the distribution. + """ + return distribution(distribution_name).files + + +def requires(distribution_name): + """ + Return a list of requirements for the named package. + + :return: An iterator of requirements, suitable for + packaging.requirement.Requirement. + """ + return distribution(distribution_name).requires + + +def packages_distributions() -> Mapping[str, List[str]]: + """ + Return a mapping of top-level packages to their + distributions. + + >>> import collections.abc + >>> pkgs = packages_distributions() + >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values()) + True + """ + pkg_to_dist = collections.defaultdict(list) + for dist in distributions(): + for pkg in (dist.read_text('top_level.txt') or '').split(): + pkg_to_dist[pkg].append(dist.metadata['Name']) + return dict(pkg_to_dist) diff --git a/Lib/importlib/metadata/_adapters.py b/Lib/importlib/metadata/_adapters.py new file mode 100644 index 0000000000..aa460d3eda --- /dev/null +++ b/Lib/importlib/metadata/_adapters.py @@ -0,0 +1,68 @@ +import re +import textwrap +import email.message + +from ._text import FoldedCase + + +class Message(email.message.Message): + multiple_use_keys = set( + map( + FoldedCase, + [ + 'Classifier', + 'Obsoletes-Dist', + 'Platform', + 'Project-URL', + 'Provides-Dist', + 'Provides-Extra', + 'Requires-Dist', + 'Requires-External', + 'Supported-Platform', + 'Dynamic', + ], + ) + ) + """ + Keys that may be indicated multiple times per PEP 566. + """ + + def __new__(cls, orig: email.message.Message): + res = super().__new__(cls) + vars(res).update(vars(orig)) + return res + + def __init__(self, *args, **kwargs): + self._headers = self._repair_headers() + + # suppress spurious error from mypy + def __iter__(self): + return super().__iter__() + + def _repair_headers(self): + def redent(value): + "Correct for RFC822 indentation" + if not value or '\n' not in value: + return value + return textwrap.dedent(' ' * 8 + value) + + headers = [(key, redent(value)) for key, value in vars(self)['_headers']] + if self._payload: + headers.append(('Description', self.get_payload())) + return headers + + @property + def json(self): + """ + Convert PackageMetadata to a JSON-compatible format + per PEP 0566. + """ + + def transform(key): + value = self.get_all(key) if key in self.multiple_use_keys else self[key] + if key == 'Keywords': + value = re.split(r'\s+', value) + tk = key.lower().replace('-', '_') + return tk, value + + return dict(map(transform, map(FoldedCase, self))) diff --git a/Lib/importlib/metadata/_collections.py b/Lib/importlib/metadata/_collections.py new file mode 100644 index 0000000000..cf0954e1a3 --- /dev/null +++ b/Lib/importlib/metadata/_collections.py @@ -0,0 +1,30 @@ +import collections + + +# from jaraco.collections 3.3 +class FreezableDefaultDict(collections.defaultdict): + """ + Often it is desirable to prevent the mutation of + a default dict after its initial construction, such + as to prevent mutation during iteration. + + >>> dd = FreezableDefaultDict(list) + >>> dd[0].append('1') + >>> dd.freeze() + >>> dd[1] + [] + >>> len(dd) + 1 + """ + + def __missing__(self, key): + return getattr(self, '_frozen', super().__missing__)(key) + + def freeze(self): + self._frozen = lambda key: self.default_factory() + + +class Pair(collections.namedtuple('Pair', 'name value')): + @classmethod + def parse(cls, text): + return cls(*map(str.strip, text.split("=", 1))) diff --git a/Lib/importlib/metadata/_functools.py b/Lib/importlib/metadata/_functools.py new file mode 100644 index 0000000000..73f50d00bc --- /dev/null +++ b/Lib/importlib/metadata/_functools.py @@ -0,0 +1,85 @@ +import types +import functools + + +# from jaraco.functools 3.3 +def method_cache(method, cache_wrapper=None): + """ + Wrap lru_cache to support storing the cache data in the object instances. + + Abstracts the common paradigm where the method explicitly saves an + underscore-prefixed protected property on first call and returns that + subsequently. + + >>> class MyClass: + ... calls = 0 + ... + ... @method_cache + ... def method(self, value): + ... self.calls += 1 + ... return value + + >>> a = MyClass() + >>> a.method(3) + 3 + >>> for x in range(75): + ... res = a.method(x) + >>> a.calls + 75 + + Note that the apparent behavior will be exactly like that of lru_cache + except that the cache is stored on each instance, so values in one + instance will not flush values from another, and when an instance is + deleted, so are the cached values for that instance. + + >>> b = MyClass() + >>> for x in range(35): + ... res = b.method(x) + >>> b.calls + 35 + >>> a.method(0) + 0 + >>> a.calls + 75 + + Note that if method had been decorated with ``functools.lru_cache()``, + a.calls would have been 76 (due to the cached value of 0 having been + flushed by the 'b' instance). + + Clear the cache with ``.cache_clear()`` + + >>> a.method.cache_clear() + + Same for a method that hasn't yet been called. + + >>> c = MyClass() + >>> c.method.cache_clear() + + Another cache wrapper may be supplied: + + >>> cache = functools.lru_cache(maxsize=2) + >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) + >>> a = MyClass() + >>> a.method2() + 3 + + Caution - do not subsequently wrap the method with another decorator, such + as ``@property``, which changes the semantics of the function. + + See also + http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ + for another implementation and additional justification. + """ + cache_wrapper = cache_wrapper or functools.lru_cache() + + def wrapper(self, *args, **kwargs): + # it's the first call, replace the method with a cached, bound method + bound_method = types.MethodType(method, self) + cached_method = cache_wrapper(bound_method) + setattr(self, method.__name__, cached_method) + return cached_method(*args, **kwargs) + + # Support cache clear even before cache has been created. + wrapper.cache_clear = lambda: None + + return wrapper diff --git a/Lib/importlib/metadata/_itertools.py b/Lib/importlib/metadata/_itertools.py new file mode 100644 index 0000000000..dd45f2f096 --- /dev/null +++ b/Lib/importlib/metadata/_itertools.py @@ -0,0 +1,19 @@ +from itertools import filterfalse + + +def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBCcAD', str.lower) --> A B C D + seen = set() + seen_add = seen.add + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen_add(k) + yield element diff --git a/Lib/importlib/metadata/_meta.py b/Lib/importlib/metadata/_meta.py new file mode 100644 index 0000000000..1a6edbf957 --- /dev/null +++ b/Lib/importlib/metadata/_meta.py @@ -0,0 +1,47 @@ +from typing import Any, Dict, Iterator, List, Protocol, TypeVar, Union + + +_T = TypeVar("_T") + + +class PackageMetadata(Protocol): + def __len__(self) -> int: + ... # pragma: no cover + + def __contains__(self, item: str) -> bool: + ... # pragma: no cover + + def __getitem__(self, key: str) -> str: + ... # pragma: no cover + + def __iter__(self) -> Iterator[str]: + ... # pragma: no cover + + def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]: + """ + Return all values associated with a possibly multi-valued key. + """ + + @property + def json(self) -> Dict[str, Union[str, List[str]]]: + """ + A JSON-compatible form of the metadata. + """ + + +class SimplePath(Protocol): + """ + A minimal subset of pathlib.Path required by PathDistribution. + """ + + def joinpath(self) -> 'SimplePath': + ... # pragma: no cover + + def __div__(self) -> 'SimplePath': + ... # pragma: no cover + + def parent(self) -> 'SimplePath': + ... # pragma: no cover + + def read_text(self) -> str: + ... # pragma: no cover diff --git a/Lib/importlib/metadata/_text.py b/Lib/importlib/metadata/_text.py new file mode 100644 index 0000000000..766979d93c --- /dev/null +++ b/Lib/importlib/metadata/_text.py @@ -0,0 +1,99 @@ +import re + +from ._functools import method_cache + + +# from jaraco.text 3.5 +class FoldedCase(str): + """ + A case insensitive string class; behaves just like str + except compares equal when the only variation is case. + + >>> s = FoldedCase('hello world') + + >>> s == 'Hello World' + True + + >>> 'Hello World' == s + True + + >>> s != 'Hello World' + False + + >>> s.index('O') + 4 + + >>> s.split('O') + ['hell', ' w', 'rld'] + + >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) + ['alpha', 'Beta', 'GAMMA'] + + Sequence membership is straightforward. + + >>> "Hello World" in [s] + True + >>> s in ["Hello World"] + True + + You may test for set inclusion, but candidate and elements + must both be folded. + + >>> FoldedCase("Hello World") in {s} + True + >>> s in {FoldedCase("Hello World")} + True + + String inclusion works as long as the FoldedCase object + is on the right. + + >>> "hello" in FoldedCase("Hello World") + True + + But not if the FoldedCase object is on the left: + + >>> FoldedCase('hello') in 'Hello World' + False + + In that case, use in_: + + >>> FoldedCase('hello').in_('Hello World') + True + + >>> FoldedCase('hello') > FoldedCase('Hello') + False + """ + + def __lt__(self, other): + return self.lower() < other.lower() + + def __gt__(self, other): + return self.lower() > other.lower() + + def __eq__(self, other): + return self.lower() == other.lower() + + def __ne__(self, other): + return self.lower() != other.lower() + + def __hash__(self): + return hash(self.lower()) + + def __contains__(self, other): + return super(FoldedCase, self).lower().__contains__(other.lower()) + + def in_(self, other): + "Does self appear in other?" + return self in FoldedCase(other) + + # cache lower since it's likely to be called frequently. + @method_cache + def lower(self): + return super(FoldedCase, self).lower() + + def index(self, sub): + return self.lower().index(sub.lower()) + + def split(self, splitter=' ', maxsplit=0): + pattern = re.compile(re.escape(splitter), re.I) + return pattern.split(self, maxsplit) diff --git a/Lib/importlib/readers.py b/Lib/importlib/readers.py new file mode 100644 index 0000000000..41089c071d --- /dev/null +++ b/Lib/importlib/readers.py @@ -0,0 +1,123 @@ +import collections +import zipfile +import pathlib +from . import abc + + +def remove_duplicates(items): + return iter(collections.OrderedDict.fromkeys(items)) + + +class FileReader(abc.TraversableResources): + def __init__(self, loader): + self.path = pathlib.Path(loader.path).parent + + def resource_path(self, resource): + """ + Return the file system path to prevent + `resources.path()` from creating a temporary + copy. + """ + return str(self.path.joinpath(resource)) + + def files(self): + return self.path + + +class ZipReader(abc.TraversableResources): + def __init__(self, loader, module): + _, _, name = module.rpartition('.') + self.prefix = loader.prefix.replace('\\', '/') + name + '/' + self.archive = loader.archive + + def open_resource(self, resource): + try: + return super().open_resource(resource) + except KeyError as exc: + raise FileNotFoundError(exc.args[0]) + + def is_resource(self, path): + # workaround for `zipfile.Path.is_file` returning true + # for non-existent paths. + target = self.files().joinpath(path) + return target.is_file() and target.exists() + + def files(self): + return zipfile.Path(self.archive, self.prefix) + + +class MultiplexedPath(abc.Traversable): + """ + Given a series of Traversable objects, implement a merged + version of the interface across all objects. Useful for + namespace packages which may be multihomed at a single + name. + """ + + def __init__(self, *paths): + self._paths = list(map(pathlib.Path, remove_duplicates(paths))) + if not self._paths: + message = 'MultiplexedPath must contain at least one path' + raise FileNotFoundError(message) + if not all(path.is_dir() for path in self._paths): + raise NotADirectoryError('MultiplexedPath only supports directories') + + def iterdir(self): + visited = [] + for path in self._paths: + for file in path.iterdir(): + if file.name in visited: + continue + visited.append(file.name) + yield file + + def read_bytes(self): + raise FileNotFoundError(f'{self} is not a file') + + def read_text(self, *args, **kwargs): + raise FileNotFoundError(f'{self} is not a file') + + def is_dir(self): + return True + + def is_file(self): + return False + + def joinpath(self, child): + # first try to find child in current paths + for file in self.iterdir(): + if file.name == child: + return file + # if it does not exist, construct it with the first path + return self._paths[0] / child + + __truediv__ = joinpath + + def open(self, *args, **kwargs): + raise FileNotFoundError(f'{self} is not a file') + + @property + def name(self): + return self._paths[0].name + + def __repr__(self): + paths = ', '.join(f"'{path}'" for path in self._paths) + return f'MultiplexedPath({paths})' + + +class NamespaceReader(abc.TraversableResources): + def __init__(self, namespace_path): + if 'NamespacePath' not in str(namespace_path): + raise ValueError('Invalid path') + self.path = MultiplexedPath(*list(namespace_path)) + + def resource_path(self, resource): + """ + Return the file system path to prevent + `resources.path()` from creating a temporary + copy. + """ + return str(self.path.joinpath(resource)) + + def files(self): + return self.path diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index fc3a1c9cab..8a98663ff8 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -1,177 +1,113 @@ import os -import tempfile +import io -from . import abc as resources_abc -from contextlib import contextmanager, suppress -from importlib import import_module +from . import _common +from ._common import as_file, files +from .abc import ResourceReader +from contextlib import suppress from importlib.abc import ResourceLoader +from importlib.machinery import ModuleSpec from io import BytesIO, TextIOWrapper from pathlib import Path from types import ModuleType -from typing import Iterable, Iterator, Optional, Set, Union # noqa: F401 +from typing import ContextManager, Iterable, Union from typing import cast from typing.io import BinaryIO, TextIO -from zipimport import ZipImportError +from collections.abc import Sequence +from functools import singledispatch __all__ = [ 'Package', 'Resource', + 'ResourceReader', + 'as_file', 'contents', + 'files', 'is_resource', 'open_binary', 'open_text', 'path', 'read_binary', 'read_text', - ] +] Package = Union[str, ModuleType] Resource = Union[str, os.PathLike] -def _get_package(package) -> ModuleType: - """Take a package name or module object and return the module. - - If a name, the module is imported. If the passed or imported module - object is not a package, raise an exception. - """ - if hasattr(package, '__spec__'): - if package.__spec__.submodule_search_locations is None: - raise TypeError('{!r} is not a package'.format( - package.__spec__.name)) - else: - return package - else: - module = import_module(package) - if module.__spec__.submodule_search_locations is None: - raise TypeError('{!r} is not a package'.format(package)) - else: - return module - - -def _normalize_path(path) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - parent, file_name = os.path.split(path) - if parent: - raise ValueError('{!r} must be only a file name'.format(path)) - else: - return file_name - - -def _get_resource_reader( - package: ModuleType) -> Optional[resources_abc.ResourceReader]: - # Return the package's loader if it's a ResourceReader. We can't use - # a issubclass() check here because apparently abc.'s __subclasscheck__() - # hook wants to create a weak reference to the object, but - # zipimport.zipimporter does not support weak references, resulting in a - # TypeError. That seems terrible. - spec = package.__spec__ - if hasattr(spec.loader, 'get_resource_reader'): - return cast(resources_abc.ResourceReader, - spec.loader.get_resource_reader(spec.name)) - return None - - -def _check_location(package): - if package.__spec__.origin is None or not package.__spec__.has_location: - raise FileNotFoundError(f'Package has no location {package!r}') - - def open_binary(package: Package, resource: Resource) -> BinaryIO: """Return a file-like object opened for binary reading of the resource.""" - resource = _normalize_path(resource) - package = _get_package(package) - reader = _get_resource_reader(package) + resource = _common.normalize_path(resource) + package = _common.get_package(package) + reader = _common.get_resource_reader(package) if reader is not None: return reader.open_resource(resource) - _check_location(package) - absolute_package_path = os.path.abspath(package.__spec__.origin) - package_path = os.path.dirname(absolute_package_path) - full_path = os.path.join(package_path, resource) - try: - return open(full_path, mode='rb') - except OSError: - # Just assume the loader is a resource loader; all the relevant - # importlib.machinery loaders are and an AttributeError for - # get_data() will make it clear what is needed from the loader. - loader = cast(ResourceLoader, package.__spec__.loader) - data = None - if hasattr(package.__spec__.loader, 'get_data'): - with suppress(OSError): - data = loader.get_data(full_path) - if data is None: - package_name = package.__spec__.name - message = '{!r} resource not found in {!r}'.format( - resource, package_name) - raise FileNotFoundError(message) - else: - return BytesIO(data) - - -def open_text(package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict') -> TextIO: + spec = cast(ModuleSpec, package.__spec__) + # Using pathlib doesn't work well here due to the lack of 'strict' + # argument for pathlib.Path.resolve() prior to Python 3.6. + if spec.submodule_search_locations is not None: + paths = spec.submodule_search_locations + elif spec.origin is not None: + paths = [os.path.dirname(os.path.abspath(spec.origin))] + + for package_path in paths: + full_path = os.path.join(package_path, resource) + try: + return open(full_path, mode='rb') + except OSError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, spec.loader) + data = None + if hasattr(spec.loader, 'get_data'): + with suppress(OSError): + data = loader.get_data(full_path) + if data is not None: + return BytesIO(data) + + raise FileNotFoundError(f'{resource!r} resource not found in {spec.name!r}') + + +def open_text( + package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict', +) -> TextIO: """Return a file-like object opened for text reading of the resource.""" - resource = _normalize_path(resource) - package = _get_package(package) - reader = _get_resource_reader(package) - if reader is not None: - return TextIOWrapper(reader.open_resource(resource), encoding, errors) - _check_location(package) - absolute_package_path = os.path.abspath(package.__spec__.origin) - package_path = os.path.dirname(absolute_package_path) - full_path = os.path.join(package_path, resource) - try: - return open(full_path, mode='r', encoding=encoding, errors=errors) - except OSError: - # Just assume the loader is a resource loader; all the relevant - # importlib.machinery loaders are and an AttributeError for - # get_data() will make it clear what is needed from the loader. - loader = cast(ResourceLoader, package.__spec__.loader) - data = None - if hasattr(package.__spec__.loader, 'get_data'): - with suppress(OSError): - data = loader.get_data(full_path) - if data is None: - package_name = package.__spec__.name - message = '{!r} resource not found in {!r}'.format( - resource, package_name) - raise FileNotFoundError(message) - else: - return TextIOWrapper(BytesIO(data), encoding, errors) + return TextIOWrapper( + open_binary(package, resource), encoding=encoding, errors=errors + ) def read_binary(package: Package, resource: Resource) -> bytes: """Return the binary contents of the resource.""" - resource = _normalize_path(resource) - package = _get_package(package) with open_binary(package, resource) as fp: return fp.read() -def read_text(package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict') -> str: +def read_text( + package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict', +) -> str: """Return the decoded string of the resource. The decoding-related arguments have the same semantics as those of bytes.decode(). """ - resource = _normalize_path(resource) - package = _get_package(package) with open_text(package, resource, encoding, errors) as fp: return fp.read() -@contextmanager -def path(package: Package, resource: Resource) -> Iterator[Path]: +def path( + package: Package, + resource: Resource, +) -> 'ContextManager[Path]': """A context manager providing a file path object to the resource. If the resource does not already exist on its own on the file system, @@ -180,39 +116,30 @@ def path(package: Package, resource: Resource) -> Iterator[Path]: raised if the file was deleted prior to the context manager exiting). """ - resource = _normalize_path(resource) - package = _get_package(package) - reader = _get_resource_reader(package) - if reader is not None: - try: - yield Path(reader.resource_path(resource)) - return - except FileNotFoundError: - pass - else: - _check_location(package) - # Fall-through for both the lack of resource_path() *and* if - # resource_path() raises FileNotFoundError. - package_directory = Path(package.__spec__.origin).parent - file_path = package_directory / resource - if file_path.exists(): - yield file_path - else: - with open_binary(package, resource) as fp: - data = fp.read() - # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' - # blocks due to the need to close the temporary file to work on - # Windows properly. - fd, raw_path = tempfile.mkstemp() - try: - os.write(fd, data) - os.close(fd) - yield Path(raw_path) - finally: - try: - os.remove(raw_path) - except FileNotFoundError: - pass + reader = _common.get_resource_reader(_common.get_package(package)) + return ( + _path_from_reader(reader, _common.normalize_path(resource)) + if reader + else _common.as_file( + _common.files(package).joinpath(_common.normalize_path(resource)) + ) + ) + + +def _path_from_reader(reader, resource): + return _path_from_resource_path(reader, resource) or _path_from_open_resource( + reader, resource + ) + + +def _path_from_resource_path(reader, resource): + with suppress(FileNotFoundError): + return Path(reader.resource_path(resource)) + + +def _path_from_open_resource(reader, resource): + saved = io.BytesIO(reader.open_resource(resource).read()) + return _common._tempfile(saved.read, suffix=resource) def is_resource(package: Package, name: str) -> bool: @@ -220,22 +147,15 @@ def is_resource(package: Package, name: str) -> bool: Directories are *not* resources. """ - package = _get_package(package) - _normalize_path(name) - reader = _get_resource_reader(package) + package = _common.get_package(package) + _common.normalize_path(name) + reader = _common.get_resource_reader(package) if reader is not None: return reader.is_resource(name) - try: - package_contents = set(contents(package)) - except (NotADirectoryError, FileNotFoundError): - return False + package_contents = set(contents(package)) if name not in package_contents: return False - # Just because the given file_name lives as an entry in the package's - # contents doesn't necessarily mean it's a resource. Directories are not - # resources, so let's try to find out if it's a directory or not. - path = Path(package.__spec__.origin).parent / name - return path.is_file() + return (_common.from_package(package) / name).is_file() def contents(package: Package) -> Iterable[str]: @@ -245,15 +165,21 @@ def contents(package: Package) -> Iterable[str]: not considered resources. Use `is_resource()` on each entry returned here to check if it is a resource or not. """ - package = _get_package(package) - reader = _get_resource_reader(package) + package = _common.get_package(package) + reader = _common.get_resource_reader(package) if reader is not None: - return reader.contents() - # Is the package a namespace package? By definition, namespace packages - # cannot have resources. We could use _check_location() and catch the - # exception, but that's extra work, so just inline the check. - elif package.__spec__.origin is None or not package.__spec__.has_location: - return () - else: - package_directory = Path(package.__spec__.origin).parent - return os.listdir(package_directory) + return _ensure_sequence(reader.contents()) + transversable = _common.from_package(package) + if transversable.is_dir(): + return list(item.name for item in transversable.iterdir()) + return [] + + +@singledispatch +def _ensure_sequence(iterable): + return list(iterable) + + +@_ensure_sequence.register(Sequence) +def _(iterable): + return iterable diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 201e0f4cb8..8623c89840 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -1,5 +1,5 @@ """Utility code for constructing importers, etc.""" -from . import abc +from ._abc import Loader from ._bootstrap import module_from_spec from ._bootstrap import _resolve_name from ._bootstrap import spec_from_loader @@ -29,8 +29,8 @@ def resolve_name(name, package): if not name.startswith('.'): return name elif not package: - raise ValueError(f'no package specified for {repr(name)} ' - '(required for relative module names)') + raise ImportError(f'no package specified for {repr(name)} ' + '(required for relative module names)') level = 0 for character in name: if character != '.': @@ -149,7 +149,8 @@ def set_package(fxn): """ @functools.wraps(fxn) def set_package_wrapper(*args, **kwargs): - warnings.warn('The import system now takes care of this automatically.', + warnings.warn('The import system now takes care of this automatically; ' + 'this decorator is slated for removal in Python 3.12', DeprecationWarning, stacklevel=2) module = fxn(*args, **kwargs) if getattr(module, '__package__', None) is None: @@ -168,7 +169,8 @@ def set_loader(fxn): """ @functools.wraps(fxn) def set_loader_wrapper(self, *args, **kwargs): - warnings.warn('The import system now takes care of this automatically.', + warnings.warn('The import system now takes care of this automatically; ' + 'this decorator is slated for removal in Python 3.12', DeprecationWarning, stacklevel=2) module = fxn(self, *args, **kwargs) if getattr(module, '__loader__', None) is None: @@ -195,7 +197,8 @@ def module_for_loader(fxn): the second argument. """ - warnings.warn('The import system now takes care of this automatically.', + warnings.warn('The import system now takes care of this automatically; ' + 'this decorator is slated for removal in Python 3.12', DeprecationWarning, stacklevel=2) @functools.wraps(fxn) def module_for_loader_wrapper(self, fullname, *args, **kwargs): @@ -232,7 +235,6 @@ def __getattribute__(self, attr): # Figure out exactly what attributes were mutated between the creation # of the module and now. attrs_then = self.__spec__.loader_state['__dict__'] - original_type = self.__spec__.loader_state['__class__'] attrs_now = self.__dict__ attrs_updated = {} for key, value in attrs_now.items(): @@ -263,7 +265,7 @@ def __delattr__(self, attr): delattr(self, attr) -class LazyLoader(abc.Loader): +class LazyLoader(Loader): """A loader that creates a module which defers loading until attribute access.""" diff --git a/Lib/test/test_importlib/builtin/test_finder.py b/Lib/test/test_importlib/builtin/test_finder.py index 084f3de6b6..6f51abab9b 100644 --- a/Lib/test/test_importlib/builtin/test_finder.py +++ b/Lib/test/test_importlib/builtin/test_finder.py @@ -5,6 +5,7 @@ import sys import unittest +import warnings @unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') @@ -58,7 +59,9 @@ class FinderTests(abc.FinderTests): def test_module(self): # Common case. with util.uncache(util.BUILTINS.good_name): - found = self.machinery.BuiltinImporter.find_module(util.BUILTINS.good_name) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + found = self.machinery.BuiltinImporter.find_module(util.BUILTINS.good_name) self.assertTrue(found) self.assertTrue(hasattr(found, 'load_module')) @@ -70,14 +73,19 @@ def test_module(self): def test_failure(self): assert 'importlib' not in sys.builtin_module_names - loader = self.machinery.BuiltinImporter.find_module('importlib') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + loader = self.machinery.BuiltinImporter.find_module('importlib') self.assertIsNone(loader) def test_ignore_path(self): # The value for 'path' should always trigger a failed import. with util.uncache(util.BUILTINS.good_name): - loader = self.machinery.BuiltinImporter.find_module(util.BUILTINS.good_name, - ['pkg']) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + loader = self.machinery.BuiltinImporter.find_module( + util.BUILTINS.good_name, + ['pkg']) self.assertIsNone(loader) diff --git a/Lib/test/test_importlib/builtin/test_loader.py b/Lib/test/test_importlib/builtin/test_loader.py index b1349ec5da..f6b6d97cd5 100644 --- a/Lib/test/test_importlib/builtin/test_loader.py +++ b/Lib/test/test_importlib/builtin/test_loader.py @@ -6,6 +6,7 @@ import sys import types import unittest +import warnings @unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') class LoaderTests(abc.LoaderTests): @@ -24,7 +25,9 @@ def verify(self, module): self.assertIn(module.__name__, sys.modules) def load_module(self, name): - return self.machinery.BuiltinImporter.load_module(name) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return self.machinery.BuiltinImporter.load_module(name) def test_module(self): # Common case. diff --git a/Lib/test/test_importlib/data/__init__.py b/Lib/test/test_importlib/data/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl new file mode 100644 index 0000000000..641ab07f7a Binary files /dev/null and b/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl differ diff --git a/Lib/test/test_importlib/data/example-21.12-py3.6.egg b/Lib/test/test_importlib/data/example-21.12-py3.6.egg new file mode 100644 index 0000000000..cdb298a19b Binary files /dev/null and b/Lib/test/test_importlib/data/example-21.12-py3.6.egg differ diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py index 7f4b2b3d16..20bf035cb5 100644 --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -1,8 +1,7 @@ from importlib import _bootstrap_external -from test import support from test.support import os_helper import unittest - +import sys from .. import util importlib = util.import_importlib('importlib') @@ -13,28 +12,30 @@ @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest(util.CASEOKTestBase): - def find_module(self): + def find_spec(self): good_name = util.EXTENSIONS.name bad_name = good_name.upper() assert good_name != bad_name finder = self.machinery.FileFinder(util.EXTENSIONS.path, (self.machinery.ExtensionFileLoader, self.machinery.EXTENSION_SUFFIXES)) - return finder.find_module(bad_name) + return finder.find_spec(bad_name) + @unittest.skipIf(sys.flags.ignore_environment, 'ignore_environment flag was set') def test_case_sensitive(self): with os_helper.EnvironmentVarGuard() as env: env.unset('PYTHONCASEOK') self.caseok_env_changed(should_exist=False) - loader = self.find_module() - self.assertIsNone(loader) + spec = self.find_spec() + self.assertIsNone(spec) + @unittest.skipIf(sys.flags.ignore_environment, 'ignore_environment flag was set') def test_case_insensitivity(self): with os_helper.EnvironmentVarGuard() as env: env.set('PYTHONCASEOK', '1') self.caseok_env_changed(should_exist=True) - loader = self.find_module() - self.assertTrue(hasattr(loader, 'load_module')) + spec = self.find_spec() + self.assertTrue(spec) (Frozen_ExtensionCaseSensitivity, diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 89855d686e..87b8bab358 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -11,18 +11,17 @@ class FinderTests(abc.FinderTests): """Test the finder for extension modules.""" - def find_module(self, fullname): + def find_spec(self, fullname): importer = self.machinery.FileFinder(util.EXTENSIONS.path, (self.machinery.ExtensionFileLoader, self.machinery.EXTENSION_SUFFIXES)) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - return importer.find_module(fullname) + + return importer.find_spec(fullname) # TODO: RUSTPYTHON @unittest.expectedFailure def test_module(self): - self.assertTrue(self.find_module(util.EXTENSIONS.name)) + self.assertTrue(self.find_spec(util.EXTENSIONS.name)) # No extension module as an __init__ available for testing. test_package = test_package_in_package = None @@ -34,7 +33,7 @@ def test_module(self): test_package_over_module = None def test_failure(self): - self.assertIsNone(self.find_module('asdfjkl;')) + self.assertIsNone(self.find_spec('asdfjkl;')) (Frozen_FinderTests, diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index e2c2c71bf6..bdf94eca15 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -1,3 +1,4 @@ +from warnings import catch_warnings from .. import abc from .. import util @@ -7,6 +8,7 @@ import sys import types import unittest +import warnings import importlib.util import importlib from test.support.script_helper import assert_python_failure @@ -20,16 +22,20 @@ def setUp(self): util.EXTENSIONS.file_path) def load_module(self, fullname): - return self.loader.load_module(fullname) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return self.loader.load_module(fullname) # TODO: RUSTPYTHON @unittest.expectedFailure def test_load_module_API(self): # Test the default argument for load_module(). - self.loader.load_module() - self.loader.load_module(None) - with self.assertRaises(ImportError): - self.load_module('XXX') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.loader.load_module() + self.loader.load_module(None) + with self.assertRaises(ImportError): + self.load_module('XXX') def test_equality(self): other = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, @@ -92,8 +98,7 @@ def test_is_package(self): @unittest.skip("TODO: RUSTPYTHON, AssertionError") class MultiPhaseExtensionModuleTests(abc.LoaderTests): - """Test loading extension modules with multi-phase initialization (PEP 489) - """ + # Test loading extension modules with multi-phase initialization (PEP 489). def setUp(self): self.name = '_testmultiphase' @@ -103,6 +108,21 @@ def setUp(self): self.loader = self.machinery.ExtensionFileLoader( self.name, self.spec.origin) + def load_module(self): + # Load the module from the test extension. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return self.loader.load_module(self.name) + + def load_module_by_name(self, fullname): + # Load a module from the test extension by name. + origin = self.spec.origin + loader = self.machinery.ExtensionFileLoader(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + # No extension module as __init__ available for testing. test_package = None @@ -113,7 +133,7 @@ def setUp(self): test_state_after_failure = None def test_module(self): - '''Test loading an extension module''' + # Test loading an extension module. with util.uncache(self.name): module = self.load_module() for attr, value in [('__name__', self.name), @@ -127,7 +147,7 @@ def test_module(self): self.machinery.ExtensionFileLoader) def test_functionality(self): - '''Test basic functionality of stuff defined in an extension module''' + # Test basic functionality of stuff defined in an extension module. with util.uncache(self.name): module = self.load_module() self.assertIsInstance(module, types.ModuleType) @@ -147,7 +167,7 @@ def test_functionality(self): self.assertEqual(module.str_const, 'something different') def test_reload(self): - '''Test that reload didn't re-set the module's attributes''' + # Test that reload didn't re-set the module's attributes. with util.uncache(self.name): module = self.load_module() ex_class = module.Example @@ -155,7 +175,7 @@ def test_reload(self): self.assertIs(ex_class, module.Example) def test_try_registration(self): - '''Assert that the PyState_{Find,Add,Remove}Module C API doesn't work''' + # Assert that the PyState_{Find,Add,Remove}Module C API doesn't work. module = self.load_module() with self.subTest('PyState_FindModule'): self.assertEqual(module.call_state_registration_func(0), None) @@ -166,28 +186,15 @@ def test_try_registration(self): with self.assertRaises(SystemError): module.call_state_registration_func(2) - def load_module(self): - '''Load the module from the test extension''' - return self.loader.load_module(self.name) - - def load_module_by_name(self, fullname): - '''Load a module from the test extension by name''' - origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) - spec = importlib.util.spec_from_loader(fullname, loader) - module = importlib.util.module_from_spec(spec) - loader.exec_module(module) - return module - def test_load_submodule(self): - '''Test loading a simulated submodule''' + # Test loading a simulated submodule. module = self.load_module_by_name('pkg.' + self.name) self.assertIsInstance(module, types.ModuleType) self.assertEqual(module.__name__, 'pkg.' + self.name) self.assertEqual(module.str_const, 'something different') def test_load_short_name(self): - '''Test loading module with a one-character name''' + # Test loading module with a one-character name. module = self.load_module_by_name('x') self.assertIsInstance(module, types.ModuleType) self.assertEqual(module.__name__, 'x') @@ -195,27 +202,27 @@ def test_load_short_name(self): self.assertNotIn('x', sys.modules) def test_load_twice(self): - '''Test that 2 loads result in 2 module objects''' + # Test that 2 loads result in 2 module objects. module1 = self.load_module_by_name(self.name) module2 = self.load_module_by_name(self.name) self.assertIsNot(module1, module2) def test_unloadable(self): - '''Test nonexistent module''' + # Test nonexistent module. name = 'asdfjkl;' with self.assertRaises(ImportError) as cm: self.load_module_by_name(name) self.assertEqual(cm.exception.name, name) def test_unloadable_nonascii(self): - '''Test behavior with nonexistent module with non-ASCII name''' + # Test behavior with nonexistent module with non-ASCII name. name = 'fo\xf3' with self.assertRaises(ImportError) as cm: self.load_module_by_name(name) self.assertEqual(cm.exception.name, name) def test_nonmodule(self): - '''Test returning a non-module object from create works''' + # Test returning a non-module object from create works. name = self.name + '_nonmodule' mod = self.load_module_by_name(name) self.assertNotEqual(type(mod), type(unittest)) @@ -223,7 +230,7 @@ def test_nonmodule(self): # issue 27782 def test_nonmodule_with_methods(self): - '''Test creating a non-module object with methods defined''' + # Test creating a non-module object with methods defined. name = self.name + '_nonmodule_with_methods' mod = self.load_module_by_name(name) self.assertNotEqual(type(mod), type(unittest)) @@ -231,14 +238,14 @@ def test_nonmodule_with_methods(self): self.assertEqual(mod.bar(10, 1), 9) def test_null_slots(self): - '''Test that NULL slots aren't a problem''' + # Test that NULL slots aren't a problem. name = self.name + '_null_slots' module = self.load_module_by_name(name) self.assertIsInstance(module, types.ModuleType) self.assertEqual(module.__name__, name) def test_bad_modules(self): - '''Test SystemError is raised for misbehaving extensions''' + # Test SystemError is raised for misbehaving extensions. for name_base in [ 'bad_slot_large', 'bad_slot_negative', @@ -262,9 +269,9 @@ def test_bad_modules(self): self.load_module_by_name(name) def test_nonascii(self): - '''Test that modules with non-ASCII names can be loaded''' + # Test that modules with non-ASCII names can be loaded. # punycode behaves slightly differently in some-ASCII and no-ASCII - # cases, so test both + # cases, so test both. cases = [ (self.name + '_zkou\u0161ka_na\u010dten\xed', 'Czech'), ('\uff3f\u30a4\u30f3\u30dd\u30fc\u30c8\u30c6\u30b9\u30c8', @@ -276,29 +283,6 @@ def test_nonascii(self): self.assertEqual(module.__name__, name) self.assertEqual(module.__doc__, "Module named in %s" % lang) - @unittest.skipIf(not hasattr(sys, 'gettotalrefcount'), - '--with-pydebug has to be enabled for this test') - def test_bad_traverse(self): - ''' Issue #32374: Test that traverse fails when accessing per-module - state before Py_mod_exec was executed. - (Multiphase initialization modules only) - ''' - script = """if True: - try: - from test import support - import importlib.util as util - spec = util.find_spec('_testmultiphase') - spec.name = '_testmultiphase_with_bad_traverse' - - with support.SuppressCrashReport(): - m = spec.loader.create_module(spec) - except: - # Prevent Python-level exceptions from - # ending the process with non-zero status - # (We are testing for a crash in C-code) - pass""" - assert_python_failure("-c", script) - (Frozen_MultiPhaseExtensionModuleTests, Source_MultiPhaseExtensionModuleTests diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py new file mode 100644 index 0000000000..12ed07d337 --- /dev/null +++ b/Lib/test/test_importlib/fixtures.py @@ -0,0 +1,287 @@ +import os +import sys +import copy +import shutil +import pathlib +import tempfile +import textwrap +import contextlib + +from test.support.os_helper import FS_NONASCII +from typing import Dict, Union + + +@contextlib.contextmanager +def tempdir(): + tmpdir = tempfile.mkdtemp() + try: + yield pathlib.Path(tmpdir) + finally: + shutil.rmtree(tmpdir) + + +@contextlib.contextmanager +def save_cwd(): + orig = os.getcwd() + try: + yield + finally: + os.chdir(orig) + + +@contextlib.contextmanager +def tempdir_as_cwd(): + with tempdir() as tmp: + with save_cwd(): + os.chdir(str(tmp)) + yield tmp + + +@contextlib.contextmanager +def install_finder(finder): + sys.meta_path.append(finder) + try: + yield + finally: + sys.meta_path.remove(finder) + + +class Fixtures: + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + +class SiteDir(Fixtures): + def setUp(self): + super(SiteDir, self).setUp() + self.site_dir = self.fixtures.enter_context(tempdir()) + + +class OnSysPath(Fixtures): + @staticmethod + @contextlib.contextmanager + def add_sys_path(dir): + sys.path[:0] = [str(dir)] + try: + yield + finally: + sys.path.remove(str(dir)) + + def setUp(self): + super(OnSysPath, self).setUp() + self.fixtures.enter_context(self.add_sys_path(self.site_dir)) + + +# Except for python/mypy#731, prefer to define +# FilesDef = Dict[str, Union['FilesDef', str]] +FilesDef = Dict[str, Union[Dict[str, Union[Dict[str, str], str]], str]] + + +class DistInfoPkg(OnSysPath, SiteDir): + files: FilesDef = { + "distinfo_pkg-1.0.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Author: Steven Ma + Version: 1.0.0 + Requires-Dist: wheel >= 1.0 + Requires-Dist: pytest; extra == 'test' + Keywords: sample package + + Once upon a time + There was a distinfo pkg + """, + "RECORD": "mod.py,sha256=abc,20\n", + "entry_points.txt": """ + [entries] + main = mod:main + ns:sub = mod:main + """, + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + def setUp(self): + super(DistInfoPkg, self).setUp() + build_files(DistInfoPkg.files, self.site_dir) + + def make_uppercase(self): + """ + Rewrite metadata with everything uppercase. + """ + shutil.rmtree(self.site_dir / "distinfo_pkg-1.0.0.dist-info") + files = copy.deepcopy(DistInfoPkg.files) + info = files["distinfo_pkg-1.0.0.dist-info"] + info["METADATA"] = info["METADATA"].upper() + build_files(files, self.site_dir) + + +class DistInfoPkgWithDot(OnSysPath, SiteDir): + files: FilesDef = { + "pkg_dot-1.0.0.dist-info": { + "METADATA": """ + Name: pkg.dot + Version: 1.0.0 + """, + }, + } + + def setUp(self): + super(DistInfoPkgWithDot, self).setUp() + build_files(DistInfoPkgWithDot.files, self.site_dir) + + +class DistInfoPkgWithDotLegacy(OnSysPath, SiteDir): + files: FilesDef = { + "pkg.dot-1.0.0.dist-info": { + "METADATA": """ + Name: pkg.dot + Version: 1.0.0 + """, + }, + "pkg.lot.egg-info": { + "METADATA": """ + Name: pkg.lot + Version: 1.0.0 + """, + }, + } + + def setUp(self): + super(DistInfoPkgWithDotLegacy, self).setUp() + build_files(DistInfoPkgWithDotLegacy.files, self.site_dir) + + +class DistInfoPkgOffPath(SiteDir): + def setUp(self): + super(DistInfoPkgOffPath, self).setUp() + build_files(DistInfoPkg.files, self.site_dir) + + +class EggInfoPkg(OnSysPath, SiteDir): + files: FilesDef = { + "egginfo_pkg.egg-info": { + "PKG-INFO": """ + Name: egginfo-pkg + Author: Steven Ma + License: Unknown + Version: 1.0.0 + Classifier: Intended Audience :: Developers + Classifier: Topic :: Software Development :: Libraries + Keywords: sample package + Description: Once upon a time + There was an egginfo package + """, + "SOURCES.txt": """ + mod.py + egginfo_pkg.egg-info/top_level.txt + """, + "entry_points.txt": """ + [entries] + main = mod:main + """, + "requires.txt": """ + wheel >= 1.0; python_version >= "2.7" + [test] + pytest + """, + "top_level.txt": "mod\n", + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + def setUp(self): + super(EggInfoPkg, self).setUp() + build_files(EggInfoPkg.files, prefix=self.site_dir) + + +class EggInfoFile(OnSysPath, SiteDir): + files: FilesDef = { + "egginfo_file.egg-info": """ + Metadata-Version: 1.0 + Name: egginfo_file + Version: 0.1 + Summary: An example package + Home-page: www.example.com + Author: Eric Haffa-Vee + Author-email: eric@example.coms + License: UNKNOWN + Description: UNKNOWN + Platform: UNKNOWN + """, + } + + def setUp(self): + super(EggInfoFile, self).setUp() + build_files(EggInfoFile.files, prefix=self.site_dir) + + +class LocalPackage: + files: FilesDef = { + "setup.py": """ + import setuptools + setuptools.setup(name="local-pkg", version="2.0.1") + """, + } + + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + self.fixtures.enter_context(tempdir_as_cwd()) + build_files(self.files) + + +def build_files(file_defs, prefix=pathlib.Path()): + """Build a set of files/directories, as described by the + + file_defs dictionary. Each key/value pair in the dictionary is + interpreted as a filename/contents pair. If the contents value is a + dictionary, a directory is created, and the dictionary interpreted + as the files within it, recursively. + + For example: + + {"README.txt": "A README file", + "foo": { + "__init__.py": "", + "bar": { + "__init__.py": "", + }, + "baz.py": "# Some code", + } + } + """ + for name, contents in file_defs.items(): + full_name = prefix / name + if isinstance(contents, dict): + full_name.mkdir() + build_files(contents, prefix=full_name) + else: + if isinstance(contents, bytes): + with full_name.open('wb') as f: + f.write(contents) + else: + with full_name.open('w', encoding='utf-8') as f: + f.write(DALS(contents)) + + +class FileBuilder: + def unicode_filename(self): + return FS_NONASCII or self.skip("File system does not support non-ascii.") + + +def DALS(str): + "Dedent and left-strip" + return textwrap.dedent(str).lstrip() + + +class NullFinder: + def find_module(self, name): + pass diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py index 4c224cc66b..0b8b6bc977 100644 --- a/Lib/test/test_importlib/frozen/test_finder.py +++ b/Lib/test/test_importlib/frozen/test_finder.py @@ -4,6 +4,7 @@ machinery = util.import_importlib('importlib.machinery') import unittest +import warnings class FindSpecTests(abc.FinderTests): @@ -53,7 +54,9 @@ class FinderTests(abc.FinderTests): def find(self, name, path=None): finder = self.machinery.FrozenImporter - return finder.find_module(name, path) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return finder.find_module(name, path) def test_module(self): name = '__hello__' diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index 5c92f8cba6..d09532c757 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -82,7 +82,7 @@ def test_module_repr_indirect(self): test_state_after_failure = None def test_unloadable(self): - assert self.machinery.FrozenImporter.find_module('_not_real') is None + assert self.machinery.FrozenImporter.find_spec('_not_real') is None with self.assertRaises(ImportError) as cm: self.exec_module('_not_real') self.assertEqual(cm.exception.name, '_not_real') @@ -168,20 +168,16 @@ def test_module_repr(self): self.assertEqual(repr_str, "") - def test_module_repr_indirect(self): - with util.uncache('__hello__'), captured_stdout(): - module = self.machinery.FrozenImporter.load_module('__hello__') - self.assertEqual(repr(module), - "") - # No way to trigger an error in a frozen module. test_state_after_failure = None def test_unloadable(self): - assert self.machinery.FrozenImporter.find_module('_not_real') is None - with self.assertRaises(ImportError) as cm: - self.machinery.FrozenImporter.load_module('_not_real') - self.assertEqual(cm.exception.name, '_not_real') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + assert self.machinery.FrozenImporter.find_module('_not_real') is None + with self.assertRaises(ImportError) as cm: + self.machinery.FrozenImporter.load_module('_not_real') + self.assertEqual(cm.exception.name, '_not_real') (Frozen_LoaderTests, diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py index 4b18093cf9..ecd83c6567 100644 --- a/Lib/test/test_importlib/import_/test___loader__.py +++ b/Lib/test/test_importlib/import_/test___loader__.py @@ -2,6 +2,7 @@ import sys import types import unittest +import warnings from .. import util @@ -45,25 +46,29 @@ def load_module(self, fullname): class LoaderAttributeTests: def test___loader___missing(self): - module = types.ModuleType('blah') - try: - del module.__loader__ - except AttributeError: - pass - loader = LoaderMock() - loader.module = module - with util.uncache('blah'), util.import_state(meta_path=[loader]): - module = self.__import__('blah') - self.assertEqual(loader, module.__loader__) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + module = types.ModuleType('blah') + try: + del module.__loader__ + except AttributeError: + pass + loader = LoaderMock() + loader.module = module + with util.uncache('blah'), util.import_state(meta_path=[loader]): + module = self.__import__('blah') + self.assertEqual(loader, module.__loader__) def test___loader___is_None(self): - module = types.ModuleType('blah') - module.__loader__ = None - loader = LoaderMock() - loader.module = module - with util.uncache('blah'), util.import_state(meta_path=[loader]): - returned_module = self.__import__('blah') - self.assertEqual(loader, module.__loader__) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + module = types.ModuleType('blah') + module.__loader__ = None + loader = LoaderMock() + loader.module = module + with util.uncache('blah'), util.import_state(meta_path=[loader]): + returned_module = self.__import__('blah') + self.assertEqual(loader, module.__loader__) (Frozen_Tests, diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index 4b17664525..7a3ac887d5 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -102,6 +102,16 @@ def __init__(self, parent): class Using__package__PEP302(Using__package__): mock_modules = util.mock_modules + def test_using___package__(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_using___package__() + + def test_spec_fallback(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_spec_fallback() + (Frozen_UsingPackagePEP302, Source_UsingPackagePEP302 @@ -159,6 +169,21 @@ def test_submodule(self): class Setting__package__PEP302(Setting__package__, unittest.TestCase): mock_modules = util.mock_modules + def test_top_level(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_top_level() + + def test_package(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_package() + + def test_submodule(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_submodule() + class Setting__package__PEP451(Setting__package__, unittest.TestCase): mock_modules = util.mock_spec diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py index 0cd9de4daf..35c26977ea 100644 --- a/Lib/test/test_importlib/import_/test_api.py +++ b/Lib/test/test_importlib/import_/test_api.py @@ -4,6 +4,7 @@ import sys import types import unittest +import warnings PKG_NAME = 'fine' SUBMOD_NAME = 'fine.bogus' @@ -100,6 +101,36 @@ def test_blocked_fromlist(self): class OldAPITests(APITest): bad_finder_loader = BadLoaderFinder + def test_raises_ModuleNotFoundError(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_raises_ModuleNotFoundError() + + def test_name_requires_rparition(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_name_requires_rparition() + + def test_negative_level(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_negative_level() + + def test_nonexistent_fromlist_entry(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_nonexistent_fromlist_entry() + + def test_fromlist_load_error_propagates(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_fromlist_load_error_propagates + + def test_blocked_fromlist(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_blocked_fromlist() + (Frozen_OldAPITests, Source_OldAPITests diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py index 8079add5b2..0f987b2210 100644 --- a/Lib/test/test_importlib/import_/test_caching.py +++ b/Lib/test/test_importlib/import_/test_caching.py @@ -3,6 +3,7 @@ import sys from types import MethodType import unittest +import warnings class UseCache: @@ -63,30 +64,36 @@ def load_module(self, fullname): # to when to use the module in sys.modules and when not to. def test_using_cache_after_loader(self): # [from cache on return] - with self.create_mock('module') as mock: - with util.import_state(meta_path=[mock]): - module = self.__import__('module') - self.assertEqual(id(module), id(sys.modules['module'])) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + with self.create_mock('module') as mock: + with util.import_state(meta_path=[mock]): + module = self.__import__('module') + self.assertEqual(id(module), id(sys.modules['module'])) # See test_using_cache_after_loader() for reasoning. def test_using_cache_for_assigning_to_attribute(self): # [from cache to attribute] - with self.create_mock('pkg.__init__', 'pkg.module') as importer: - with util.import_state(meta_path=[importer]): - module = self.__import__('pkg.module') - self.assertTrue(hasattr(module, 'module')) - self.assertEqual(id(module.module), - id(sys.modules['pkg.module'])) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + with self.create_mock('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg.module') + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(id(module.module), + id(sys.modules['pkg.module'])) # See test_using_cache_after_loader() for reasoning. def test_using_cache_for_fromlist(self): # [from cache for fromlist] - with self.create_mock('pkg.__init__', 'pkg.module') as importer: - with util.import_state(meta_path=[importer]): - module = self.__import__('pkg', fromlist=['module']) - self.assertTrue(hasattr(module, 'module')) - self.assertEqual(id(module.module), - id(sys.modules['pkg.module'])) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + with self.create_mock('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg', fromlist=['module']) + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(id(module.module), + id(sys.modules['pkg.module'])) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py index 018c172176..deb21710a6 100644 --- a/Lib/test/test_importlib/import_/test_fromlist.py +++ b/Lib/test/test_importlib/import_/test_fromlist.py @@ -24,7 +24,7 @@ def test_return_from_import(self): def test_return_from_from_import(self): # [from return] - with util.mock_modules('pkg.__init__', 'pkg.module')as importer: + with util.mock_spec('pkg.__init__', 'pkg.module')as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg.module', fromlist=['attr']) self.assertEqual(module.__name__, 'pkg.module') @@ -52,14 +52,14 @@ class HandlingFromlist: def test_object(self): # [object case] - with util.mock_modules('module') as importer: + with util.mock_spec('module') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('module', fromlist=['attr']) self.assertEqual(module.__name__, 'module') def test_nonexistent_object(self): # [bad object] - with util.mock_modules('module') as importer: + with util.mock_spec('module') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('module', fromlist=['non_existent']) self.assertEqual(module.__name__, 'module') @@ -67,7 +67,7 @@ def test_nonexistent_object(self): def test_module_from_package(self): # [module] - with util.mock_modules('pkg.__init__', 'pkg.module') as importer: + with util.mock_spec('pkg.__init__', 'pkg.module') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['module']) self.assertEqual(module.__name__, 'pkg') @@ -75,7 +75,7 @@ def test_module_from_package(self): self.assertEqual(module.module.__name__, 'pkg.module') def test_nonexistent_from_package(self): - with util.mock_modules('pkg.__init__') as importer: + with util.mock_spec('pkg.__init__') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['non_existent']) self.assertEqual(module.__name__, 'pkg') @@ -87,7 +87,7 @@ def test_module_from_package_triggers_ModuleNotFoundError(self): # ModuleNotFoundError propagate. def module_code(): import i_do_not_exist - with util.mock_modules('pkg.__init__', 'pkg.mod', + with util.mock_spec('pkg.__init__', 'pkg.mod', module_code={'pkg.mod': module_code}) as importer: with util.import_state(meta_path=[importer]): with self.assertRaises(ModuleNotFoundError) as exc: @@ -95,14 +95,14 @@ def module_code(): self.assertEqual('i_do_not_exist', exc.exception.name) def test_empty_string(self): - with util.mock_modules('pkg.__init__', 'pkg.mod') as importer: + with util.mock_spec('pkg.__init__', 'pkg.mod') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg.mod', fromlist=['']) self.assertEqual(module.__name__, 'pkg.mod') def basic_star_test(self, fromlist=['*']): # [using *] - with util.mock_modules('pkg.__init__', 'pkg.module') as mock: + with util.mock_spec('pkg.__init__', 'pkg.module') as mock: with util.import_state(meta_path=[mock]): mock['pkg'].__all__ = ['module'] module = self.__import__('pkg', fromlist=fromlist) @@ -119,7 +119,7 @@ def test_fromlist_as_tuple(self): def test_star_with_others(self): # [using * with others] - context = util.mock_modules('pkg.__init__', 'pkg.module1', 'pkg.module2') + context = util.mock_spec('pkg.__init__', 'pkg.module1', 'pkg.module2') with context as mock: with util.import_state(meta_path=[mock]): mock['pkg'].__all__ = ['module1'] @@ -131,7 +131,7 @@ def test_star_with_others(self): self.assertEqual(module.module2.__name__, 'pkg.module2') def test_nonexistent_in_all(self): - with util.mock_modules('pkg.__init__') as importer: + with util.mock_spec('pkg.__init__') as importer: with util.import_state(meta_path=[importer]): importer['pkg'].__all__ = ['non_existent'] module = self.__import__('pkg', fromlist=['*']) @@ -139,7 +139,7 @@ def test_nonexistent_in_all(self): self.assertFalse(hasattr(module, 'non_existent')) def test_star_in_all(self): - with util.mock_modules('pkg.__init__') as importer: + with util.mock_spec('pkg.__init__') as importer: with util.import_state(meta_path=[importer]): importer['pkg'].__all__ = ['*'] module = self.__import__('pkg', fromlist=['*']) @@ -147,7 +147,7 @@ def test_star_in_all(self): self.assertFalse(hasattr(module, '*')) def test_invalid_type(self): - with util.mock_modules('pkg.__init__') as importer: + with util.mock_spec('pkg.__init__') as importer: with util.import_state(meta_path=[importer]), \ warnings.catch_warnings(): warnings.simplefilter('error', BytesWarning) @@ -157,7 +157,7 @@ def test_invalid_type(self): self.__import__('pkg', fromlist=iter([b'attr'])) def test_invalid_type_in_all(self): - with util.mock_modules('pkg.__init__') as importer: + with util.mock_spec('pkg.__init__') as importer: with util.import_state(meta_path=[importer]), \ warnings.catch_warnings(): warnings.simplefilter('error', BytesWarning) diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py index 1ee79f3bc3..e19dd0e5bf 100644 --- a/Lib/test/test_importlib/import_/test_meta_path.py +++ b/Lib/test/test_importlib/import_/test_meta_path.py @@ -102,8 +102,20 @@ def test_with_path(self): self.assertEqual(args[0], mod_name) self.assertIs(args[1], path) +class CallSignoreSuppressImportWarning(CallSignature): -class CallSignaturePEP302(CallSignature): + def test_no_path(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_no_path() + + def test_with_path(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + super().test_no_path() + + +class CallSignaturePEP302(CallSignoreSuppressImportWarning): mock_modules = util.mock_modules finder_name = 'find_module' diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py index 7beaee1ca0..3b976fdfa9 100644 --- a/Lib/test/test_importlib/import_/test_path.py +++ b/Lib/test/test_importlib/import_/test_path.py @@ -77,7 +77,8 @@ def test_empty_path_hooks(self): with util.import_state(path_importer_cache={}, path_hooks=[], path=[path_entry]): with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') + warnings.simplefilter('always', ImportWarning) + warnings.simplefilter('ignore', DeprecationWarning) self.assertIsNone(self.find('os')) self.assertIsNone(sys.path_importer_cache[path_entry]) self.assertEqual(len(w), 1) @@ -125,12 +126,16 @@ def find_module(self, fullname): failing_finder.to_return = None path = 'testing path' with util.import_state(path_importer_cache={path: failing_finder}): - self.assertIsNone( + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + self.assertIsNone( self.machinery.PathFinder.find_spec('whatever', [path])) success_finder = TestFinder() success_finder.to_return = __loader__ with util.import_state(path_importer_cache={path: success_finder}): - spec = self.machinery.PathFinder.find_spec('whatever', [path]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + spec = self.machinery.PathFinder.find_spec('whatever', [path]) self.assertEqual(spec.loader, __loader__) def test_finder_with_find_loader(self): @@ -141,12 +146,16 @@ def find_loader(self, fullname): return self.loader, self.portions path = 'testing path' with util.import_state(path_importer_cache={path: TestFinder()}): - self.assertIsNone( + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + self.assertIsNone( self.machinery.PathFinder.find_spec('whatever', [path])) success_finder = TestFinder() success_finder.loader = __loader__ with util.import_state(path_importer_cache={path: success_finder}): - spec = self.machinery.PathFinder.find_spec('whatever', [path]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + spec = self.machinery.PathFinder.find_spec('whatever', [path]) self.assertEqual(spec.loader, __loader__) def test_finder_with_find_spec(self): @@ -210,7 +219,9 @@ def test_invalidate_caches_clear_out_None(self): class FindModuleTests(FinderTests): def find(self, *args, **kwargs): - return self.machinery.PathFinder.find_module(*args, **kwargs) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return self.machinery.PathFinder.find_module(*args, **kwargs) def check_found(self, found, importer): self.assertIs(found, importer) @@ -250,7 +261,9 @@ def find_module(fullname): with util.import_state(path=[Finder.path_location]+sys.path[:], path_hooks=[Finder]): - self.machinery.PathFinder.find_spec('importlib') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + self.machinery.PathFinder.find_spec('importlib') def test_finder_with_failing_find_module(self): # PathEntryFinder with find_module() defined should work. @@ -268,7 +281,10 @@ def find_module(fullname): with util.import_state(path=[Finder.path_location]+sys.path[:], path_hooks=[Finder]): - self.machinery.PathFinder.find_module('importlib') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + warnings.simplefilter("ignore", DeprecationWarning) + self.machinery.PathFinder.find_module('importlib') (Frozen_PEFTests, diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 8a95a32109..41aa182699 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -133,6 +133,7 @@ def callback(global_): self.assertEqual(module.__name__, 'pkg') self.assertTrue(hasattr(module, 'subpkg2')) self.assertEqual(module.subpkg2.attr, 'pkg.subpkg2.__init__') + self.relative_import_test(create, globals_, callback) def test_deep_import(self): # [deep import] @@ -156,7 +157,7 @@ def test_too_high_from_package(self): {'__name__': 'pkg', '__path__': ['blah']}) def callback(global_): self.__import__('pkg') - with self.assertRaises(ValueError): + with self.assertRaises(ImportError): self.__import__('', global_, fromlist=['top_level'], level=2) self.relative_import_test(create, globals_, callback) @@ -167,7 +168,7 @@ def test_too_high_from_module(self): globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'} def callback(global_): self.__import__('pkg') - with self.assertRaises(ValueError): + with self.assertRaises(ImportError): self.__import__('', global_, fromlist=['top_level'], level=2) self.relative_import_test(create, globals_, callback) diff --git a/Lib/test/test_importlib/namespacedata01/binary.file b/Lib/test/test_importlib/namespacedata01/binary.file new file mode 100644 index 0000000000..eaf36c1dac Binary files /dev/null and b/Lib/test/test_importlib/namespacedata01/binary.file differ diff --git a/Lib/test/test_importlib/namespacedata01/utf-16.file b/Lib/test/test_importlib/namespacedata01/utf-16.file new file mode 100644 index 0000000000..2cb772295e Binary files /dev/null and b/Lib/test/test_importlib/namespacedata01/utf-16.file differ diff --git a/Lib/test/test_importlib/namespacedata01/utf-8.file b/Lib/test/test_importlib/namespacedata01/utf-8.file new file mode 100644 index 0000000000..1c0132ad90 --- /dev/null +++ b/Lib/test/test_importlib/namespacedata01/utf-8.file @@ -0,0 +1 @@ +Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/partial/cfimport.py b/Lib/test/test_importlib/partial/cfimport.py new file mode 100644 index 0000000000..c92d2fe1dd --- /dev/null +++ b/Lib/test/test_importlib/partial/cfimport.py @@ -0,0 +1,38 @@ +import os +import sys +import threading +import traceback + + +NLOOPS = 50 +NTHREADS = 30 + + +def t1(): + try: + from concurrent.futures import ThreadPoolExecutor + except Exception: + traceback.print_exc() + os._exit(1) + +def t2(): + try: + from concurrent.futures.thread import ThreadPoolExecutor + except Exception: + traceback.print_exc() + os._exit(1) + +def main(): + for j in range(NLOOPS): + threads = [] + for i in range(NTHREADS): + threads.append(threading.Thread(target=t2 if i % 1 else t1)) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + sys.modules.pop('concurrent.futures', None) + sys.modules.pop('concurrent.futures.thread', None) + +if __name__ == "__main__": + main() diff --git a/Lib/test/test_importlib/partial/pool_in_threads.py b/Lib/test/test_importlib/partial/pool_in_threads.py new file mode 100644 index 0000000000..faa7867b81 --- /dev/null +++ b/Lib/test/test_importlib/partial/pool_in_threads.py @@ -0,0 +1,27 @@ +import multiprocessing +import os +import threading +import traceback + + +def t(): + try: + with multiprocessing.Pool(1): + pass + except Exception: + traceback.print_exc() + os._exit(1) + + +def main(): + threads = [] + for i in range(20): + threads.append(threading.Thread(target=t)) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +if __name__ == "__main__": + main() diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py index f7e2773908..19543f4a66 100644 --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -67,7 +67,7 @@ class CaseSensitivityTestPEP302(CaseSensitivityTest): def find(self, finder): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - return finder.find_module(self.name) + return finder.find_module(self.name) (Frozen_CaseSensitivityTestPEP302, diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index 8c74151aa0..8205ff62a9 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -17,7 +17,7 @@ import unittest import warnings -from test.support.import_helper import unload, make_legacy_pyc +from test.support.import_helper import make_legacy_pyc, unload from test.test_py_compile import without_source_date_epoch from test.test_py_compile import SourceDateEpochTestMeta @@ -124,7 +124,7 @@ def test_module_reuse(self): module = loader.load_module('_temp') module_id = id(module) module_dict_id = id(module.__dict__) - with open(mapping['_temp'], 'w') as file: + with open(mapping['_temp'], 'w', encoding='utf-8') as file: file.write("testing_var = 42\n") with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) @@ -145,7 +145,7 @@ def test_state_after_failure(self): orig_module = types.ModuleType(name) for attr in attributes: setattr(orig_module, attr, value) - with open(mapping[name], 'w') as file: + with open(mapping[name], 'w', encoding='utf-8') as file: file.write('+++ bad syntax +++') loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) with self.assertRaises(SyntaxError): @@ -162,7 +162,7 @@ def test_state_after_failure(self): # [syntax error] def test_bad_syntax(self): with util.create_modules('_temp') as mapping: - with open(mapping['_temp'], 'w') as file: + with open(mapping['_temp'], 'w', encoding='utf-8') as file: file.write('=') loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) with self.assertRaises(SyntaxError): @@ -175,7 +175,7 @@ def test_file_from_empty_string_dir(self): # Loading a module found from an empty string entry on sys.path should # not only work, but keep all attributes relative. file_path = '_temp.py' - with open(file_path, 'w') as file: + with open(file_path, 'w', encoding='utf-8') as file: file.write("# test file for importlib") try: with util.uncache('_temp'): @@ -199,7 +199,7 @@ def test_timestamp_overflow(self): with util.create_modules('_temp') as mapping: source = mapping['_temp'] compiled = self.util.cache_from_source(source) - with open(source, 'w') as f: + with open(source, 'w', encoding='utf-8') as f: f.write("x = 5") try: os.utime(source, (2 ** 33 - 5, 2 ** 33 - 5)) @@ -328,7 +328,7 @@ def test_unchecked_hash_based_pyc(self): @unittest.skip("TODO: RUSTPYTHON; successful only for Frozen") @util.writes_bytecode_files - def test_overiden_unchecked_hash_based_pyc(self): + def test_overridden_unchecked_hash_based_pyc(self): with util.create_modules('_temp') as mapping, \ unittest.mock.patch('_imp.check_hash_based_pycs', 'always'): source = mapping['_temp'] @@ -674,7 +674,7 @@ def test_read_only_bytecode(self): os.chmod(bytecode_path, stat.S_IWUSR) -# TODO: RustPython +# TODO: RUSTPYTHON # class SourceLoaderBadBytecodeTestPEP451( # SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451): # pass @@ -687,6 +687,7 @@ def test_read_only_bytecode(self): # util=importlib_util) +# TODO: RUSTPYTHON # class SourceLoaderBadBytecodeTestPEP302( # SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): # pass @@ -773,7 +774,7 @@ def test_non_code_marshal(self): self._test_non_code_marshal(del_source=True) -# TODO: RustPython +# TODO: RUSTPYTHON # class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, # BadBytecodeTestPEP451): # pass diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index e9207deedc..1cf6719c12 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -127,7 +127,7 @@ def test_empty_string_for_dir(self): # The empty string from sys.path means to search in the cwd. finder = self.machinery.FileFinder('', (self.machinery.SourceFileLoader, self.machinery.SOURCE_SUFFIXES)) - with open('mod.py', 'w') as file: + with open('mod.py', 'w', encoding='utf-8') as file: file.write("# test file for importlib") try: loader = self._find(finder, 'mod', loader_only=True) @@ -186,6 +186,10 @@ def test_ignore_file(self): found = self._find(finder, 'doesnotexist') self.assertEqual(found, self.NOT_FOUND) + # TODO: RUSTPYTHON + if sys.platform == 'win32': + test_ignore_file = unittest.expectedFailure(test_ignore_file) + class FinderTestsPEP451(FinderTests): diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py index f90ea94408..2df276cbcb 100644 --- a/Lib/test/test_importlib/source/test_source_encoding.py +++ b/Lib/test/test_importlib/source/test_source_encoding.py @@ -23,7 +23,7 @@ class EncodingTest: PEP 263 specifies how that can change on a per-file basis. Either the first or second line can contain the encoding line [encoding first line] - encoding second line]. If the file has the BOM marker it is considered UTF-8 + [encoding second line]. If the file has the BOM marker it is considered UTF-8 implicitly [BOM]. If any encoding is specified it must be UTF-8, else it is an error [BOM and utf-8][BOM conflict]. diff --git a/Lib/test/test_importlib/stubs.py b/Lib/test/test_importlib/stubs.py new file mode 100644 index 0000000000..e5b011c399 --- /dev/null +++ b/Lib/test/test_importlib/stubs.py @@ -0,0 +1,10 @@ +import unittest + + +class fake_filesystem_unittest: + """ + Stubbed version of the pyfakefs module + """ + class TestCase(unittest.TestCase): + def setUpPyfakefs(self): + self.skipTest("pyfakefs not available") diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index ca2d9a7b8a..08a416803b 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -2,6 +2,7 @@ import marshal import os import sys +from test import support from test.support import import_helper import types import unittest @@ -29,7 +30,7 @@ def setUp(self): self.superclasses = [getattr(self.abc, class_name) for class_name in self.superclass_names] if hasattr(self, 'subclass_names'): - # Because import_helper.import_fresh_module() creates a new + # Because test.support.import_fresh_module() creates a new # importlib._bootstrap per module, inheritance checks fail when # checking across module boundaries (i.e. the _bootstrap in abc is # not the same as the one in machinery). That means stealing one of @@ -54,7 +55,7 @@ def test_superclasses(self): class MetaPathFinder(InheritanceTests): - superclass_names = ['Finder'] + superclass_names = [] subclass_names = ['BuiltinImporter', 'FrozenImporter', 'PathFinder', 'WindowsRegistryFinder'] @@ -65,7 +66,7 @@ class MetaPathFinder(InheritanceTests): class PathEntryFinder(InheritanceTests): - superclass_names = ['Finder'] + superclass_names = [] subclass_names = ['FileFinder'] @@ -219,12 +220,14 @@ def test_load_module(self): def test_module_repr(self): mod = types.ModuleType('blah') - with self.assertRaises(NotImplementedError): - self.ins.module_repr(mod) - original_repr = repr(mod) - mod.__loader__ = self.ins - # Should still return a proper repr. - self.assertTrue(repr(mod)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + with self.assertRaises(NotImplementedError): + self.ins.module_repr(mod) + original_repr = repr(mod) + mod.__loader__ = self.ins + # Should still return a proper repr. + self.assertTrue(repr(mod)) (Frozen_LDefaultTests, @@ -337,7 +340,9 @@ def test_is_resource(self): self.ins.is_resource('dummy_file') def test_contents(self): - self.assertEqual([], list(self.ins.contents())) + with self.assertRaises(FileNotFoundError): + self.ins.contents() + (Frozen_RRDefaultTests, Source_RRDefaultsTests @@ -357,13 +362,27 @@ def find_spec(self, fullname, path, target=None): return MetaPathSpecFinder() - def test_no_spec(self): + def test_find_module(self): finder = self.finder(None) path = ['a', 'b', 'c'] name = 'blah' with self.assertWarns(DeprecationWarning): found = finder.find_module(name, path) self.assertIsNone(found) + + def test_find_spec_with_explicit_target(self): + loader = object() + spec = self.util.spec_from_loader('blah', loader) + finder = self.finder(spec) + found = finder.find_spec('blah', 'blah', None) + self.assertEqual(found, spec) + + def test_no_spec(self): + finder = self.finder(None) + path = ['a', 'b', 'c'] + name = 'blah' + found = finder.find_spec(name, path, None) + self.assertIsNone(found) self.assertEqual(name, finder.called_for[0]) self.assertEqual(path, finder.called_for[1]) @@ -371,9 +390,8 @@ def test_spec(self): loader = object() spec = self.util.spec_from_loader('blah', loader) finder = self.finder(spec) - with self.assertWarns(DeprecationWarning): - found = finder.find_module('blah', None) - self.assertIs(found, spec.loader) + found = finder.find_spec('blah', None) + self.assertIs(found, spec) (Frozen_MPFFindModuleTests, @@ -444,32 +462,36 @@ def is_package(self, fullname): return SpecLoader() def test_fresh(self): - loader = self.loader() - name = 'blah' - with test_util.uncache(name): - loader.load_module(name) - module = loader.found - self.assertIs(sys.modules[name], module) - self.assertEqual(loader, module.__loader__) - self.assertEqual(loader, module.__spec__.loader) - self.assertEqual(name, module.__name__) - self.assertEqual(name, module.__spec__.name) - self.assertIsNotNone(module.__path__) - self.assertIsNotNone(module.__path__, - module.__spec__.submodule_search_locations) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + loader = self.loader() + name = 'blah' + with test_util.uncache(name): + loader.load_module(name) + module = loader.found + self.assertIs(sys.modules[name], module) + self.assertEqual(loader, module.__loader__) + self.assertEqual(loader, module.__spec__.loader) + self.assertEqual(name, module.__name__) + self.assertEqual(name, module.__spec__.name) + self.assertIsNotNone(module.__path__) + self.assertIsNotNone(module.__path__, + module.__spec__.submodule_search_locations) def test_reload(self): - name = 'blah' - loader = self.loader() - module = types.ModuleType(name) - module.__spec__ = self.util.spec_from_loader(name, loader) - module.__loader__ = loader - with test_util.uncache(name): - sys.modules[name] = module - loader.load_module(name) - found = loader.found - self.assertIs(found, sys.modules[name]) - self.assertIs(module, sys.modules[name]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + name = 'blah' + loader = self.loader() + module = types.ModuleType(name) + module.__spec__ = self.util.spec_from_loader(name, loader) + module.__loader__ = loader + with test_util.uncache(name): + sys.modules[name] = module + loader.load_module(name) + found = loader.found + self.assertIs(found, sys.modules[name]) + self.assertIs(module, sys.modules[name]) (Frozen_LoaderLoadModuleTests, @@ -825,25 +847,29 @@ def test_load_module(self): # Loading a module should set __name__, __loader__, __package__, # __path__ (for packages), __file__, and __cached__. # The module should also be put into sys.modules. - with test_util.uncache(self.name): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = self.loader.load_module(self.name) - self.verify_module(module) - self.assertEqual(module.__path__, [os.path.dirname(self.path)]) - self.assertIn(self.name, sys.modules) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + with test_util.uncache(self.name): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertEqual(module.__path__, [os.path.dirname(self.path)]) + self.assertIn(self.name, sys.modules) def test_package_settings(self): # __package__ needs to be set, while __path__ is set on if the module # is a package. # Testing the values for a package are covered by test_load_module. - self.setUp(is_package=False) - with test_util.uncache(self.name): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = self.loader.load_module(self.name) - self.verify_module(module) - self.assertFalse(hasattr(module, '__path__')) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + self.setUp(is_package=False) + with test_util.uncache(self.name): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertFalse(hasattr(module, '__path__')) # TODO: RUSTPYTHON @unittest.expectedFailure diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index cdc2e8040b..763b2add07 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -7,8 +7,8 @@ import os.path import sys from test import support -from test.support import os_helper from test.support import import_helper +from test.support import os_helper import types import unittest import warnings @@ -20,7 +20,7 @@ class ImportModuleTests: def test_module_import(self): # Test importing a top-level module. - with test_util.mock_modules('top_level') as mock: + with test_util.mock_spec('top_level') as mock: with test_util.import_state(meta_path=[mock]): module = self.init.import_module('top_level') self.assertEqual(module.__name__, 'top_level') @@ -30,7 +30,7 @@ def test_absolute_package_import(self): pkg_name = 'pkg' pkg_long_name = '{0}.__init__'.format(pkg_name) name = '{0}.mod'.format(pkg_name) - with test_util.mock_modules(pkg_long_name, name) as mock: + with test_util.mock_spec(pkg_long_name, name) as mock: with test_util.import_state(meta_path=[mock]): module = self.init.import_module(name) self.assertEqual(module.__name__, name) @@ -42,7 +42,7 @@ def test_shallow_relative_package_import(self): module_name = 'mod' absolute_name = '{0}.{1}'.format(pkg_name, module_name) relative_name = '.{0}'.format(module_name) - with test_util.mock_modules(pkg_long_name, absolute_name) as mock: + with test_util.mock_spec(pkg_long_name, absolute_name) as mock: with test_util.import_state(meta_path=[mock]): self.init.import_module(pkg_name) module = self.init.import_module(relative_name, pkg_name) @@ -50,7 +50,7 @@ def test_shallow_relative_package_import(self): def test_deep_relative_package_import(self): modules = ['a.__init__', 'a.b.__init__', 'a.c'] - with test_util.mock_modules(*modules) as mock: + with test_util.mock_spec(*modules) as mock: with test_util.import_state(meta_path=[mock]): self.init.import_module('a') self.init.import_module('a.b') @@ -63,7 +63,7 @@ def test_absolute_import_with_package(self): pkg_name = 'pkg' pkg_long_name = '{0}.__init__'.format(pkg_name) name = '{0}.mod'.format(pkg_name) - with test_util.mock_modules(pkg_long_name, name) as mock: + with test_util.mock_spec(pkg_long_name, name) as mock: with test_util.import_state(meta_path=[mock]): self.init.import_module(pkg_name) module = self.init.import_module(name, pkg_name) @@ -88,7 +88,7 @@ def load_b(): b_load_count += 1 code = {'a': load_a, 'a.b': load_b} modules = ['a.__init__', 'a.b'] - with test_util.mock_modules(*modules, module_code=code) as mock: + with test_util.mock_spec(*modules, module_code=code) as mock: with test_util.import_state(meta_path=[mock]): self.init.import_module('a.b') self.assertEqual(b_load_count, 1) @@ -151,6 +151,7 @@ def test_success(self): with test_util.import_state(meta_path=[self.FakeMetaFinder]): with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) + warnings.simplefilter('ignore', ImportWarning) self.assertEqual((name, None), self.init.find_loader(name)) def test_success_path(self): @@ -161,6 +162,7 @@ def test_success_path(self): with test_util.import_state(meta_path=[self.FakeMetaFinder]): with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) + warnings.simplefilter('ignore', ImportWarning) self.assertEqual((name, path), self.init.find_loader(name, path)) @@ -212,8 +214,8 @@ def code(): module = type(sys)('top_level') module.spam = 3 sys.modules['top_level'] = module - mock = test_util.mock_modules('top_level', - module_code={'top_level': code}) + mock = test_util.mock_spec('top_level', + module_code={'top_level': code}) with mock: with test_util.import_state(meta_path=[mock]): module = self.init.import_module('top_level') @@ -310,7 +312,7 @@ def test_reload_namespace_changed(self): '__file__': None, } os.mkdir(name) - with open(bad_path, 'w') as init_file: + with open(bad_path, 'w', encoding='utf-8') as init_file: init_file.write('eggs = None') module = self.init.import_module(name) ns = vars(module).copy() @@ -365,7 +367,7 @@ def test_reload_submodule(self): def test_module_missing_spec(self): #Test that reload() throws ModuleNotFounderror when reloading - # a module who's missing a spec. (bpo-29851) + # a module whose missing a spec. (bpo-29851) name = 'spam' with test_util.uncache(name): module = sys.modules[name] = types.ModuleType(name) @@ -438,9 +440,9 @@ def test_everyone_has___loader__(self): with self.subTest(name=name): self.assertTrue(hasattr(module, '__loader__'), '{!r} lacks a __loader__ attribute'.format(name)) - if self.machinery.BuiltinImporter.find_module(name): + if self.machinery.BuiltinImporter.find_spec(name): self.assertIsNot(module.__loader__, None) - elif self.machinery.FrozenImporter.find_module(name): + elif self.machinery.FrozenImporter.find_spec(name): self.assertIsNot(module.__loader__, None) def test_everyone_has___spec__(self): @@ -448,9 +450,9 @@ def test_everyone_has___spec__(self): if isinstance(module, types.ModuleType): with self.subTest(name=name): self.assertTrue(hasattr(module, '__spec__')) - if self.machinery.BuiltinImporter.find_module(name): + if self.machinery.BuiltinImporter.find_spec(name): self.assertIsNot(module.__spec__, None) - elif self.machinery.FrozenImporter.find_module(name): + elif self.machinery.FrozenImporter.find_spec(name): self.assertIsNot(module.__spec__, None) diff --git a/Lib/test/test_importlib/test_files.py b/Lib/test/test_importlib/test_files.py new file mode 100644 index 0000000000..481829b742 --- /dev/null +++ b/Lib/test/test_importlib/test_files.py @@ -0,0 +1,39 @@ +import typing +import unittest + +from importlib import resources +from importlib.abc import Traversable +from . import data01 +from . import util + + +class FilesTests: + def test_read_bytes(self): + files = resources.files(self.data) + actual = files.joinpath('utf-8.file').read_bytes() + assert actual == b'Hello, UTF-8 world!\n' + + def test_read_text(self): + files = resources.files(self.data) + actual = files.joinpath('utf-8.file').read_text(encoding='utf-8') + assert actual == 'Hello, UTF-8 world!\n' + + @unittest.skipUnless( + hasattr(typing, 'runtime_checkable'), + "Only suitable when typing supports runtime_checkable", + ) + def test_traversable(self): + assert isinstance(resources.files(self.data), Traversable) + + +class OpenDiskTests(FilesTests, unittest.TestCase): + def setUp(self): + self.data = data01 + + +class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index 9afc1559b4..78372059cc 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -137,22 +137,16 @@ def test_all_locks(self): self.assertEqual(0, len(self.bootstrap._module_locks), self.bootstrap._module_locks) -# TODO: RustPython +# TODO: RUSTPYTHON # (Frozen_LifetimeTests, # Source_LifetimeTests # ) = test_util.test_both(LifetimeTests, init=init) -@threading_helper.reap_threads -def test_main(): - support.run_unittest(Frozen_ModuleLockAsRLockTests, - Source_ModuleLockAsRLockTests, - Frozen_DeadlockAvoidanceTests, - Source_DeadlockAvoidanceTests, - # Frozen_LifetimeTests, - # Source_LifetimeTests - ) +def setUpModule(): + thread_info = threading_helper.threading_setup() + unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) if __name__ == '__main__': - test_main() + unittets.main() diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py new file mode 100644 index 0000000000..52cb63712a --- /dev/null +++ b/Lib/test/test_importlib/test_main.py @@ -0,0 +1,273 @@ +import re +import json +import pickle +import textwrap +import unittest +import warnings +import importlib.metadata + +try: + import pyfakefs.fake_filesystem_unittest as ffs +except ImportError: + from .stubs import fake_filesystem_unittest as ffs + +from . import fixtures +from importlib.metadata import ( + Distribution, + EntryPoint, + PackageNotFoundError, + distributions, + entry_points, + metadata, + version, +) + + +class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + dist = Distribution.from_name('distinfo-pkg') + assert isinstance(dist.version, str) + assert re.match(self.version_pattern, dist.version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(PackageNotFoundError): + Distribution.from_name('does-not-exist') + + def test_package_not_found_mentions_metadata(self): + # When a package is not found, that could indicate that the + # packgae is not installed or that it is installed without + # metadata. Ensure the exception mentions metadata to help + # guide users toward the cause. See #124. + with self.assertRaises(PackageNotFoundError) as ctx: + Distribution.from_name('does-not-exist') + + assert "metadata" in str(ctx.exception) + + def test_new_style_classes(self): + self.assertIsInstance(Distribution, type) + + +class ImportTests(fixtures.DistInfoPkg, unittest.TestCase): + def test_import_nonexistent_module(self): + # Ensure that the MetadataPathFinder does not crash an import of a + # non-existent module. + with self.assertRaises(ImportError): + importlib.import_module('does_not_exist') + + def test_resolve(self): + ep = entry_points(group='entries')['main'] + self.assertEqual(ep.load().__name__, "main") + + def test_entrypoint_with_colon_in_name(self): + ep = entry_points(group='entries')['ns:sub'] + self.assertEqual(ep.value, 'mod:main') + + def test_resolve_without_attr(self): + ep = EntryPoint( + name='ep', + value='importlib.metadata', + group='grp', + ) + assert ep.load() is importlib.metadata + + +class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def pkg_with_dashes(site_dir): + """ + Create minimal metadata for a package with dashes + in the name (and thus underscores in the filename). + """ + metadata_dir = site_dir / 'my_pkg.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w', encoding='utf-8') as strm: + strm.write('Version: 1.0\n') + return 'my-pkg' + + def test_dashes_in_dist_name_found_as_underscores(self): + # For a package with a dash in the name, the dist-info metadata + # uses underscores in the name. Ensure the metadata loads. + pkg_name = self.pkg_with_dashes(self.site_dir) + assert version(pkg_name) == '1.0' + + @staticmethod + def pkg_with_mixed_case(site_dir): + """ + Create minimal metadata for a package with mixed case + in the name. + """ + metadata_dir = site_dir / 'CherryPy.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w', encoding='utf-8') as strm: + strm.write('Version: 1.0\n') + return 'CherryPy' + + def test_dist_name_found_as_any_case(self): + # Ensure the metadata loads when queried with any case. + pkg_name = self.pkg_with_mixed_case(self.site_dir) + assert version(pkg_name) == '1.0' + assert version(pkg_name.lower()) == '1.0' + assert version(pkg_name.upper()) == '1.0' + + +class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def pkg_with_non_ascii_description(site_dir): + """ + Create minimal metadata for a package with non-ASCII in + the description. + """ + metadata_dir = site_dir / 'portend.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w', encoding='utf-8') as fp: + fp.write('Description: pôrˈtend') + return 'portend' + + @staticmethod + def pkg_with_non_ascii_description_egg_info(site_dir): + """ + Create minimal metadata for an egg-info package with + non-ASCII in the description. + """ + metadata_dir = site_dir / 'portend.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w', encoding='utf-8') as fp: + fp.write( + textwrap.dedent( + """ + Name: portend + + pôrˈtend + """ + ).strip() + ) + return 'portend' + + def test_metadata_loads(self): + pkg_name = self.pkg_with_non_ascii_description(self.site_dir) + meta = metadata(pkg_name) + assert meta['Description'] == 'pôrˈtend' + + def test_metadata_loads_egg_info(self): + pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir) + meta = metadata(pkg_name) + assert meta['Description'] == 'pôrˈtend' + + +class DiscoveryTests(fixtures.EggInfoPkg, fixtures.DistInfoPkg, unittest.TestCase): + def test_package_discovery(self): + dists = list(distributions()) + assert all(isinstance(dist, Distribution) for dist in dists) + assert any(dist.metadata['Name'] == 'egginfo-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) + + def test_invalid_usage(self): + with self.assertRaises(ValueError): + list(distributions(context='something', name='else')) + + +class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + def test_egg_info(self): + # make an `EGG-INFO` directory that's unrelated + self.site_dir.joinpath('EGG-INFO').mkdir() + # used to crash with `IsADirectoryError` + with self.assertRaises(PackageNotFoundError): + version('unknown-package') + + def test_egg(self): + egg = self.site_dir.joinpath('foo-3.6.egg') + egg.mkdir() + with self.add_sys_path(egg): + with self.assertRaises(PackageNotFoundError): + version('foo') + + +class MissingSysPath(fixtures.OnSysPath, unittest.TestCase): + site_dir = '/does-not-exist' + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + importlib.metadata.distributions() + + +class InaccessibleSysPath(fixtures.OnSysPath, ffs.TestCase): + site_dir = '/access-denied' + + def setUp(self): + super(InaccessibleSysPath, self).setUp() + self.setUpPyfakefs() + self.fs.create_dir(self.site_dir, perm_bits=000) + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + list(importlib.metadata.distributions()) + + +class TestEntryPoints(unittest.TestCase): + def __init__(self, *args): + super(TestEntryPoints, self).__init__(*args) + self.ep = importlib.metadata.EntryPoint('name', 'value', 'group') + + def test_entry_point_pickleable(self): + revived = pickle.loads(pickle.dumps(self.ep)) + assert revived == self.ep + + def test_immutable(self): + """EntryPoints should be immutable""" + with self.assertRaises(AttributeError): + self.ep.name = 'badactor' + + def test_repr(self): + assert 'EntryPoint' in repr(self.ep) + assert 'name=' in repr(self.ep) + assert "'name'" in repr(self.ep) + + def test_hashable(self): + # EntryPoints should be hashable. + hash(self.ep) + + def test_json_dump(self): + # json should not expect to be able to dump an EntryPoint. + with self.assertRaises(Exception): + with warnings.catch_warnings(record=True): + json.dumps(self.ep) + + def test_module(self): + assert self.ep.module == 'value' + + def test_attr(self): + assert self.ep.attr is None + + def test_sortable(self): + # EntryPoint objects are sortable, but result is undefined. + sorted( + [ + EntryPoint('b', 'val', 'group'), + EntryPoint('a', 'val', 'group'), + ] + ) + + +class FileSystem( + fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder, unittest.TestCase +): + def test_unicode_dir_on_sys_path(self): + # Ensure a Unicode subdirectory of a directory on sys.path + # does not crash. + fixtures.build_files( + {self.unicode_filename(): {}}, + prefix=self.site_dir, + ) + list(distributions()) diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py new file mode 100644 index 0000000000..600a6e42f6 --- /dev/null +++ b/Lib/test/test_importlib/test_metadata_api.py @@ -0,0 +1,335 @@ +import re +import textwrap +import unittest +import warnings +import importlib +import contextlib + +from . import fixtures +from importlib.metadata import ( + Distribution, + PackageNotFoundError, + distribution, + entry_points, + files, + metadata, + requires, + version, +) + + +@contextlib.contextmanager +def suppress_known_deprecation(): + with warnings.catch_warnings(record=True) as ctx: + warnings.simplefilter('default') + yield ctx + + +class APITests( + fixtures.EggInfoPkg, + fixtures.DistInfoPkg, + fixtures.DistInfoPkgWithDot, + fixtures.EggInfoFile, + unittest.TestCase, +): + + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + pkg_version = version('egginfo-pkg') + assert isinstance(pkg_version, str) + assert re.match(self.version_pattern, pkg_version) + + def test_retrieves_version_of_distinfo_pkg(self): + pkg_version = version('distinfo-pkg') + assert isinstance(pkg_version, str) + assert re.match(self.version_pattern, pkg_version) + + # TODO: RUSTPYTHON + def test_for_name_does_not_exist(self): + with self.assertRaises(PackageNotFoundError): + distribution('does-not-exist') + + def test_name_normalization(self): + names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.dot' + + def test_prefix_not_matched(self): + prefixes = 'p', 'pkg', 'pkg.' + for prefix in prefixes: + with self.subTest(prefix): + with self.assertRaises(PackageNotFoundError): + distribution(prefix) + + def test_for_top_level(self): + self.assertEqual( + distribution('egginfo-pkg').read_text('top_level.txt').strip(), 'mod' + ) + + def test_read_text(self): + top_level = [ + path for path in files('egginfo-pkg') if path.name == 'top_level.txt' + ][0] + self.assertEqual(top_level.read_text(), 'mod\n') + + def test_entry_points(self): + eps = entry_points() + assert 'entries' in eps.groups + entries = eps.select(group='entries') + assert 'main' in entries.names + ep = entries['main'] + self.assertEqual(ep.value, 'mod:main') + self.assertEqual(ep.extras, []) + + def test_entry_points_distribution(self): + entries = entry_points(group='entries') + for entry in ("main", "ns:sub"): + ep = entries[entry] + self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) + self.assertEqual(ep.dist.version, "1.0.0") + + def test_entry_points_unique_packages(self): + # Entry points should only be exposed for the first package + # on sys.path with a given name. + alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + alt_pkg = { + "distinfo_pkg-1.1.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Version: 1.1.0 + """, + "entry_points.txt": """ + [entries] + main = mod:altmain + """, + }, + } + fixtures.build_files(alt_pkg, alt_site_dir) + entries = entry_points(group='entries') + assert not any( + ep.dist.name == 'distinfo-pkg' and ep.dist.version == '1.0.0' + for ep in entries + ) + # ns:sub doesn't exist in alt_pkg + assert 'ns:sub' not in entries + + def test_entry_points_missing_name(self): + with self.assertRaises(KeyError): + entry_points(group='entries')['missing'] + + def test_entry_points_missing_group(self): + assert entry_points(group='missing') == () + + def test_entry_points_dict_construction(self): + # Prior versions of entry_points() returned simple lists and + # allowed casting those lists into maps by name using ``dict()``. + # Capture this now deprecated use-case. + with suppress_known_deprecation() as caught: + eps = dict(entry_points(group='entries')) + + assert 'main' in eps + assert eps['main'] == entry_points(group='entries')['main'] + + # check warning + expected = next(iter(caught)) + assert expected.category is DeprecationWarning + assert "Construction of dict of EntryPoints is deprecated" in str(expected) + + def test_entry_points_by_index(self): + """ + Prior versions of Distribution.entry_points would return a + tuple that allowed access by index. + Capture this now deprecated use-case + See python/importlib_metadata#300 and bpo-44246. + """ + eps = distribution('distinfo-pkg').entry_points + with suppress_known_deprecation() as caught: + eps[0] + + # check warning + expected = next(iter(caught)) + assert expected.category is DeprecationWarning + assert "Accessing entry points by index is deprecated" in str(expected) + + def test_entry_points_groups_getitem(self): + # Prior versions of entry_points() returned a dict. Ensure + # that callers using '.__getitem__()' are supported but warned to + # migrate. + with suppress_known_deprecation(): + entry_points()['entries'] == entry_points(group='entries') + + with self.assertRaises(KeyError): + entry_points()['missing'] + + def test_entry_points_groups_get(self): + # Prior versions of entry_points() returned a dict. Ensure + # that callers using '.get()' are supported but warned to + # migrate. + with suppress_known_deprecation(): + entry_points().get('missing', 'default') == 'default' + entry_points().get('entries', 'default') == entry_points()['entries'] + entry_points().get('missing', ()) == () + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_entry_points_allows_no_attributes(self): + ep = entry_points().select(group='entries', name='main') + with self.assertRaises(AttributeError): + ep.foo = 4 + + def test_metadata_for_this_package(self): + md = metadata('egginfo-pkg') + assert md['author'] == 'Steven Ma' + assert md['LICENSE'] == 'Unknown' + assert md['Name'] == 'egginfo-pkg' + classifiers = md.get_all('Classifier') + assert 'Topic :: Software Development :: Libraries' in classifiers + + @staticmethod + def _test_files(files): + root = files[0].root + for file in files: + assert file.root == root + assert not file.hash or file.hash.value + assert not file.hash or file.hash.mode == 'sha256' + assert not file.size or file.size >= 0 + assert file.locate().exists() + assert isinstance(file.read_binary(), bytes) + if file.name.endswith('.py'): + file.read_text() + + def test_file_hash_repr(self): + assertRegex = self.assertRegex + + util = [p for p in files('distinfo-pkg') if p.name == 'mod.py'][0] + assertRegex(repr(util.hash), '') + + def test_files_dist_info(self): + self._test_files(files('distinfo-pkg')) + + def test_files_egg_info(self): + self._test_files(files('egginfo-pkg')) + + def test_version_egg_info_file(self): + self.assertEqual(version('egginfo-file'), '0.1') + + def test_requires_egg_info_file(self): + requirements = requires('egginfo-file') + self.assertIsNone(requirements) + + def test_requires_egg_info(self): + deps = requires('egginfo-pkg') + assert len(deps) == 2 + assert any(dep == 'wheel >= 1.0; python_version >= "2.7"' for dep in deps) + + def test_requires_egg_info_empty(self): + fixtures.build_files( + { + 'requires.txt': '', + }, + self.site_dir.joinpath('egginfo_pkg.egg-info'), + ) + deps = requires('egginfo-pkg') + assert deps == [] + + def test_requires_dist_info(self): + deps = requires('distinfo-pkg') + assert len(deps) == 2 + assert all(deps) + assert 'wheel >= 1.0' in deps + assert "pytest; extra == 'test'" in deps + + def test_more_complex_deps_requires_text(self): + requires = textwrap.dedent( + """ + dep1 + dep2 + + [:python_version < "3"] + dep3 + + [extra1] + dep4 + dep6@ git+https://example.com/python/dep.git@v1.0.0 + + [extra2:python_version < "3"] + dep5 + """ + ) + deps = sorted(Distribution._deps_from_requires_text(requires)) + expected = [ + 'dep1', + 'dep2', + 'dep3; python_version < "3"', + 'dep4; extra == "extra1"', + 'dep5; (python_version < "3") and extra == "extra2"', + 'dep6@ git+https://example.com/python/dep.git@v1.0.0 ; extra == "extra1"', + ] + # It's important that the environment marker expression be + # wrapped in parentheses to avoid the following 'and' binding more + # tightly than some other part of the environment expression. + + assert deps == expected + + def test_as_json(self): + md = metadata('distinfo-pkg').json + assert 'name' in md + assert md['keywords'] == ['sample', 'package'] + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + assert len(md['requires_dist']) == 2 + + def test_as_json_egg_info(self): + md = metadata('egginfo-pkg').json + assert 'name' in md + assert md['keywords'] == ['sample', 'package'] + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + assert len(md['classifier']) == 2 + + def test_as_json_odd_case(self): + self.make_uppercase() + md = metadata('distinfo-pkg').json + assert 'name' in md + assert len(md['requires_dist']) == 2 + assert md['keywords'] == ['SAMPLE', 'PACKAGE'] + + +class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase): + def test_name_normalization(self): + names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.dot' + + def test_name_normalization_versionless_egg_info(self): + names = 'pkg.lot', 'pkg_lot', 'pkg-lot', 'pkg..lot', 'Pkg.Lot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.lot' + + +class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase): + def test_find_distributions_specified_path(self): + dists = Distribution.discover(path=[str(self.site_dir)]) + assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) + + def test_distribution_at_pathlib(self): + # Demonstrate how to load metadata direct from a directory. + dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' + dist = Distribution.at(dist_info_path) + assert dist.version == '1.0.0' + + def test_distribution_at_str(self): + dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' + dist = Distribution.at(str(dist_info_path)) + assert dist.version == '1.0.0' + + +class InvalidateCache(unittest.TestCase): + def test_invalidate_cache(self): + # No externally observable behavior, but ensures test coverage... + importlib.invalidate_caches() diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index a8f95a035e..295f97e3e1 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -2,7 +2,9 @@ import importlib import os import sys +import tempfile import unittest +import warnings from test.test_importlib import util @@ -82,7 +84,10 @@ def test_cant_import_other(self): def test_module_repr(self): import foo.one - self.assertEqual(repr(foo), "") + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self.assertEqual(foo.__spec__.loader.module_repr(foo), + "") class DynamicPathNamespacePackage(NamespacePackageTest): @@ -124,6 +129,40 @@ def test_imports(self): self.assertEqual(foo.two.attr, 'portion2 foo two') +class SeparatedNamespacePackagesCreatedWhileRunning(NamespacePackageTest): + paths = ['portion1'] + + def test_invalidate_caches(self): + with tempfile.TemporaryDirectory() as temp_dir: + # we manipulate sys.path before anything is imported to avoid + # accidental cache invalidation when changing it + sys.path.append(temp_dir) + + import foo.one + self.assertEqual(foo.one.attr, 'portion1 foo one') + + # the module does not exist, so it cannot be imported + with self.assertRaises(ImportError): + import foo.just_created + + # util.create_modules() manipulates sys.path + # so we must create the modules manually instead + namespace_path = os.path.join(temp_dir, 'foo') + os.mkdir(namespace_path) + module_path = os.path.join(namespace_path, 'just_created.py') + with open(module_path, 'w', encoding='utf-8') as file: + file.write('attr = "just_created foo"') + + # the module is not known, so it cannot be imported yet + with self.assertRaises(ImportError): + import foo.just_created + + # but after explicit cache invalidation, it is importable + importlib.invalidate_caches() + import foo.just_created + self.assertEqual(foo.just_created.attr, 'just_created foo') + + class SeparatedOverlappingNamespacePackages(NamespacePackageTest): paths = ['portion1', 'both_portions'] diff --git a/Lib/test/test_importlib/test_open.py b/Lib/test/test_importlib/test_open.py index fd6e84b70f..1d0ba5a653 100644 --- a/Lib/test/test_importlib/test_open.py +++ b/Lib/test/test_importlib/test_open.py @@ -29,34 +29,32 @@ def test_open_text_default_encoding(self): self.assertEqual(result, 'Hello, UTF-8 world!\n') def test_open_text_given_encoding(self): - with resources.open_text( - self.data, 'utf-16.file', 'utf-16', 'strict') as fp: + with resources.open_text(self.data, 'utf-16.file', 'utf-16', 'strict') as fp: result = fp.read() self.assertEqual(result, 'Hello, UTF-16 world!\n') def test_open_text_with_errors(self): # Raises UnicodeError without the 'errors' argument. - with resources.open_text( - self.data, 'utf-16.file', 'utf-8', 'strict') as fp: + with resources.open_text(self.data, 'utf-16.file', 'utf-8', 'strict') as fp: self.assertRaises(UnicodeError, fp.read) - with resources.open_text( - self.data, 'utf-16.file', 'utf-8', 'ignore') as fp: + with resources.open_text(self.data, 'utf-16.file', 'utf-8', 'ignore') as fp: result = fp.read() self.assertEqual( result, 'H\x00e\x00l\x00l\x00o\x00,\x00 ' '\x00U\x00T\x00F\x00-\x001\x006\x00 ' - '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00') + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00', + ) def test_open_binary_FileNotFoundError(self): self.assertRaises( - FileNotFoundError, - resources.open_binary, self.data, 'does-not-exist') + FileNotFoundError, resources.open_binary, self.data, 'does-not-exist' + ) def test_open_text_FileNotFoundError(self): self.assertRaises( - FileNotFoundError, - resources.open_text, self.data, 'does-not-exist') + FileNotFoundError, resources.open_text, self.data, 'does-not-exist' + ) class OpenDiskTests(OpenTests, unittest.TestCase): @@ -64,6 +62,19 @@ def setUp(self): self.data = data01 +class OpenDiskNamespaceTests(OpenTests, unittest.TestCase): + def setUp(self): + from . import namespacedata01 + + self.data = namespacedata01 + + # TODO: RUSTPYTHON + import sys + if sys.platform == 'win32': + @unittest.expectedFailure + def test_open_text_default_encoding(self): + super().test_open_text_default_encoding() + class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): pass diff --git a/Lib/test/test_importlib/test_path.py b/Lib/test/test_importlib/test_path.py index 2d3dcda7ed..d6ed09a9e0 100644 --- a/Lib/test/test_importlib/test_path.py +++ b/Lib/test/test_importlib/test_path.py @@ -1,3 +1,4 @@ +import io import unittest from importlib import resources @@ -17,6 +18,7 @@ def test_reading(self): # Test also implicitly verifies the returned object is a pathlib.Path # instance. with resources.path(self.data, 'utf-8.file') as path: + self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) # pathlib.Path.read_text() was introduced in Python 3.5. with path.open('r', encoding='utf-8') as file: text = file.read() @@ -26,6 +28,24 @@ def test_reading(self): class PathDiskTests(PathTests, unittest.TestCase): data = data01 + def test_natural_path(self): + # Guarantee the internal implementation detail that + # file-system-backed resources do not get the tempdir + # treatment. + with resources.path(self.data, 'utf-8.file') as path: + assert 'data' in str(path) + + +class PathMemoryTests(PathTests, unittest.TestCase): + def setUp(self): + file = io.BytesIO(b'Hello, UTF-8 world!\n') + self.addCleanup(file.close) + self.data = util.create_package( + file=file, path=FileNotFoundError("package exists only in memory") + ) + self.data.__spec__.origin = None + self.data.__spec__.has_location = False + class PathZipTests(PathTests, util.ZipSetup, unittest.TestCase): def test_remove_in_context_manager(self): diff --git a/Lib/test/test_importlib/test_pkg_import.py b/Lib/test/test_importlib/test_pkg_import.py new file mode 100644 index 0000000000..66f5f8bc25 --- /dev/null +++ b/Lib/test/test_importlib/test_pkg_import.py @@ -0,0 +1,80 @@ +import os +import sys +import shutil +import string +import random +import tempfile +import unittest + +from importlib.util import cache_from_source +from test.support.os_helper import create_empty_file + +class TestImport(unittest.TestCase): + + def __init__(self, *args, **kw): + self.package_name = 'PACKAGE_' + while self.package_name in sys.modules: + self.package_name += random.choice(string.ascii_letters) + self.module_name = self.package_name + '.foo' + unittest.TestCase.__init__(self, *args, **kw) + + def remove_modules(self): + for module_name in (self.package_name, self.module_name): + if module_name in sys.modules: + del sys.modules[module_name] + + def setUp(self): + self.test_dir = tempfile.mkdtemp() + sys.path.append(self.test_dir) + self.package_dir = os.path.join(self.test_dir, + self.package_name) + os.mkdir(self.package_dir) + create_empty_file(os.path.join(self.package_dir, '__init__.py')) + self.module_path = os.path.join(self.package_dir, 'foo.py') + + def tearDown(self): + shutil.rmtree(self.test_dir) + self.assertNotEqual(sys.path.count(self.test_dir), 0) + sys.path.remove(self.test_dir) + self.remove_modules() + + def rewrite_file(self, contents): + compiled_path = cache_from_source(self.module_path) + if os.path.exists(compiled_path): + os.remove(compiled_path) + with open(self.module_path, 'w', encoding='utf-8') as f: + f.write(contents) + + def test_package_import__semantics(self): + + # Generate a couple of broken modules to try importing. + + # ...try loading the module when there's a SyntaxError + self.rewrite_file('for') + try: __import__(self.module_name) + except SyntaxError: pass + else: raise RuntimeError('Failed to induce SyntaxError') # self.fail()? + self.assertNotIn(self.module_name, sys.modules) + self.assertFalse(hasattr(sys.modules[self.package_name], 'foo')) + + # ...make up a variable name that isn't bound in __builtins__ + var = 'a' + while var in dir(__builtins__): + var += random.choice(string.ascii_letters) + + # ...make a module that just contains that + self.rewrite_file(var) + + try: __import__(self.module_name) + except NameError: pass + else: raise RuntimeError('Failed to induce NameError.') + + # ...now change the module so that the NameError doesn't + # happen + self.rewrite_file('%s = 1' % var) + module = __import__(self.module_name).foo + self.assertEqual(getattr(module, var), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_importlib/test_read.py b/Lib/test/test_importlib/test_read.py index ff78d0b294..f6ec13af62 100644 --- a/Lib/test/test_importlib/test_read.py +++ b/Lib/test/test_importlib/test_read.py @@ -25,20 +25,19 @@ def test_read_text_default_encoding(self): self.assertEqual(result, 'Hello, UTF-8 world!\n') def test_read_text_given_encoding(self): - result = resources.read_text( - self.data, 'utf-16.file', encoding='utf-16') + result = resources.read_text(self.data, 'utf-16.file', encoding='utf-16') self.assertEqual(result, 'Hello, UTF-16 world!\n') def test_read_text_with_errors(self): # Raises UnicodeError without the 'errors' argument. - self.assertRaises( - UnicodeError, resources.read_text, self.data, 'utf-16.file') + self.assertRaises(UnicodeError, resources.read_text, self.data, 'utf-16.file') result = resources.read_text(self.data, 'utf-16.file', errors='ignore') self.assertEqual( result, 'H\x00e\x00l\x00l\x00o\x00,\x00 ' '\x00U\x00T\x00F\x00-\x001\x006\x00 ' - '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00') + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00', + ) class ReadDiskTests(ReadTests, unittest.TestCase): @@ -48,13 +47,11 @@ class ReadDiskTests(ReadTests, unittest.TestCase): class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): submodule = import_module('ziptestdata.subdirectory') - result = resources.read_binary( - submodule, 'binary.file') + result = resources.read_binary(submodule, 'binary.file') self.assertEqual(result, b'\0\1\2\3') def test_read_submodule_resource_by_name(self): - result = resources.read_binary( - 'ziptestdata.subdirectory', 'binary.file') + result = resources.read_binary('ziptestdata.subdirectory', 'binary.file') self.assertEqual(result, b'\0\1\2\3') diff --git a/Lib/test/test_importlib/test_reader.py b/Lib/test/test_importlib/test_reader.py new file mode 100644 index 0000000000..9d20c976b8 --- /dev/null +++ b/Lib/test/test_importlib/test_reader.py @@ -0,0 +1,128 @@ +import os.path +import sys +import pathlib +import unittest + +from importlib import import_module +from importlib.readers import MultiplexedPath, NamespaceReader + + +class MultiplexedPathTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + path = pathlib.Path(__file__).parent / 'namespacedata01' + cls.folder = str(path) + + def test_init_no_paths(self): + with self.assertRaises(FileNotFoundError): + MultiplexedPath() + + def test_init_file(self): + with self.assertRaises(NotADirectoryError): + MultiplexedPath(os.path.join(self.folder, 'binary.file')) + + def test_iterdir(self): + contents = {path.name for path in MultiplexedPath(self.folder).iterdir()} + try: + contents.remove('__pycache__') + except (KeyError, ValueError): + pass + self.assertEqual(contents, {'binary.file', 'utf-16.file', 'utf-8.file'}) + + def test_iterdir_duplicate(self): + data01 = os.path.abspath(os.path.join(__file__, '..', 'data01')) + contents = { + path.name for path in MultiplexedPath(self.folder, data01).iterdir() + } + for remove in ('__pycache__', '__init__.pyc'): + try: + contents.remove(remove) + except (KeyError, ValueError): + pass + self.assertEqual( + contents, + {'__init__.py', 'binary.file', 'subdirectory', 'utf-16.file', 'utf-8.file'}, + ) + + def test_is_dir(self): + self.assertEqual(MultiplexedPath(self.folder).is_dir(), True) + + def test_is_file(self): + self.assertEqual(MultiplexedPath(self.folder).is_file(), False) + + def test_open_file(self): + path = MultiplexedPath(self.folder) + with self.assertRaises(FileNotFoundError): + path.read_bytes() + with self.assertRaises(FileNotFoundError): + path.read_text() + with self.assertRaises(FileNotFoundError): + path.open() + + def test_join_path(self): + prefix = os.path.abspath(os.path.join(__file__, '..')) + data01 = os.path.join(prefix, 'data01') + path = MultiplexedPath(self.folder, data01) + self.assertEqual( + str(path.joinpath('binary.file'))[len(prefix) + 1 :], + os.path.join('namespacedata01', 'binary.file'), + ) + self.assertEqual( + str(path.joinpath('subdirectory'))[len(prefix) + 1 :], + os.path.join('data01', 'subdirectory'), + ) + self.assertEqual( + str(path.joinpath('imaginary'))[len(prefix) + 1 :], + os.path.join('namespacedata01', 'imaginary'), + ) + + def test_repr(self): + self.assertEqual( + repr(MultiplexedPath(self.folder)), + f"MultiplexedPath('{self.folder}')", + ) + + def test_name(self): + self.assertEqual( + MultiplexedPath(self.folder).name, + os.path.basename(self.folder), + ) + + +class NamespaceReaderTest(unittest.TestCase): + site_dir = str(pathlib.Path(__file__).parent) + + @classmethod + def setUpClass(cls): + sys.path.append(cls.site_dir) + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.site_dir) + + def test_init_error(self): + with self.assertRaises(ValueError): + NamespaceReader(['path1', 'path2']) + + def test_resource_path(self): + namespacedata01 = import_module('namespacedata01') + reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) + + root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + self.assertEqual( + reader.resource_path('binary.file'), os.path.join(root, 'binary.file') + ) + self.assertEqual( + reader.resource_path('imaginary'), os.path.join(root, 'imaginary') + ) + + def test_files(self): + namespacedata01 = import_module('namespacedata01') + reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) + root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + self.assertIsInstance(reader.files(), MultiplexedPath) + self.assertEqual(repr(reader.files()), f"MultiplexedPath('{root}')") + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py index f88d92d154..003f7e95ad 100644 --- a/Lib/test/test_importlib/test_resource.py +++ b/Lib/test/test_importlib/test_resource.py @@ -1,10 +1,14 @@ import sys import unittest +import uuid +import pathlib from . import data01 from . import zipdata01, zipdata02 from . import util from importlib import resources, import_module +from test.support import import_helper +from test.support.os_helper import unlink class ResourceTests: @@ -23,20 +27,21 @@ def test_is_resource_subresource_directory(self): def test_contents(self): contents = set(resources.contents(self.data)) # There may be cruft in the directory listing of the data directory. - # Under Python 3 we could have a __pycache__ directory, and under - # Python 2 we could have .pyc files. These are both artifacts of the - # test suite importing these modules and writing these caches. They - # aren't germane to this test, so just filter them out. + # It could have a __pycache__ directory, + # an artifact of the + # test suite importing these modules, which + # are not germane to this test, so just filter them out. contents.discard('__pycache__') - contents.discard('__init__.pyc') - contents.discard('__init__.pyo') - self.assertEqual(contents, { - '__init__.py', - 'subdirectory', - 'utf-8.file', - 'binary.file', - 'utf-16.file', - }) + self.assertEqual( + contents, + { + '__init__.py', + 'subdirectory', + 'utf-8.file', + 'binary.file', + 'utf-16.file', + }, + ) class ResourceDiskTests(ResourceTests, unittest.TestCase): @@ -51,27 +56,26 @@ class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): class ResourceLoaderTests(unittest.TestCase): def test_resource_contents(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C']) - self.assertEqual( - set(resources.contents(package)), - {'A', 'B', 'C'}) + file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + ) + self.assertEqual(set(resources.contents(package)), {'A', 'B', 'C'}) def test_resource_is_resource(self): package = util.create_package( - file=data01, path=data01.__file__, - contents=['A', 'B', 'C', 'D/E', 'D/F']) + file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + ) self.assertTrue(resources.is_resource(package, 'B')) def test_resource_directory_is_not_resource(self): package = util.create_package( - file=data01, path=data01.__file__, - contents=['A', 'B', 'C', 'D/E', 'D/F']) + file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + ) self.assertFalse(resources.is_resource(package, 'D')) def test_resource_missing_is_not_resource(self): package = util.create_package( - file=data01, path=data01.__file__, - contents=['A', 'B', 'C', 'D/E', 'D/F']) + file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + ) self.assertFalse(resources.is_resource(package, 'Z')) @@ -82,84 +86,168 @@ def test_package_has_no_reader_fallback(self): # 2. Are not on the file system # 3. Are not in a zip file module = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C']) + file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + ) # Give the module a dummy loader. module.__loader__ = object() # Give the module a dummy origin. module.__file__ = '/path/which/shall/not/be/named' - if sys.version_info >= (3,): - module.__spec__.loader = module.__loader__ - module.__spec__.origin = module.__file__ + module.__spec__.loader = module.__loader__ + module.__spec__.origin = module.__file__ self.assertFalse(resources.is_resource(module, 'A')) -class ResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore +class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): + ZIP_MODULE = zipdata01 # type: ignore + + def test_is_submodule_resource(self): + submodule = import_module('ziptestdata.subdirectory') + self.assertTrue(resources.is_resource(submodule, 'binary.file')) + + def test_read_submodule_resource_by_name(self): + self.assertTrue( + resources.is_resource('ziptestdata.subdirectory', 'binary.file') + ) + + def test_submodule_contents(self): + submodule = import_module('ziptestdata.subdirectory') + self.assertEqual( + set(resources.contents(submodule)), {'__init__.py', 'binary.file'} + ) + + def test_submodule_contents_by_name(self): + self.assertEqual( + set(resources.contents('ziptestdata.subdirectory')), + {'__init__.py', 'binary.file'}, + ) + + +class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): + ZIP_MODULE = zipdata02 # type: ignore def test_unrelated_contents(self): - # https://gitlab.com/python-devs/importlib_resources/issues/44 - # - # Here we have a zip file with two unrelated subpackages. The bug - # reports that getting the contents of a resource returns unrelated - # files. + """ + Test thata zip with two unrelated subpackages return + distinct resources. Ref python/importlib_resources#44. + """ self.assertEqual( - set(resources.contents('ziptestdata.one')), - {'__init__.py', 'resource1.txt'}) + set(resources.contents('ziptestdata.one')), {'__init__.py', 'resource1.txt'} + ) self.assertEqual( - set(resources.contents('ziptestdata.two')), - {'__init__.py', 'resource2.txt'}) + set(resources.contents('ziptestdata.two')), {'__init__.py', 'resource2.txt'} + ) -class SubdirectoryResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata01 # type: ignore +class DeletingZipsTest(unittest.TestCase): + """Having accessed resources in a zip file should not keep an open + reference to the zip. + """ + + ZIP_MODULE = zipdata01 + + def setUp(self): + modules = import_helper.modules_setup() + self.addCleanup(import_helper.modules_cleanup, *modules) + + data_path = pathlib.Path(self.ZIP_MODULE.__file__) + data_dir = data_path.parent + self.source_zip_path = data_dir / 'ziptestdata.zip' + self.zip_path = pathlib.Path(f'{uuid.uuid4()}.zip').absolute() + self.zip_path.write_bytes(self.source_zip_path.read_bytes()) + sys.path.append(str(self.zip_path)) + self.data = import_module('ziptestdata') + + def tearDown(self): + try: + sys.path.remove(str(self.zip_path)) + except ValueError: + pass + + try: + del sys.path_importer_cache[str(self.zip_path)] + del sys.modules[self.data.__name__] + except KeyError: + pass + + try: + unlink(self.zip_path) + except OSError: + # If the test fails, this will probably fail too + pass + + def test_contents_does_not_keep_open(self): + c = resources.contents('ziptestdata') + self.zip_path.unlink() + del c + + def test_is_resource_does_not_keep_open(self): + c = resources.is_resource('ziptestdata', 'binary.file') + self.zip_path.unlink() + del c + + def test_is_resource_failure_does_not_keep_open(self): + c = resources.is_resource('ziptestdata', 'not-present') + self.zip_path.unlink() + del c + + @unittest.skip("Desired but not supported.") + def test_path_does_not_keep_open(self): + c = resources.path('ziptestdata', 'binary.file') + self.zip_path.unlink() + del c + + def test_entered_path_does_not_keep_open(self): + # This is what certifi does on import to make its bundle + # available for the process duration. + c = resources.path('ziptestdata', 'binary.file').__enter__() + self.zip_path.unlink() + del c + + def test_read_binary_does_not_keep_open(self): + c = resources.read_binary('ziptestdata', 'binary.file') + self.zip_path.unlink() + del c + + def test_read_text_does_not_keep_open(self): + c = resources.read_text('ziptestdata', 'utf-8.file', encoding='utf-8') + self.zip_path.unlink() + del c + + +class ResourceFromNamespaceTest01(unittest.TestCase): + site_dir = str(pathlib.Path(__file__).parent) + + @classmethod + def setUpClass(cls): + sys.path.append(cls.site_dir) + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.site_dir) def test_is_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') self.assertTrue( - resources.is_resource(submodule, 'binary.file')) + resources.is_resource(import_module('namespacedata01'), 'binary.file') + ) def test_read_submodule_resource_by_name(self): - self.assertTrue( - resources.is_resource('ziptestdata.subdirectory', 'binary.file')) + self.assertTrue(resources.is_resource('namespacedata01', 'binary.file')) def test_submodule_contents(self): - submodule = import_module('ziptestdata.subdirectory') - self.assertEqual( - set(resources.contents(submodule)), - {'__init__.py', 'binary.file'}) + contents = set(resources.contents(import_module('namespacedata01'))) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) def test_submodule_contents_by_name(self): - self.assertEqual( - set(resources.contents('ziptestdata.subdirectory')), - {'__init__.py', 'binary.file'}) - - -class NamespaceTest(unittest.TestCase): - def test_namespaces_cannot_have_resources(self): - contents = resources.contents('test.test_importlib.data03.namespace') - self.assertFalse(list(contents)) - # Even though there is a file in the namespace directory, it is not - # considered a resource, since namespace packages can't have them. - self.assertFalse(resources.is_resource( - 'test.test_importlib.data03.namespace', - 'resource1.txt')) - # We should get an exception if we try to read it or open it. - self.assertRaises( - FileNotFoundError, - resources.open_text, - 'test.test_importlib.data03.namespace', 'resource1.txt') - self.assertRaises( - FileNotFoundError, - resources.open_binary, - 'test.test_importlib.data03.namespace', 'resource1.txt') - self.assertRaises( - FileNotFoundError, - resources.read_text, - 'test.test_importlib.data03.namespace', 'resource1.txt') - self.assertRaises( - FileNotFoundError, - resources.read_binary, - 'test.test_importlib.data03.namespace', 'resource1.txt') + contents = set(resources.contents('namespacedata01')) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/test_spec.py b/Lib/test/test_importlib/test_spec.py index da478e1bda..a61e63fc2d 100644 --- a/Lib/test/test_importlib/test_spec.py +++ b/Lib/test/test_importlib/test_spec.py @@ -303,32 +303,38 @@ def exec_module(self, module): self.assertNotIn(self.spec.name, sys.modules) def test_load_legacy(self): - self.spec.loader = LegacyLoader() - with CleanImport(self.spec.name): - loaded = self.bootstrap._load(self.spec) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + self.spec.loader = LegacyLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) - self.assertEqual(loaded.ham, -1) + self.assertEqual(loaded.ham, -1) def test_load_legacy_attributes(self): - self.spec.loader = LegacyLoader() - with CleanImport(self.spec.name): - loaded = self.bootstrap._load(self.spec) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + self.spec.loader = LegacyLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) - self.assertIs(loaded.__loader__, self.spec.loader) - self.assertEqual(loaded.__package__, self.spec.parent) - self.assertIs(loaded.__spec__, self.spec) + self.assertIs(loaded.__loader__, self.spec.loader) + self.assertEqual(loaded.__package__, self.spec.parent) + self.assertIs(loaded.__spec__, self.spec) def test_load_legacy_attributes_immutable(self): module = object() - class ImmutableLoader(TestLoader): - def load_module(self, name): - sys.modules[name] = module - return module - self.spec.loader = ImmutableLoader() - with CleanImport(self.spec.name): - loaded = self.bootstrap._load(self.spec) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + class ImmutableLoader(TestLoader): + def load_module(self, name): + sys.modules[name] = module + return module + self.spec.loader = ImmutableLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) - self.assertIs(sys.modules[self.spec.name], module) + self.assertIs(sys.modules[self.spec.name], module) # reload() @@ -382,11 +388,13 @@ def test_reload_init_module_attrs(self): self.assertFalse(hasattr(loaded, '__cached__')) def test_reload_legacy(self): - self.spec.loader = LegacyLoader() - with CleanImport(self.spec.name): - loaded = self.bootstrap._load(self.spec) - reloaded = self.bootstrap._exec(self.spec, loaded) - installed = sys.modules[self.spec.name] + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + self.spec.loader = LegacyLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) + reloaded = self.bootstrap._exec(self.spec, loaded) + installed = sys.modules[self.spec.name] self.assertEqual(loaded.ham, -1) self.assertIs(reloaded, loaded) @@ -498,7 +506,7 @@ class FactoryTests: def setUp(self): self.name = 'spam' - self.path = 'spam.py' + self.path = os.path.abspath('spam.py') self.cached = self.util.cache_from_source(self.path) self.loader = TestLoader() self.fileloader = TestLoader(self.path) @@ -637,7 +645,7 @@ def test_spec_from_loader_is_package_true_with_fileloader(self): self.assertEqual(spec.loader, self.fileloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, ['']) + self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -650,8 +658,8 @@ def test_spec_from_file_location_default(self): # Need to use a circuitous route to get at importlib.machinery to make # sure the same class object is used in the isinstance() check as # would have been used to create the loader. - self.assertIsInstance(spec.loader, - self.util.abc.machinery.SourceFileLoader) + SourceFileLoader = self.util.spec_from_file_location.__globals__['SourceFileLoader'] + self.assertIsInstance(spec.loader, SourceFileLoader) self.assertEqual(spec.loader.name, self.name) self.assertEqual(spec.loader.path, self.path) self.assertEqual(spec.origin, self.path) @@ -736,7 +744,7 @@ def test_spec_from_file_location_smsl_empty(self): self.assertEqual(spec.loader, self.fileloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, ['']) + self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -761,7 +769,7 @@ def test_spec_from_file_location_smsl_default(self): self.assertEqual(spec.loader, self.pkgloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, ['']) + self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -809,10 +817,22 @@ def is_package(self, name): self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) + def test_spec_from_file_location_relative_path(self): + spec = self.util.spec_from_file_location(self.name, + os.path.basename(self.path), loader=self.fileloader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.fileloader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) -(Frozen_FactoryTests, - Source_FactoryTests - ) = test_util.test_both(FactoryTests, util=util, machinery=machinery) +# TODO: RUSTPYTHON +# (Frozen_FactoryTests, +# Source_FactoryTests +# ) = test_util.test_both(FactoryTests, util=util, machinery=machinery) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py new file mode 100644 index 0000000000..766d84fda8 --- /dev/null +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -0,0 +1,284 @@ +# This is a variant of the very old (early 90's) file +# Demo/threads/bug.py. It simply provokes a number of threads into +# trying to import the same module "at the same time". +# There are no pleasant failure modes -- most likely is that Python +# complains several times about module random having no attribute +# randrange, and then Python hangs. + +import _imp as imp +import os +import importlib +import sys +import time +import shutil +import threading +import unittest +from unittest import mock +from test.support import verbose +from test.support.import_helper import forget +from test.support.os_helper import (TESTFN, unlink, rmtree) +from test.support import script_helper, threading_helper + +def task(N, done, done_tasks, errors): + try: + # We don't use modulefinder but still import it in order to stress + # importing of different modules from several threads. + if len(done_tasks) % 2: + import modulefinder + import random + else: + import random + import modulefinder + # This will fail if random is not completely initialized + x = random.randrange(1, 3) + except Exception as e: + errors.append(e.with_traceback(None)) + finally: + done_tasks.append(threading.get_ident()) + finished = len(done_tasks) == N + if finished: + done.set() + +def mock_register_at_fork(func): + # bpo-30599: Mock os.register_at_fork() when importing the random module, + # since this function doesn't allow to unregister callbacks and would leak + # memory. + return mock.patch('os.register_at_fork', create=True)(func) + +# Create a circular import structure: A -> C -> B -> D -> A +# NOTE: `time` is already loaded and therefore doesn't threaten to deadlock. + +circular_imports_modules = { + 'A': """if 1: + import time + time.sleep(%(delay)s) + x = 'a' + import C + """, + 'B': """if 1: + import time + time.sleep(%(delay)s) + x = 'b' + import D + """, + 'C': """import B""", + 'D': """import A""", +} + +class Finder: + """A dummy finder to detect concurrent access to its find_spec() + method.""" + + def __init__(self): + self.numcalls = 0 + self.x = 0 + self.lock = threading.Lock() + + def find_spec(self, name, path=None, target=None): + # Simulate some thread-unsafe behaviour. If calls to find_spec() + # are properly serialized, `x` will end up the same as `numcalls`. + # Otherwise not. + assert imp.lock_held() + with self.lock: + self.numcalls += 1 + x = self.x + time.sleep(0.01) + self.x = x + 1 + +class FlushingFinder: + """A dummy finder which flushes sys.path_importer_cache when it gets + called.""" + + def find_spec(self, name, path=None, target=None): + sys.path_importer_cache.clear() + + +class ThreadedImportTests(unittest.TestCase): + + def setUp(self): + self.old_random = sys.modules.pop('random', None) + + def tearDown(self): + # If the `random` module was already initialized, we restore the + # old module at the end so that pickling tests don't fail. + # See http://bugs.python.org/issue3657#msg110461 + if self.old_random is not None: + sys.modules['random'] = self.old_random + + @mock_register_at_fork + def check_parallel_module_init(self, mock_os): + if imp.lock_held(): + # This triggers on, e.g., from test import autotest. + raise unittest.SkipTest("can't run when import lock is held") + + done = threading.Event() + for N in (20, 50) * 3: + if verbose: + print("Trying", N, "threads ...", end=' ') + # Make sure that random and modulefinder get reimported freshly + for modname in ['random', 'modulefinder']: + try: + del sys.modules[modname] + except KeyError: + pass + errors = [] + done_tasks = [] + done.clear() + t0 = time.monotonic() + with threading_helper.start_threads( + threading.Thread(target=task, args=(N, done, done_tasks, errors,)) + for i in range(N)): + pass + completed = done.wait(10 * 60) + dt = time.monotonic() - t0 + if verbose: + print("%.1f ms" % (dt*1e3), flush=True, end=" ") + dbg_info = 'done: %s/%s' % (len(done_tasks), N) + self.assertFalse(errors, dbg_info) + self.assertTrue(completed, dbg_info) + if verbose: + print("OK.") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_parallel_module_init(self): + self.check_parallel_module_init() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_parallel_meta_path(self): + finder = Finder() + sys.meta_path.insert(0, finder) + try: + self.check_parallel_module_init() + self.assertGreater(finder.numcalls, 0) + self.assertEqual(finder.x, finder.numcalls) + finally: + sys.meta_path.remove(finder) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_parallel_path_hooks(self): + # Here the Finder instance is only used to check concurrent calls + # to path_hook(). + finder = Finder() + # In order for our path hook to be called at each import, we need + # to flush the path_importer_cache, which we do by registering a + # dedicated meta_path entry. + flushing_finder = FlushingFinder() + def path_hook(path): + finder.find_spec('') + raise ImportError + sys.path_hooks.insert(0, path_hook) + sys.meta_path.append(flushing_finder) + try: + # Flush the cache a first time + flushing_finder.find_spec('') + numtests = self.check_parallel_module_init() + self.assertGreater(finder.numcalls, 0) + self.assertEqual(finder.x, finder.numcalls) + finally: + sys.meta_path.remove(flushing_finder) + sys.path_hooks.remove(path_hook) + + def test_import_hangers(self): + # In case this test is run again, make sure the helper module + # gets loaded from scratch again. + try: + del sys.modules['test.test_importlib.threaded_import_hangers'] + except KeyError: + pass + import test.test_importlib.threaded_import_hangers + self.assertFalse(test.test_importlib.threaded_import_hangers.errors) + + def test_circular_imports(self): + # The goal of this test is to exercise implementations of the import + # lock which use a per-module lock, rather than a global lock. + # In these implementations, there is a possible deadlock with + # circular imports, for example: + # - thread 1 imports A (grabbing the lock for A) which imports B + # - thread 2 imports B (grabbing the lock for B) which imports A + # Such implementations should be able to detect such situations and + # resolve them one way or the other, without freezing. + # NOTE: our test constructs a slightly less trivial import cycle, + # in order to better stress the deadlock avoidance mechanism. + delay = 0.5 + os.mkdir(TESTFN) + self.addCleanup(shutil.rmtree, TESTFN) + sys.path.insert(0, TESTFN) + self.addCleanup(sys.path.remove, TESTFN) + for name, contents in circular_imports_modules.items(): + contents = contents % {'delay': delay} + with open(os.path.join(TESTFN, name + ".py"), "wb") as f: + f.write(contents.encode('utf-8')) + self.addCleanup(forget, name) + + importlib.invalidate_caches() + results = [] + def import_ab(): + import A + results.append(getattr(A, 'x', None)) + def import_ba(): + import B + results.append(getattr(B, 'x', None)) + t1 = threading.Thread(target=import_ab) + t2 = threading.Thread(target=import_ba) + t1.start() + t2.start() + t1.join() + t2.join() + self.assertEqual(set(results), {'a', 'b'}) + + @mock_register_at_fork + def test_side_effect_import(self, mock_os): + code = """if 1: + import threading + def target(): + import random + t = threading.Thread(target=target) + t.start() + t.join() + t = None""" + sys.path.insert(0, os.curdir) + self.addCleanup(sys.path.remove, os.curdir) + filename = TESTFN + ".py" + with open(filename, "wb") as f: + f.write(code.encode('utf-8')) + self.addCleanup(unlink, filename) + self.addCleanup(forget, TESTFN) + self.addCleanup(rmtree, '__pycache__') + importlib.invalidate_caches() + __import__(TESTFN) + del sys.modules[TESTFN] + + @unittest.skip("TODO: RUSTPYTHON; hang") + def test_concurrent_futures_circular_import(self): + # Regression test for bpo-43515 + fn = os.path.join(os.path.dirname(__file__), + 'partial', 'cfimport.py') + script_helper.assert_python_ok(fn) + + def test_multiprocessing_pool_circular_import(self): + # Regression test for bpo-41567 + fn = os.path.join(os.path.dirname(__file__), + 'partial', 'pool_in_threads.py') + script_helper.assert_python_ok(fn) + + # TODO: RUSTPYTHON + if sys.platform == 'win32': + test_multiprocessing_pool_circular_import = unittest.expectedFailure(test_multiprocessing_pool_circular_import) + + +def setUpModule(): + thread_info = threading_helper.threading_setup() + unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) + try: + old_switchinterval = sys.getswitchinterval() + unittest.addModuleCleanup(sys.setswitchinterval, old_switchinterval) + sys.setswitchinterval(1e-5) + except AttributeError: + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 19122c2ee1..9e2c294373 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -4,7 +4,6 @@ machinery = util.import_importlib('importlib.machinery') importlib_util = util.import_importlib('importlib.util') -import contextlib import importlib.util import os import pathlib @@ -382,7 +381,7 @@ def test_absolute_within_package(self): def test_no_package(self): # .bacon in '' - with self.assertRaises(ValueError): + with self.assertRaises(ImportError): self.util.resolve_name('.bacon', '') def test_in_package(self): @@ -397,7 +396,7 @@ def test_other_package(self): def test_escape(self): # ..bacon in spam - with self.assertRaises(ValueError): + with self.assertRaises(ImportError): self.util.resolve_name('..bacon', 'spam') @@ -525,7 +524,7 @@ def test_find_relative_module_missing_package(self): with util.temp_module(name, pkg=True) as pkg_dir: fullname, _ = util.submodule(name, subname, pkg_dir) relname = '.' + subname - with self.assertRaises(ValueError): + with self.assertRaises(ImportError): self.util.find_spec(relname) self.assertNotIn(name, sorted(sys.modules)) self.assertNotIn(fullname, sorted(sys.modules)) @@ -558,6 +557,7 @@ def test_incorporates_rn(self): Source_MagicNumberTests ) = util.test_both(MagicNumberTests, util=importlib_util) + # TODO: RUSTPYTHON @unittest.expectedFailure def test_incorporates_rn_MONKEYPATCH(self): @@ -566,6 +566,7 @@ def test_incorporates_rn_MONKEYPATCH(self): # TODO: RUSTPYTHON Frozen_MagicNumberTests.test_incorporates_rn = test_incorporates_rn_MONKEYPATCH + class PEP3147Tests: """Tests of PEP 3147-related functions: cache_from_source and source_from_cache.""" @@ -861,23 +862,21 @@ class MagicNumberTests(unittest.TestCase): 'only applies to candidate or final python release levels' ) def test_magic_number(self): - """ - Each python minor release should generally have a MAGIC_NUMBER - that does not change once the release reaches candidate status. - - Once a release reaches candidate status, the value of the constant - EXPECTED_MAGIC_NUMBER in this test should be changed. - This test will then check that the actual MAGIC_NUMBER matches - the expected value for the release. - - In exceptional cases, it may be required to change the MAGIC_NUMBER - for a maintenance release. In this case the change should be - discussed in python-dev. If a change is required, community - stakeholders such as OS package maintainers must be notified - in advance. Such exceptional releases will then require an - adjustment to this test case. - """ - EXPECTED_MAGIC_NUMBER = 3410 + # Each python minor release should generally have a MAGIC_NUMBER + # that does not change once the release reaches candidate status. + + # Once a release reaches candidate status, the value of the constant + # EXPECTED_MAGIC_NUMBER in this test should be changed. + # This test will then check that the actual MAGIC_NUMBER matches + # the expected value for the release. + + # In exceptional cases, it may be required to change the MAGIC_NUMBER + # for a maintenance release. In this case the change should be + # discussed in python-dev. If a change is required, community + # stakeholders such as OS package maintainers must be notified + # in advance. Such exceptional releases will then require an + # adjustment to this test case. + EXPECTED_MAGIC_NUMBER = 3439 actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') msg = ( diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index de53680cc3..a791c0b450 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -5,9 +5,9 @@ import re import sys import unittest +import warnings from test import support from test.support import import_helper -from distutils.util import get_platform from contextlib import contextmanager from .util import temp_module @@ -18,6 +18,25 @@ EnumKey, CloseKey, DeleteKey, OpenKey ) +def get_platform(): + # Port of distutils.util.get_platform(). + TARGET_TO_PLAT = { + 'x86' : 'win32', + 'x64' : 'win-amd64', + 'arm' : 'win-arm32', + } + if ('VSCMD_ARG_TGT_ARCH' in os.environ and + os.environ['VSCMD_ARG_TGT_ARCH'] in TARGET_TO_PLAT): + return TARGET_TO_PLAT[os.environ['VSCMD_ARG_TGT_ARCH']] + elif 'amd64' in sys.version.lower(): + return 'win-amd64' + elif '(arm)' in sys.version.lower(): + return 'win-arm32' + elif '(arm64)' in sys.version.lower(): + return 'win-arm64' + else: + return sys.platform + def delete_registry_tree(root, subkey): try: hkey = OpenKey(root, subkey, access=KEY_ALL_ACCESS) @@ -42,17 +61,28 @@ def setup_module(machinery, name, path=None): root = machinery.WindowsRegistryFinder.REGISTRY_KEY key = root.format(fullname=name, sys_version='%d.%d' % sys.version_info[:2]) + base_key = "Software\\Python\\PythonCore\\{}.{}".format( + sys.version_info.major, sys.version_info.minor) + assert key.casefold().startswith(base_key.casefold()), ( + "expected key '{}' to start with '{}'".format(key, base_key)) try: with temp_module(name, "a = 1") as location: + try: + OpenKey(HKEY_CURRENT_USER, base_key) + if machinery.WindowsRegistryFinder.DEBUG_BUILD: + delete_key = os.path.dirname(key) + else: + delete_key = key + except OSError: + delete_key = base_key subkey = CreateKey(HKEY_CURRENT_USER, key) if path is None: path = location + ".py" SetValue(subkey, "", REG_SZ, path) yield finally: - if machinery.WindowsRegistryFinder.DEBUG_BUILD: - key = os.path.dirname(key) - delete_registry_tree(HKEY_CURRENT_USER, key) + if delete_key: + delete_registry_tree(HKEY_CURRENT_USER, delete_key) @unittest.skipUnless(sys.platform.startswith('win'), 'requires Windows') @@ -66,19 +96,25 @@ def test_find_spec_missing(self): self.assertIs(spec, None) def test_find_module_missing(self): - loader = self.machinery.WindowsRegistryFinder.find_module('spam') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + loader = self.machinery.WindowsRegistryFinder.find_module('spam') self.assertIs(loader, None) def test_module_found(self): with setup_module(self.machinery, self.test_module): - loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module) spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) self.assertIsNot(loader, None) self.assertIsNot(spec, None) def test_module_not_found(self): with setup_module(self.machinery, self.test_module, path="."): - loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module) spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) self.assertIsNone(loader) self.assertIsNone(spec) @@ -103,10 +139,55 @@ def test_tagged_suffix(self): self.assertIn(expected_tag, suffixes) - # Ensure the tags are in the correct order + # Ensure the tags are in the correct order. tagged_i = suffixes.index(expected_tag) self.assertLess(tagged_i, untagged_i) (Frozen_WindowsExtensionSuffixTests, Source_WindowsExtensionSuffixTests ) = test_util.test_both(WindowsExtensionSuffixTests, machinery=machinery) + + +@unittest.skipUnless(sys.platform.startswith('win'), 'requires Windows') +class WindowsBootstrapPathTests(unittest.TestCase): + def check_join(self, expected, *inputs): + from importlib._bootstrap_external import _path_join + actual = _path_join(*inputs) + if expected.casefold() == actual.casefold(): + return + self.assertEqual(expected, actual) + + def test_path_join(self): + self.check_join(r"C:\A\B", "C:\\", "A", "B") + self.check_join(r"C:\A\B", "D:\\", "D", "C:\\", "A", "B") + self.check_join(r"C:\A\B", "C:\\", "A", "C:B") + self.check_join(r"C:\A\B", "C:\\", "A\\B") + self.check_join(r"C:\A\B", r"C:\A\B") + + self.check_join("D:A", r"D:", "A") + self.check_join("D:A", r"C:\B\C", "D:", "A") + self.check_join("D:A", r"C:\B\C", r"D:A") + + self.check_join(r"A\B\C", "A", "B", "C") + self.check_join(r"A\B\C", "A", r"B\C") + self.check_join(r"A\B/C", "A", "B/C") + self.check_join(r"A\B\C", "A/", "B\\", "C") + + # Dots are not normalised by this function + self.check_join(r"A\../C", "A", "../C") + self.check_join(r"A.\.\B", "A.", ".", "B") + + self.check_join(r"\\Server\Share\A\B\C", r"\\Server\Share", "A", "B", "C") + self.check_join(r"\\Server\Share\A\B\C", r"\\Server\Share", "D", r"\A", "B", "C") + self.check_join(r"\\Server\Share\A\B\C", r"\\Server2\Share2", "D", + r"\\Server\Share", "A", "B", "C") + self.check_join(r"\\Server\Share\A\B\C", r"\\Server", r"\Share", "A", "B", "C") + self.check_join(r"\\Server\Share", r"\\Server\Share") + self.check_join(r"\\Server\Share\\", r"\\Server\Share\\") + + # Handle edge cases with empty segments + self.check_join("C:\\A", "C:/A", "") + self.check_join("C:\\", "C:/", "") + self.check_join("C:", "C:", "") + self.check_join("//Server/Share\\", "//Server/Share/", "") + self.check_join("//Server/Share\\", "//Server/Share", "") diff --git a/Lib/test/test_importlib/test_zip.py b/Lib/test/test_importlib/test_zip.py new file mode 100644 index 0000000000..bf16a3b95e --- /dev/null +++ b/Lib/test/test_importlib/test_zip.py @@ -0,0 +1,82 @@ +import sys +import unittest + +from contextlib import ExitStack +from importlib.metadata import ( + PackageNotFoundError, + distribution, + distributions, + entry_points, + files, + version, +) +from importlib import resources + +from test.support import requires_zlib + + +@requires_zlib() +class TestZip(unittest.TestCase): + root = 'test.test_importlib.data' + + def _fixture_on_path(self, filename): + pkg_file = resources.files(self.root).joinpath(filename) + file = self.resources.enter_context(resources.as_file(pkg_file)) + assert file.name.startswith('example-'), file.name + sys.path.insert(0, str(file)) + self.resources.callback(sys.path.pop, 0) + + def setUp(self): + # Find the path to the example-*.whl so we can add it to the front of + # sys.path, where we'll then try to find the metadata thereof. + self.resources = ExitStack() + self.addCleanup(self.resources.close) + self._fixture_on_path('example-21.12-py3-none-any.whl') + + def test_zip_version(self): + self.assertEqual(version('example'), '21.12') + + def test_zip_version_does_not_match(self): + with self.assertRaises(PackageNotFoundError): + version('definitely-not-installed') + + def test_zip_entry_points(self): + scripts = entry_points(group='console_scripts') + entry_point = scripts['example'] + self.assertEqual(entry_point.value, 'example:main') + entry_point = scripts['Example'] + self.assertEqual(entry_point.value, 'example:main') + + def test_missing_metadata(self): + self.assertIsNone(distribution('example').read_text('does not exist')) + + def test_case_insensitive(self): + self.assertEqual(version('Example'), '21.12') + + def test_files(self): + for file in files('example'): + path = str(file.dist.locate_file(file)) + assert '.whl/' in path, path + + def test_one_distribution(self): + dists = list(distributions(path=sys.path[:1])) + assert len(dists) == 1 + + +@requires_zlib() +class TestEgg(TestZip): + def setUp(self): + # Find the path to the example-*.egg so we can add it to the front of + # sys.path, where we'll then try to find the metadata thereof. + self.resources = ExitStack() + self.addCleanup(self.resources.close) + self._fixture_on_path('example-21.12-py3.6.egg') + + def test_files(self): + for file in files('example'): + path = str(file.dist.locate_file(file)) + assert '.egg/' in path, path + + def test_normalized_name(self): + dist = distribution('example') + assert dist._normalized_name == 'example' diff --git a/Lib/test/test_importlib/threaded_import_hangers.py b/Lib/test/test_importlib/threaded_import_hangers.py new file mode 100644 index 0000000000..5484e60a0d --- /dev/null +++ b/Lib/test/test_importlib/threaded_import_hangers.py @@ -0,0 +1,45 @@ +# This is a helper module for test_threaded_import. The test imports this +# module, and this module tries to run various Python library functions in +# their own thread, as a side effect of being imported. If the spawned +# thread doesn't complete in TIMEOUT seconds, an "appeared to hang" message +# is appended to the module-global `errors` list. That list remains empty +# if (and only if) all functions tested complete. + +TIMEOUT = 10 + +import threading + +import tempfile +import os.path + +errors = [] + +# This class merely runs a function in its own thread T. The thread importing +# this module holds the import lock, so if the function called by T tries +# to do its own imports it will block waiting for this module's import +# to complete. +class Worker(threading.Thread): + def __init__(self, function, args): + threading.Thread.__init__(self) + self.function = function + self.args = args + + def run(self): + self.function(*self.args) + +for name, func, args in [ + # Bug 147376: TemporaryFile hung on Windows, starting in Python 2.4. + ("tempfile.TemporaryFile", lambda: tempfile.TemporaryFile().close(), ()), + + # The real cause for bug 147376: ntpath.abspath() caused the hang. + ("os.path.abspath", os.path.abspath, ('.',)), + ]: + + try: + t = Worker(func, args) + t.start() + t.join(TIMEOUT) + if t.is_alive(): + errors.append("%s appeared to hang" % name) + finally: + del t diff --git a/Lib/test/test_importlib/update-zips.py b/Lib/test/test_importlib/update-zips.py new file mode 100755 index 0000000000..9ef0224ca6 --- /dev/null +++ b/Lib/test/test_importlib/update-zips.py @@ -0,0 +1,53 @@ +""" +Generate the zip test data files. + +Run to build the tests/zipdataNN/ziptestdata.zip files from +files in tests/dataNN. + +Replaces the file with the working copy, but does commit anything +to the source repo. +""" + +import contextlib +import os +import pathlib +import zipfile + + +def main(): + """ + >>> from unittest import mock + >>> monkeypatch = getfixture('monkeypatch') + >>> monkeypatch.setattr(zipfile, 'ZipFile', mock.MagicMock()) + >>> print(); main() # print workaround for bpo-32509 + + ...data01... -> ziptestdata/... + ... + ...data02... -> ziptestdata/... + ... + """ + suffixes = '01', '02' + tuple(map(generate, suffixes)) + + +def generate(suffix): + root = pathlib.Path(__file__).parent.relative_to(os.getcwd()) + zfpath = root / f'zipdata{suffix}/ziptestdata.zip' + with zipfile.ZipFile(zfpath, 'w') as zf: + for src, rel in walk(root / f'data{suffix}'): + dst = 'ziptestdata' / pathlib.PurePosixPath(rel.as_posix()) + print(src, '->', dst) + zf.write(src, dst) + + +def walk(datapath): + for dirpath, dirnames, filenames in os.walk(datapath): + with contextlib.suppress(KeyError): + dirnames.remove('__pycache__') + for filename in filenames: + res = pathlib.Path(dirpath) / filename + rel = res.relative_to(datapath) + yield res, rel + + +__name__ == '__main__' and main() diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index f3ee0f2175..854506518c 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -7,11 +7,13 @@ from importlib import machinery, util, invalidate_caches from importlib.abc import ResourceReader import io +import marshal import os import os.path from pathlib import Path, PurePath from test import support -from test.support import os_helper, import_helper +from test.support import import_helper +from test.support import os_helper import unittest import sys import tempfile @@ -114,11 +116,21 @@ def case_insensitive_tests(test): def submodule(parent, name, pkg_dir, content=''): path = os.path.join(pkg_dir, name + '.py') - with open(path, 'w') as subfile: + with open(path, 'w', encoding='utf-8') as subfile: subfile.write(content) return '{}.{}'.format(parent, name), path +def get_code_from_pyc(pyc_path): + """Reads a pyc file and returns the unmarshalled code object within. + + No header validation is performed. + """ + with open(pyc_path, 'rb') as pyc_f: + pyc_f.seek(16) + return marshal.load(pyc_f) + + @contextlib.contextmanager def uncache(*names): """Uncache a module from sys.modules. @@ -164,7 +176,7 @@ def temp_module(name, content='', *, pkg=False): content = '' if content is not None: # not a namespace package - with open(modpath, 'w') as modfile: + with open(modpath, 'w', encoding='utf-8') as modfile: modfile.write(content) yield location @@ -295,7 +307,7 @@ def writes_bytecode_files(fxn): """Decorator to protect sys.dont_write_bytecode from mutation and to skip tests that require it to be set to False.""" if sys.dont_write_bytecode: - return lambda *args, **kwargs: None + return unittest.skip(fxn) @functools.wraps(fxn) def wrapper(*args, **kwargs): original = sys.dont_write_bytecode @@ -372,7 +384,7 @@ def create_modules(*names): os.mkdir(file_path) created_paths.append(file_path) file_path = os.path.join(file_path, name_parts[-1] + '.py') - with open(file_path, 'w') as file: + with open(file_path, 'w', encoding='utf-8') as file: file.write(source.format(name)) created_paths.append(file_path) mapping[name] = file_path @@ -444,7 +456,7 @@ def contents(self): yield entry name = 'testingpackage' - # Unforunately importlib.util.module_from_spec() was not introduced until + # Unfortunately importlib.util.module_from_spec() was not introduced until # Python 3.5. module = types.ModuleType(name) loader = Reader() @@ -489,7 +501,7 @@ def test_absolute_path(self): self.execute(data01, full_path) def test_relative_path(self): - # A reative path is a ValueError. + # A relative path is a ValueError. with self.assertRaises(ValueError): self.execute(data01, '../data01/utf-8.file') diff --git a/Lib/test/test_importlib/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/zipdata01/ziptestdata.zip index 8d8fa97f19..9a3bb0739f 100644 Binary files a/Lib/test/test_importlib/zipdata01/ziptestdata.zip and b/Lib/test/test_importlib/zipdata01/ziptestdata.zip differ diff --git a/Lib/test/test_importlib/zipdata02/ziptestdata.zip b/Lib/test/test_importlib/zipdata02/ziptestdata.zip index 6f348899a8..d63ff512d2 100644 Binary files a/Lib/test/test_importlib/zipdata02/ziptestdata.zip and b/Lib/test/test_importlib/zipdata02/ziptestdata.zip differ diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 9e3894ef2c..b291d53016 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -230,8 +230,6 @@ def testBadMagic(self): TESTMOD + pyc_ext: (NOW, badmagic_pyc)} self.doTest(".py", files, TESTMOD) - # TODO: RUSTPYTHON, zipimport.ZipImportError: can't find module 'ziptestmodule' - @unittest.expectedFailure def testBadMagic2(self): # make pyc magic word invalid, causing an ImportError badmagic_pyc = bytearray(test_pyc) @@ -438,8 +436,6 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) - # TODO: RUSTPYTHON, AttributeError: 'zipimporter' object has no attribute 'find_spec' - @unittest.expectedFailure def testZipImporterMethods(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep @@ -463,15 +459,15 @@ def testZipImporterMethods(self): # PEP 302 with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - find_mod = zi.find_module('spam') - self.assertIsNotNone(find_mod) - self.assertIsInstance(find_mod, zipimport.zipimporter) - self.assertFalse(find_mod.is_package('spam')) - load_mod = find_mod.load_module('spam') - self.assertEqual(find_mod.get_filename('spam'), load_mod.__file__) - - mod = zi.load_module(TESTPACK) - self.assertEqual(zi.get_filename(TESTPACK), mod.__file__) + find_mod = zi.find_module('spam') + self.assertIsNotNone(find_mod) + self.assertIsInstance(find_mod, zipimport.zipimporter) + self.assertFalse(find_mod.is_package('spam')) + load_mod = find_mod.load_module('spam') + self.assertEqual(find_mod.get_filename('spam'), load_mod.__file__) + + mod = zi.load_module(TESTPACK) + self.assertEqual(zi.get_filename(TESTPACK), mod.__file__) # PEP 451 spec = zi.find_spec('spam') @@ -513,8 +509,6 @@ def testZipImporterMethods(self): self.assertEqual(zi2.archive, TEMP_ZIP) self.assertEqual(zi2.prefix, TESTPACK + os.sep) - # TODO: RUSTPYTHON, AttributeError: 'zipimporter' object has no attribute 'invalidate_caches' - @unittest.expectedFailure def testInvalidateCaches(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep @@ -557,8 +551,6 @@ def testInvalidateCaches(self): self.assertIsNone(zipimport._zip_directory_cache.get(zi.archive)) self.assertIsNone(zi.find_spec("name_does_not_matter")) - # TODO: RUSTPYTHON, AttributeError: 'zipimporter' object has no attribute 'find_spec' - @unittest.expectedFailure def testZipImporterMethodsInSubDirectory(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep @@ -580,8 +572,8 @@ def testZipImporterMethodsInSubDirectory(self): # PEP 302 with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - mod = zi.load_module(TESTPACK2) - self.assertEqual(zi.get_filename(TESTPACK2), mod.__file__) + mod = zi.load_module(TESTPACK2) + self.assertEqual(zi.get_filename(TESTPACK2), mod.__file__) # PEP 451 spec = zi.find_spec(TESTPACK2) mod = importlib.util.module_from_spec(spec) @@ -596,13 +588,13 @@ def testZipImporterMethodsInSubDirectory(self): # PEP 302 with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - find_mod_dotted = zi2.find_module(TESTMOD) - self.assertIsNotNone(find_mod_dotted) - self.assertIsInstance(find_mod_dotted, zipimport.zipimporter) - self.assertFalse(zi2.is_package(TESTMOD)) - load_mod = find_mod_dotted.load_module(TESTMOD) - self.assertEqual( - find_mod_dotted.get_filename(TESTMOD), load_mod.__file__) + find_mod_dotted = zi2.find_module(TESTMOD) + self.assertIsNotNone(find_mod_dotted) + self.assertIsInstance(find_mod_dotted, zipimport.zipimporter) + self.assertFalse(zi2.is_package(TESTMOD)) + load_mod = find_mod_dotted.load_module(TESTMOD) + self.assertEqual( + find_mod_dotted.get_filename(TESTMOD), load_mod.__file__) # PEP 451 spec = zi2.find_spec(TESTMOD) @@ -736,8 +728,6 @@ def testTraceback(self): files = {TESTMOD + ".py": (NOW, raise_src)} self.doTest(None, files, TESTMOD, call=self.doTraceback) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, "need an unencodable filename") def testUnencodable(self): @@ -842,7 +832,7 @@ def _testBogusZipFile(self): try: with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - self.assertRaises(TypeError, z.load_module, None) + self.assertRaises(TypeError, z.load_module, None) self.assertRaises(TypeError, z.find_module, None) self.assertRaises(TypeError, z.find_spec, None) self.assertRaises(TypeError, z.exec_module, None) @@ -857,7 +847,7 @@ def _testBogusZipFile(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - self.assertRaises(error, z.load_module, 'abc') + self.assertRaises(error, z.load_module, 'abc') self.assertRaises(error, z.get_code, 'abc') self.assertRaises(OSError, z.get_data, 'abc') self.assertRaises(error, z.get_source, 'abc') @@ -867,7 +857,7 @@ def _testBogusZipFile(self): def tearDownModule(): - os_helper.unlink(TESTMOD) + os_helper.unlink(TESTMOD) if __name__ == "__main__": diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 5ef0a17c2a..25eaee9c0f 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -22,6 +22,7 @@ import marshal # for loads import sys # for modules import time # for mktime +import _warnings # For warn() __all__ = ['ZipImportError', 'zipimporter'] @@ -42,7 +43,7 @@ class ZipImportError(ImportError): STRING_END_ARCHIVE = b'PK\x05\x06' MAX_COMMENT_LEN = (1 << 16) - 1 -class zipimporter: +class zipimporter(_bootstrap_external._LoaderBasics): """zipimporter(archivepath) -> zipimporter object Create a new zipimporter instance. 'archivepath' must be a path to @@ -115,7 +116,12 @@ def find_loader(self, fullname, path=None): full path name if it's possibly a portion of a namespace package, or None otherwise. The optional 'path' argument is ignored -- it's there for compatibility with the importer protocol. + + Deprecated since Python 3.10. Use find_spec() instead. """ + _warnings.warn("zipimporter.find_loader() is deprecated and slated for " + "removal in Python 3.12; use find_spec() instead", + DeprecationWarning) mi = _get_module_info(self, fullname) if mi is not None: # This is a module or package. @@ -146,15 +152,46 @@ def find_module(self, fullname, path=None): instance itself if the module was found, or None if it wasn't. The optional 'path' argument is ignored -- it's there for compatibility with the importer protocol. + + Deprecated since Python 3.10. Use find_spec() instead. """ + _warnings.warn("zipimporter.find_module() is deprecated and slated for " + "removal in Python 3.12; use find_spec() instead", + DeprecationWarning) return self.find_loader(fullname, path)[0] + def find_spec(self, fullname, target=None): + """Create a ModuleSpec for the specified module. + + Returns None if the module cannot be found. + """ + module_info = _get_module_info(self, fullname) + if module_info is not None: + return _bootstrap.spec_from_loader(fullname, self, is_package=module_info) + else: + # Not a module or regular package. See if this is a directory, and + # therefore possibly a portion of a namespace package. + + # We're only interested in the last path component of fullname + # earlier components are recorded in self.prefix. + modpath = _get_module_path(self, fullname) + if _is_dir(self, modpath): + # This is possibly a portion of a namespace + # package. Return the string representing its path, + # without a trailing separator. + path = f'{self.archive}{path_sep}{modpath}' + spec = _bootstrap.ModuleSpec(name=fullname, loader=None, + is_package=True) + spec.submodule_search_locations.append(path) + return spec + else: + return None def get_code(self, fullname): """get_code(fullname) -> code object. Return the code object for the specified module. Raise ZipImportError - if the module couldn't be found. + if the module couldn't be imported. """ code, ispackage, modpath = _get_module_code(self, fullname) return code @@ -184,7 +221,8 @@ def get_data(self, pathname): def get_filename(self, fullname): """get_filename(fullname) -> filename string. - Return the filename for the specified module. + Return the filename for the specified module or raise ZipImportError + if it couldn't be imported. """ # Deciding the filename requires working out where the code # would come from if the module was actually loaded @@ -236,8 +274,13 @@ def load_module(self, fullname): Load the module specified by 'fullname'. 'fullname' must be the fully qualified (dotted) module name. It returns the imported - module, or raises ZipImportError if it wasn't found. + module, or raises ZipImportError if it could not be imported. + + Deprecated since Python 3.10. Use exec_module() instead. """ + msg = ("zipimport.zipimporter.load_module() is deprecated and slated for " + "removal in Python 3.12; use exec_module() instead") + _warnings.warn(msg, DeprecationWarning) code, ispackage, modpath = _get_module_code(self, fullname) mod = sys.modules.get(fullname) if mod is None or not isinstance(mod, _module_type): @@ -280,11 +323,18 @@ def get_resource_reader(self, fullname): return None except ZipImportError: return None - if not _ZipImportResourceReader._registered: - from importlib.abc import ResourceReader - ResourceReader.register(_ZipImportResourceReader) - _ZipImportResourceReader._registered = True - return _ZipImportResourceReader(self, fullname) + from importlib.readers import ZipReader + return ZipReader(self, fullname) + + + def invalidate_caches(self): + """Reload the file data of the archive path.""" + try: + self._files = _read_directory(self.archive) + _zip_directory_cache[self.archive] = self._files + except ZipImportError: + _zip_directory_cache.pop(self.archive, None) + self._files = {} def __repr__(self): @@ -580,20 +630,15 @@ def _eq_mtime(t1, t2): # Given the contents of a .py[co] file, unmarshal the data -# and return the code object. Return None if it the magic word doesn't -# match, or if the recorded .py[co] metadata does not match the source, -# (we do this instead of raising an exception as we fall back -# to .py if available and we don't want to mask other errors). +# and return the code object. Raises ImportError it the magic word doesn't +# match, or if the recorded .py[co] metadata does not match the source. def _unmarshal_code(self, pathname, fullpath, fullname, data): exc_details = { 'name': fullname, 'path': fullpath, } - try: - flags = _bootstrap_external._classify_pyc(data, fullname, exc_details) - except ImportError: - return None + flags = _bootstrap_external._classify_pyc(data, fullname, exc_details) hash_based = flags & 0b1 != 0 if hash_based: @@ -607,11 +652,8 @@ def _unmarshal_code(self, pathname, fullpath, fullname, data): source_bytes, ) - try: - _bootstrap_external._validate_hash_pyc( - data, source_hash, fullname, exc_details) - except ImportError: - return None + _bootstrap_external._validate_hash_pyc( + data, source_hash, fullname, exc_details) else: source_mtime, source_size = \ _get_mtime_and_size_of_source(self, fullpath) @@ -697,6 +739,7 @@ def _get_pyc_source(self, path): # 'fullname'. def _get_module_code(self, fullname): path = _get_module_path(self, fullname) + import_error = None for suffix, isbytecode, ispackage in _zip_searchorder: fullpath = path + suffix _bootstrap._verbose_message('trying {}{}{}', self.archive, path_sep, fullpath, verbosity=2) @@ -707,8 +750,12 @@ def _get_module_code(self, fullname): else: modpath = toc_entry[0] data = _get_data(self.archive, toc_entry) + code = None if isbytecode: - code = _unmarshal_code(self, modpath, fullpath, fullname, data) + try: + code = _unmarshal_code(self, modpath, fullpath, fullname, data) + except ImportError as exc: + import_error = exc else: code = _compile_source(modpath, data) if code is None: @@ -718,75 +765,8 @@ def _get_module_code(self, fullname): modpath = toc_entry[0] return code, ispackage, modpath else: - raise ZipImportError(f"can't find module {fullname!r}", name=fullname) - - -class _ZipImportResourceReader: - """Private class used to support ZipImport.get_resource_reader(). - - This class is allowed to reference all the innards and private parts of - the zipimporter. - """ - _registered = False - - def __init__(self, zipimporter, fullname): - self.zipimporter = zipimporter - self.fullname = fullname - - def open_resource(self, resource): - fullname_as_path = self.fullname.replace('.', '/') - path = f'{fullname_as_path}/{resource}' - from io import BytesIO - try: - return BytesIO(self.zipimporter.get_data(path)) - except OSError: - raise FileNotFoundError(path) - - def resource_path(self, resource): - # All resources are in the zip file, so there is no path to the file. - # Raising FileNotFoundError tells the higher level API to extract the - # binary data and create a temporary file. - raise FileNotFoundError - - def is_resource(self, name): - # Maybe we could do better, but if we can get the data, it's a - # resource. Otherwise it isn't. - fullname_as_path = self.fullname.replace('.', '/') - path = f'{fullname_as_path}/{name}' - try: - self.zipimporter.get_data(path) - except OSError: - return False - return True - - def contents(self): - # This is a bit convoluted, because fullname will be a module path, - # but _files is a list of file names relative to the top of the - # archive's namespace. We want to compare file paths to find all the - # names of things inside the module represented by fullname. So we - # turn the module path of fullname into a file path relative to the - # top of the archive, and then we iterate through _files looking for - # names inside that "directory". - from pathlib import Path - fullname_path = Path(self.zipimporter.get_filename(self.fullname)) - relative_path = fullname_path.relative_to(self.zipimporter.archive) - # Don't forget that fullname names a package, so its path will include - # __init__.py, which we want to ignore. - assert relative_path.name == '__init__.py' - package_path = relative_path.parent - subdirs_seen = set() - for filename in self.zipimporter._files: - try: - relative = Path(filename).relative_to(package_path) - except ValueError: - continue - # If the path of the file (which is relative to the top of the zip - # namespace), relative to the package given when the resource - # reader was created, has a parent, then it's a name in a - # subdirectory and thus we skip it. - parent_name = relative.parent.name - if len(parent_name) == 0: - yield relative.name - elif parent_name not in subdirs_seen: - subdirs_seen.add(parent_name) - yield parent_name + if import_error: + msg = f"module load failed: {import_error}" + raise ZipImportError(msg, name=fullname) from import_error + else: + raise ZipImportError(f"can't find module {fullname!r}", name=fullname)