From dcbcef63af391d45fc8d87b37db1ca1c86e2867d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 23 Oct 2020 15:52:18 -0700 Subject: [PATCH 1/3] bpo-42133: update parts of the stdlib to fall back to `__spec__.parent` when `__loader__` is missing --- Doc/whatsnew/3.10.rst | 24 +++++++++++++++++++ Lib/doctest.py | 18 ++++++++------ Lib/inspect.py | 7 ++++-- Lib/linecache.py | 23 +++++++++++------- Lib/site.py | 11 +++++++-- .../2020-10-23-15-47-47.bpo-42133.BzizYV.rst | 2 ++ 6 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index e464be6eb7e951..898f00b7720703 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -186,6 +186,12 @@ by :func:`curses.color_content`, :func:`curses.init_color`, support is provided by the underlying ncurses library. (Contributed by Jeffrey Kintscher and Hans Petter Jansson in :issue:`36982`.) +doctest +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.parent``. +(Contributed by Brett Cannon in :issue:`42133`.) + encodings --------- :func:`encodings.normalize_encoding` now ignores non-ASCII characters. @@ -198,6 +204,18 @@ Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and :func:`~glob.iglob` which allow to specify the root directory for searching. (Contributed by Serhiy Storchaka in :issue:`38144`.) +inspect +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.parent``. +(Contributed by Brett Cannon in :issue:`42133`.) + +linecache +--------- + +When a module does not define ``__loader__``, fall back to ``__spec__.parent``. +(Contributed by Brett Cannon in :issue:`42133`.) + os -- @@ -210,6 +228,12 @@ py_compile Added ``--quiet`` option to command-line interface of :mod:`py_compile`. (Contributed by Gregory Schevchenko in :issue:`38731`.) +site +---- + +When a module does not define ``__loader__``, fall back to ``__spec__.parent``. +(Contributed by Brett Cannon in :issue:`42133`.) + sys --- diff --git a/Lib/doctest.py b/Lib/doctest.py index baa503c83f8757..5bb35c9715d1e9 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -222,13 +222,17 @@ def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) filename = _module_relative_path(package, filename) - if getattr(package, '__loader__', None) is not None: - if hasattr(package.__loader__, 'get_data'): - file_contents = package.__loader__.get_data(filename) - file_contents = file_contents.decode(encoding) - # get_data() opens files as 'rb', so one must do the equivalent - # conversion as universal newlines would do. - return _newline_convert(file_contents), filename + if (loader := getattr(package, '__loader__', None)) is None: + try: + loader = package.__spec__.loader + except AttributeError: + pass + if hasattr(loader, 'get_data'): + file_contents = loader.get_data(filename) + file_contents = file_contents.decode(encoding) + # get_data() opens files as 'rb', so one must do the equivalent + # conversion as universal newlines would do. + return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename diff --git a/Lib/inspect.py b/Lib/inspect.py index ac127cbd725b9b..7412d0e837cf14 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -707,10 +707,13 @@ def getsourcefile(object): if os.path.exists(filename): return filename # only return a non-existent filename if the module has a PEP 302 loader - if getattr(getmodule(object, filename), '__loader__', None) is not None: + module = getmodule(object, filename) + if getattr(module, '__loader__', None) is not None: + return filename + elif getattr(getattr(module, "__spec__", None), "loader", None) is not None: return filename # or it is in the linecache - if filename in linecache.cache: + elif filename in linecache.cache: return filename def getabsfile(object, _filename=None): diff --git a/Lib/linecache.py b/Lib/linecache.py index fa5dbd09eab869..0e4245b594594d 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -165,13 +165,18 @@ def lazycache(filename, module_globals): if not filename or (filename.startswith('<') and filename.endswith('>')): return False # Try for a __loader__, if available - if module_globals and '__loader__' in module_globals: - name = module_globals.get('__name__') - loader = module_globals['__loader__'] - get_source = getattr(loader, 'get_source', None) - - if name and get_source: - get_lines = functools.partial(get_source, name) - cache[filename] = (get_lines,) - return True + if module_globals and '__name__' in module_globals: + name = module_globals['__name__'] + if (loader := module_globals.get('__loader__')) is None: + if spec := module_globals.get('__spec__'): + try: + loader = spec.loader + except AttributeError: + pass + get_source = getattr(loader, 'get_source', None) + + if name and get_source: + get_lines = functools.partial(get_source, name) + cache[filename] = (get_lines,) + return True return False diff --git a/Lib/site.py b/Lib/site.py index 4d3b869fff77a0..3a0f619d71c86e 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -105,8 +105,15 @@ def makepath(*paths): def abs_paths(): """Set all module __file__ and __cached__ attributes to an absolute path""" for m in set(sys.modules.values()): - if (getattr(getattr(m, '__loader__', None), '__module__', None) not in - ('_frozen_importlib', '_frozen_importlib_external')): + loader_module = None + try: + loader_module = m.__loader__.__module__ + except AttributeError: + try: + loader_module = m.__spec__.loader.__module__ + except AttributeError: + pass + if loader_module not in {'_frozen_importlib', '_frozen_importlib_external'}: continue # don't mess with a PEP 302-supplied __file__ try: m.__file__ = os.path.abspath(m.__file__) diff --git a/Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst b/Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst new file mode 100644 index 00000000000000..a75907a2286b64 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst @@ -0,0 +1,2 @@ +Update various modules in the stdlib to fall back on `__spec__.parent` when +`__loader__` isn't defined on a module. From 98f300dfcefda37a66db2ec6454280854d4023aa Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 6 Nov 2020 15:44:38 -0800 Subject: [PATCH 2/3] Fix indentation --- Lib/linecache.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Lib/linecache.py b/Lib/linecache.py index 0e4245b594594d..513b17e999880b 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -166,17 +166,17 @@ def lazycache(filename, module_globals): return False # Try for a __loader__, if available if module_globals and '__name__' in module_globals: - name = module_globals['__name__'] - if (loader := module_globals.get('__loader__')) is None: - if spec := module_globals.get('__spec__'): - try: - loader = spec.loader - except AttributeError: - pass - get_source = getattr(loader, 'get_source', None) - - if name and get_source: - get_lines = functools.partial(get_source, name) - cache[filename] = (get_lines,) - return True + name = module_globals['__name__'] + if (loader := module_globals.get('__loader__')) is None: + if spec := module_globals.get('__spec__'): + try: + loader = spec.loader + except AttributeError: + pass + get_source = getattr(loader, 'get_source', None) + + if name and get_source: + get_lines = functools.partial(get_source, name) + cache[filename] = (get_lines,) + return True return False From db2410d9744f8be0647892a64e7922a80d11c320 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 6 Nov 2020 16:12:19 -0800 Subject: [PATCH 3/3] Mean to say `loader`, not `parent` --- Doc/whatsnew/3.10.rst | 8 ++++---- .../next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index e15af218903fac..a735bf235435ca 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -189,7 +189,7 @@ support is provided by the underlying ncurses library. doctest ------- -When a module does not define ``__loader__``, fall back to ``__spec__.parent``. +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. (Contributed by Brett Cannon in :issue:`42133`.) encodings @@ -207,13 +207,13 @@ Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and inspect ------- -When a module does not define ``__loader__``, fall back to ``__spec__.parent``. +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. (Contributed by Brett Cannon in :issue:`42133`.) linecache --------- -When a module does not define ``__loader__``, fall back to ``__spec__.parent``. +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. (Contributed by Brett Cannon in :issue:`42133`.) os @@ -238,7 +238,7 @@ instead of :mod:`pickle` protocol ``3`` when creating shelves. site ---- -When a module does not define ``__loader__``, fall back to ``__spec__.parent``. +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. (Contributed by Brett Cannon in :issue:`42133`.) sys diff --git a/Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst b/Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst index a75907a2286b64..f3cfa1a8dce338 100644 --- a/Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst +++ b/Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst @@ -1,2 +1,2 @@ -Update various modules in the stdlib to fall back on `__spec__.parent` when +Update various modules in the stdlib to fall back on `__spec__.loader` when `__loader__` isn't defined on a module.