From f8cb92aea8de911ed3796a8d26421f63ede7b333 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 1 Mar 2020 11:31:07 -0600 Subject: [PATCH 01/10] Roughly move implementations as they were moved to port to cpython 3.7. --- .gitattributes | 2 - .gitignore | 102 ------- .gitlab-ci.yml | 36 --- LICENSE | 13 - {importlib_resources => Lib/importlib}/abc.py | 0 .../_py3.py => Lib/importlib/resources.py | 0 .../tests/test_importlib}/__init__.py | 0 .../tests/test_importlib}/data01/__init__.py | 0 .../tests/test_importlib}/data01/binary.file | Bin .../data01/subdirectory/__init__.py | 0 .../data01/subdirectory/binary.file | Bin .../tests/test_importlib}/data01/utf-16.file | Bin .../tests/test_importlib}/data01/utf-8.file | 0 .../tests/test_importlib}/data02/__init__.py | 0 .../test_importlib}/data02/one/__init__.py | 0 .../test_importlib}/data02/one/resource1.txt | 0 .../test_importlib}/data02/two/__init__.py | 0 .../test_importlib}/data02/two/resource2.txt | 0 .../tests/test_importlib}/data03/__init__.py | 0 .../data03/namespace/portion1/__init__.py | 0 .../data03/namespace/portion2/__init__.py | 0 .../data03/namespace/resource1.txt | 0 .../tests/test_importlib}/test_open.py | 0 .../tests/test_importlib}/test_path.py | 0 .../tests/test_importlib}/test_read.py | 0 .../tests/test_importlib}/test_resource.py | 0 .../tests/test_importlib}/util.py | 0 .../test_importlib}/zipdata01/__init__.py | 0 .../test_importlib}/zipdata01/ziptestdata.zip | Bin .../test_importlib}/zipdata02/__init__.py | 0 .../test_importlib}/zipdata02/ziptestdata.zip | Bin MANIFEST.in | 5 - README.rst | 28 -- coverage.ini | 26 -- coverplug.py | 21 -- importlib_resources/__init__.py | 36 --- importlib_resources/_compat.py | 23 -- importlib_resources/_py2.py | 270 ------------------ importlib_resources/docs/_static/.ignoreme | 0 importlib_resources/docs/changelog.rst | 61 ---- importlib_resources/docs/conf.py | 180 ------------ importlib_resources/docs/index.rst | 56 ---- importlib_resources/docs/migration.rst | 160 ----------- importlib_resources/docs/using.rst | 175 ------------ importlib_resources/version.txt | 1 - pyproject.toml | 2 - release.sh | 10 - setup.cfg | 47 --- setup.py | 2 - tox.ini | 72 ----- update-zips.py | 34 --- version.py | 9 - 52 files changed, 1371 deletions(-) delete mode 100644 .gitattributes delete mode 100644 .gitignore delete mode 100644 .gitlab-ci.yml delete mode 100644 LICENSE rename {importlib_resources => Lib/importlib}/abc.py (100%) rename importlib_resources/_py3.py => Lib/importlib/resources.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/__init__.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data01/__init__.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data01/binary.file (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data01/subdirectory/__init__.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data01/subdirectory/binary.file (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data01/utf-16.file (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data01/utf-8.file (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data02/__init__.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data02/one/__init__.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data02/one/resource1.txt (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data02/two/__init__.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data02/two/resource2.txt (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data03/__init__.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data03/namespace/portion1/__init__.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data03/namespace/portion2/__init__.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/data03/namespace/resource1.txt (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/test_open.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/test_path.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/test_read.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/test_resource.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/util.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/zipdata01/__init__.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/zipdata01/ziptestdata.zip (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/zipdata02/__init__.py (100%) rename {importlib_resources/tests => Lib/tests/test_importlib}/zipdata02/ziptestdata.zip (100%) delete mode 100644 MANIFEST.in delete mode 100644 README.rst delete mode 100644 coverage.ini delete mode 100644 coverplug.py delete mode 100644 importlib_resources/__init__.py delete mode 100644 importlib_resources/_compat.py delete mode 100644 importlib_resources/_py2.py delete mode 100644 importlib_resources/docs/_static/.ignoreme delete mode 100644 importlib_resources/docs/changelog.rst delete mode 100644 importlib_resources/docs/conf.py delete mode 100644 importlib_resources/docs/index.rst delete mode 100644 importlib_resources/docs/migration.rst delete mode 100644 importlib_resources/docs/using.rst delete mode 100644 importlib_resources/version.txt delete mode 100644 pyproject.toml delete mode 100755 release.sh delete mode 100644 setup.cfg delete mode 100644 setup.py delete mode 100644 tox.ini delete mode 100755 update-zips.py delete mode 100644 version.py diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index e6805180..00000000 --- a/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -*.file binary -*.zip binary diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 87274500..00000000 --- a/.gitignore +++ /dev/null @@ -1,102 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -/diffcov.html diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index f78884c5..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,36 +0,0 @@ -image: quay.io/python-devs/ci-image - -stages: - - test - - codecov - -qa: - script: - - tox -e qa - -tests: - script: - - tox -e py27-nocov,py34-nocov,py35-nocov,py36-nocov - -coverage: - script: - - tox -e py27-cov,py34-cov,py35-cov,py36-cov - artifacts: - paths: - - coverage.xml - -diffcov: - script: - - tox -e py27-diffcov,py34-diffcov,py35-diffcov,py36-diffcov - -docs: - script: - - tox -e docs - -codecov: - stage: codecov - dependencies: - - coverage - script: - - codecov -t $CODECOV_TOKEN - when: on_success diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 7e479106..00000000 --- a/LICENSE +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2017-2018 Brett Cannon, Barry Warsaw - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/importlib_resources/abc.py b/Lib/importlib/abc.py similarity index 100% rename from importlib_resources/abc.py rename to Lib/importlib/abc.py diff --git a/importlib_resources/_py3.py b/Lib/importlib/resources.py similarity index 100% rename from importlib_resources/_py3.py rename to Lib/importlib/resources.py diff --git a/importlib_resources/tests/__init__.py b/Lib/tests/test_importlib/__init__.py similarity index 100% rename from importlib_resources/tests/__init__.py rename to Lib/tests/test_importlib/__init__.py diff --git a/importlib_resources/tests/data01/__init__.py b/Lib/tests/test_importlib/data01/__init__.py similarity index 100% rename from importlib_resources/tests/data01/__init__.py rename to Lib/tests/test_importlib/data01/__init__.py diff --git a/importlib_resources/tests/data01/binary.file b/Lib/tests/test_importlib/data01/binary.file similarity index 100% rename from importlib_resources/tests/data01/binary.file rename to Lib/tests/test_importlib/data01/binary.file diff --git a/importlib_resources/tests/data01/subdirectory/__init__.py b/Lib/tests/test_importlib/data01/subdirectory/__init__.py similarity index 100% rename from importlib_resources/tests/data01/subdirectory/__init__.py rename to Lib/tests/test_importlib/data01/subdirectory/__init__.py diff --git a/importlib_resources/tests/data01/subdirectory/binary.file b/Lib/tests/test_importlib/data01/subdirectory/binary.file similarity index 100% rename from importlib_resources/tests/data01/subdirectory/binary.file rename to Lib/tests/test_importlib/data01/subdirectory/binary.file diff --git a/importlib_resources/tests/data01/utf-16.file b/Lib/tests/test_importlib/data01/utf-16.file similarity index 100% rename from importlib_resources/tests/data01/utf-16.file rename to Lib/tests/test_importlib/data01/utf-16.file diff --git a/importlib_resources/tests/data01/utf-8.file b/Lib/tests/test_importlib/data01/utf-8.file similarity index 100% rename from importlib_resources/tests/data01/utf-8.file rename to Lib/tests/test_importlib/data01/utf-8.file diff --git a/importlib_resources/tests/data02/__init__.py b/Lib/tests/test_importlib/data02/__init__.py similarity index 100% rename from importlib_resources/tests/data02/__init__.py rename to Lib/tests/test_importlib/data02/__init__.py diff --git a/importlib_resources/tests/data02/one/__init__.py b/Lib/tests/test_importlib/data02/one/__init__.py similarity index 100% rename from importlib_resources/tests/data02/one/__init__.py rename to Lib/tests/test_importlib/data02/one/__init__.py diff --git a/importlib_resources/tests/data02/one/resource1.txt b/Lib/tests/test_importlib/data02/one/resource1.txt similarity index 100% rename from importlib_resources/tests/data02/one/resource1.txt rename to Lib/tests/test_importlib/data02/one/resource1.txt diff --git a/importlib_resources/tests/data02/two/__init__.py b/Lib/tests/test_importlib/data02/two/__init__.py similarity index 100% rename from importlib_resources/tests/data02/two/__init__.py rename to Lib/tests/test_importlib/data02/two/__init__.py diff --git a/importlib_resources/tests/data02/two/resource2.txt b/Lib/tests/test_importlib/data02/two/resource2.txt similarity index 100% rename from importlib_resources/tests/data02/two/resource2.txt rename to Lib/tests/test_importlib/data02/two/resource2.txt diff --git a/importlib_resources/tests/data03/__init__.py b/Lib/tests/test_importlib/data03/__init__.py similarity index 100% rename from importlib_resources/tests/data03/__init__.py rename to Lib/tests/test_importlib/data03/__init__.py diff --git a/importlib_resources/tests/data03/namespace/portion1/__init__.py b/Lib/tests/test_importlib/data03/namespace/portion1/__init__.py similarity index 100% rename from importlib_resources/tests/data03/namespace/portion1/__init__.py rename to Lib/tests/test_importlib/data03/namespace/portion1/__init__.py diff --git a/importlib_resources/tests/data03/namespace/portion2/__init__.py b/Lib/tests/test_importlib/data03/namespace/portion2/__init__.py similarity index 100% rename from importlib_resources/tests/data03/namespace/portion2/__init__.py rename to Lib/tests/test_importlib/data03/namespace/portion2/__init__.py diff --git a/importlib_resources/tests/data03/namespace/resource1.txt b/Lib/tests/test_importlib/data03/namespace/resource1.txt similarity index 100% rename from importlib_resources/tests/data03/namespace/resource1.txt rename to Lib/tests/test_importlib/data03/namespace/resource1.txt diff --git a/importlib_resources/tests/test_open.py b/Lib/tests/test_importlib/test_open.py similarity index 100% rename from importlib_resources/tests/test_open.py rename to Lib/tests/test_importlib/test_open.py diff --git a/importlib_resources/tests/test_path.py b/Lib/tests/test_importlib/test_path.py similarity index 100% rename from importlib_resources/tests/test_path.py rename to Lib/tests/test_importlib/test_path.py diff --git a/importlib_resources/tests/test_read.py b/Lib/tests/test_importlib/test_read.py similarity index 100% rename from importlib_resources/tests/test_read.py rename to Lib/tests/test_importlib/test_read.py diff --git a/importlib_resources/tests/test_resource.py b/Lib/tests/test_importlib/test_resource.py similarity index 100% rename from importlib_resources/tests/test_resource.py rename to Lib/tests/test_importlib/test_resource.py diff --git a/importlib_resources/tests/util.py b/Lib/tests/test_importlib/util.py similarity index 100% rename from importlib_resources/tests/util.py rename to Lib/tests/test_importlib/util.py diff --git a/importlib_resources/tests/zipdata01/__init__.py b/Lib/tests/test_importlib/zipdata01/__init__.py similarity index 100% rename from importlib_resources/tests/zipdata01/__init__.py rename to Lib/tests/test_importlib/zipdata01/__init__.py diff --git a/importlib_resources/tests/zipdata01/ziptestdata.zip b/Lib/tests/test_importlib/zipdata01/ziptestdata.zip similarity index 100% rename from importlib_resources/tests/zipdata01/ziptestdata.zip rename to Lib/tests/test_importlib/zipdata01/ziptestdata.zip diff --git a/importlib_resources/tests/zipdata02/__init__.py b/Lib/tests/test_importlib/zipdata02/__init__.py similarity index 100% rename from importlib_resources/tests/zipdata02/__init__.py rename to Lib/tests/test_importlib/zipdata02/__init__.py diff --git a/importlib_resources/tests/zipdata02/ziptestdata.zip b/Lib/tests/test_importlib/zipdata02/ziptestdata.zip similarity index 100% rename from importlib_resources/tests/zipdata02/ziptestdata.zip rename to Lib/tests/test_importlib/zipdata02/ziptestdata.zip diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 03c3d3c6..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include *.py MANIFEST.in LICENSE README.rst -global-include *.txt *.rst *.ini *.cfg *.toml -exclude .gitignore -prune build -prune .tox diff --git a/README.rst b/README.rst deleted file mode 100644 index 393d33a8..00000000 --- a/README.rst +++ /dev/null @@ -1,28 +0,0 @@ -========================= - ``importlib_resources`` -========================= - -``importlib_resources`` is a backport of Python 3.7's standard library -`importlib.resources -`_ -module for Python 2.7, and 3.4 through 3.6. Users of Python 3.7 and beyond -should use the standard library module, since for these versions, -``importlib_resources`` just delegates to that module. - -The key goal of this module is to replace parts of `pkg_resources -`_ with a -solution in Python's stdlib that relies on well-defined APIs. This makes -reading resources included in packages easier, with more stable and consistent -semantics. - -Note that ``pip 10`` is required if you are going to ``pip install -importlib_resources``. - - -Project details -=============== - - * Project home: https://gitlab.com/python-devs/importlib_resources - * Report bugs at: https://gitlab.com/python-devs/importlib_resources/issues - * Code hosting: https://gitlab.com/python-devs/importlib_resources.git - * Documentation: http://importlib_resources.readthedocs.io/ diff --git a/coverage.ini b/coverage.ini deleted file mode 100644 index adcb579d..00000000 --- a/coverage.ini +++ /dev/null @@ -1,26 +0,0 @@ -[run] -branch = true -parallel = true -omit = - setup* - .tox/*/lib/python*/site-packages/* - */tests/*.py - */testing/*.py - importlib_resources/_py${OMIT}.py - importlib_resources/__init__.py - importlib_resources/_compat.py - importlib_resources/abc.py -plugins = - coverplug - -[report] -exclude_lines = - pragma: nocover - raise NotImplementedError - raise AssertionError - assert\s - -[paths] -source = - importlib_resources - .tox/*/lib/python*/site-packages/importlib_resources diff --git a/coverplug.py b/coverplug.py deleted file mode 100644 index 0b0c7cb5..00000000 --- a/coverplug.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Coverage plugin to add exclude lines based on the Python version.""" - -import sys - -from coverage import CoveragePlugin - - -class MyConfigPlugin(CoveragePlugin): - def configure(self, config): - opt_name = 'report:exclude_lines' - exclude_lines = config.get_option(opt_name) - # Python >= 3.6 has os.PathLike. - if sys.version_info >= (3, 6): - exclude_lines.append('pragma: >=36') - else: - exclude_lines.append('pragma: <=35') - config.set_option(opt_name, exclude_lines) - - -def coverage_init(reg, options): - reg.add_configurer(MyConfigPlugin()) diff --git a/importlib_resources/__init__.py b/importlib_resources/__init__.py deleted file mode 100644 index fab437a4..00000000 --- a/importlib_resources/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Read resources contained within a package.""" - -import sys - - -__all__ = [ - 'contents', - 'is_resource', - 'open_binary', - 'open_text', - 'path', - 'read_binary', - 'read_text', - ] - - -# Use the Python 3.7 stdlib implementation if available. -if sys.version_info >= (3, 7): - from importlib.resources import ( - Package, Resource, contents, is_resource, open_binary, open_text, path, - read_binary, read_text) - from importlib.abc import ResourceReader - __all__.extend(['Package', 'Resource', 'ResourceReader']) -elif sys.version_info >= (3,): - from importlib_resources._py3 import ( - Package, Resource, contents, is_resource, open_binary, open_text, path, - read_binary, read_text) - from importlib_resources.abc import ResourceReader - __all__.extend(['Package', 'Resource', 'ResourceReader']) -else: - from importlib_resources._py2 import ( - contents, is_resource, open_binary, open_text, path, read_binary, - read_text) - - -__version__ = read_text('importlib_resources', 'version.txt').strip() diff --git a/importlib_resources/_compat.py b/importlib_resources/_compat.py deleted file mode 100644 index 28d61276..00000000 --- a/importlib_resources/_compat.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import absolute_import - -# flake8: noqa - -try: - from pathlib import Path, PurePath -except ImportError: - from pathlib2 import Path, PurePath # type: ignore - - -try: - from abc import ABC # type: ignore -except ImportError: - from abc import ABCMeta - - class ABC(object): # type: ignore - __metaclass__ = ABCMeta - - -try: - FileNotFoundError = FileNotFoundError # type: ignore -except NameError: - FileNotFoundError = OSError diff --git a/importlib_resources/_py2.py b/importlib_resources/_py2.py deleted file mode 100644 index 376f0e38..00000000 --- a/importlib_resources/_py2.py +++ /dev/null @@ -1,270 +0,0 @@ -import os -import errno -import tempfile - -from ._compat import FileNotFoundError -from contextlib import contextmanager -from importlib import import_module -from io import BytesIO, TextIOWrapper, open as io_open -from pathlib2 import Path -from zipfile import ZipFile - - -def _get_package(package): - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - if isinstance(package, basestring): # noqa: F821 - module = import_module(package) - else: - module = package - if not hasattr(module, '__path__'): - raise TypeError("{!r} is not a package".format(package)) - return module - - -def _normalize_path(path): - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError("{!r} must be only a file name".format(path)) - else: - return file_name - - -def open_binary(package, resource): - """Return a file-like object opened for binary reading of the resource.""" - resource = _normalize_path(resource) - package = _get_package(package) - # Using pathlib doesn't work well here due to the lack of 'strict' argument - # for pathlib.Path.resolve() prior to Python 3.6. - package_path = os.path.dirname(package.__file__) - relative_path = os.path.join(package_path, resource) - full_path = os.path.abspath(relative_path) - try: - return io_open(full_path, 'rb') - except IOError: - # This might be a package in a zip file. zipimport provides a loader - # with a functioning get_data() method, however we have to strip the - # archive (i.e. the .zip file's name) off the front of the path. This - # is because the zipimport loader in Python 2 doesn't actually follow - # PEP 302. It should allow the full path, but actually requires that - # the path be relative to the zip file. - try: - loader = package.__loader__ - full_path = relative_path[len(loader.archive)+1:] - data = loader.get_data(full_path) - except (IOError, AttributeError): - package_name = package.__name__ - message = '{!r} resource not found in {!r}'.format( - resource, package_name) - raise FileNotFoundError(message) - else: - return BytesIO(data) - - -def open_text(package, resource, encoding='utf-8', errors='strict'): - """Return a file-like object opened for text reading of the resource.""" - resource = _normalize_path(resource) - package = _get_package(package) - # Using pathlib doesn't work well here due to the lack of 'strict' argument - # for pathlib.Path.resolve() prior to Python 3.6. - package_path = os.path.dirname(package.__file__) - relative_path = os.path.join(package_path, resource) - full_path = os.path.abspath(relative_path) - try: - return io_open(full_path, mode='r', encoding=encoding, errors=errors) - except IOError: - # This might be a package in a zip file. zipimport provides a loader - # with a functioning get_data() method, however we have to strip the - # archive (i.e. the .zip file's name) off the front of the path. This - # is because the zipimport loader in Python 2 doesn't actually follow - # PEP 302. It should allow the full path, but actually requires that - # the path be relative to the zip file. - try: - loader = package.__loader__ - full_path = relative_path[len(loader.archive)+1:] - data = loader.get_data(full_path) - except (IOError, AttributeError): - package_name = package.__name__ - message = '{!r} resource not found in {!r}'.format( - resource, package_name) - raise FileNotFoundError(message) - else: - return TextIOWrapper(BytesIO(data), encoding, errors) - - -def read_binary(package, resource): - """Return the binary contents of the resource.""" - resource = _normalize_path(resource) - package = _get_package(package) - with open_binary(package, resource) as fp: - return fp.read() - - -def read_text(package, resource, encoding='utf-8', errors='strict'): - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - resource = _normalize_path(resource) - package = _get_package(package) - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -@contextmanager -def path(package, resource): - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - resource = _normalize_path(resource) - package = _get_package(package) - package_directory = Path(package.__file__).parent - file_path = package_directory / resource - # If the file actually exists on the file system, just return it. - # Otherwise, it's probably in a zip file, so we need to create a temporary - # file and copy the contents into that file, hence the contextmanager to - # clean up the temp file resource. - if file_path.exists(): - yield file_path - else: - with open_binary(package, resource) as fp: - data = fp.read() - # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' - # blocks due to the need to close the temporary file to work on Windows - # properly. - fd, raw_path = tempfile.mkstemp() - try: - os.write(fd, data) - os.close(fd) - yield Path(raw_path) - finally: - try: - os.remove(raw_path) - except FileNotFoundError: - pass - - -def is_resource(package, name): - """True if name is a resource inside package. - - Directories are *not* resources. - """ - package = _get_package(package) - _normalize_path(name) - try: - package_contents = set(contents(package)) - except OSError as error: - if error.errno not in (errno.ENOENT, errno.ENOTDIR): - # We won't hit this in the Python 2 tests, so it'll appear - # uncovered. We could mock os.listdir() to return a non-ENOENT or - # ENOTDIR, but then we'd have to depend on another external - # library since Python 2 doesn't have unittest.mock. It's not - # worth it. - raise # pragma: nocover - return False - if name not in package_contents: - return False - # Just because the given file_name lives as an entry in the package's - # contents doesn't necessarily mean it's a resource. Directories are not - # resources, so let's try to find out if it's a directory or not. - path = Path(package.__file__).parent / name - if path.is_file(): - return True - if path.is_dir(): - return False - # If it's not a file and it's not a directory, what is it? Well, this - # means the file doesn't exist on the file system, so it probably lives - # inside a zip file. We have to crack open the zip, look at its table of - # contents, and make sure that this entry doesn't have sub-entries. - archive_path = package.__loader__.archive # type: ignore - package_directory = Path(package.__file__).parent - with ZipFile(archive_path) as zf: - toc = zf.namelist() - relpath = package_directory.relative_to(archive_path) - candidate_path = relpath / name - for entry in toc: # pragma: nobranch - try: - relative_to_candidate = Path(entry).relative_to(candidate_path) - except ValueError: - # The two paths aren't relative to each other so we can ignore it. - continue - # Since directories aren't explicitly listed in the zip file, we must - # infer their 'directory-ness' by looking at the number of path - # components in the path relative to the package resource we're - # looking up. If there are zero additional parts, it's a file, i.e. a - # resource. If there are more than zero it's a directory, i.e. not a - # resource. It has to be one of these two cases. - return len(relative_to_candidate.parts) == 0 - # I think it's impossible to get here. It would mean that we are looking - # for a resource in a zip file, there's an entry matching it in the return - # value of contents(), but we never actually found it in the zip's table of - # contents. - raise AssertionError('Impossible situation') - - -def contents(package): - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - package = _get_package(package) - package_directory = Path(package.__file__).parent - try: - return os.listdir(str(package_directory)) - except OSError as error: - if error.errno not in (errno.ENOENT, errno.ENOTDIR): - # We won't hit this in the Python 2 tests, so it'll appear - # uncovered. We could mock os.listdir() to return a non-ENOENT or - # ENOTDIR, but then we'd have to depend on another external - # library since Python 2 doesn't have unittest.mock. It's not - # worth it. - raise # pragma: nocover - # The package is probably in a zip file. - archive_path = getattr(package.__loader__, 'archive', None) - if archive_path is None: - raise - relpath = package_directory.relative_to(archive_path) - with ZipFile(archive_path) as zf: - toc = zf.namelist() - subdirs_seen = set() # type: Set - subdirs_returned = [] - for filename in toc: - path = Path(filename) - # Strip off any path component parts that are in common with the - # package directory, relative to the zip archive's file system - # path. This gives us all the parts that live under the named - # package inside the zip file. If the length of these subparts is - # exactly 1, then it is situated inside the package. The resulting - # length will be 0 if it's above the package, and it will be - # greater than 1 if it lives in a subdirectory of the package - # directory. - # - # However, since directories themselves don't appear in the zip - # archive as a separate entry, we need to return the first path - # component for any case that has > 1 subparts -- but only once! - if path.parts[:len(relpath.parts)] != relpath.parts: - continue - subparts = path.parts[len(relpath.parts):] - if len(subparts) == 1: - subdirs_returned.append(subparts[0]) - elif len(subparts) > 1: # pragma: nobranch - subdir = subparts[0] - if subdir not in subdirs_seen: - subdirs_seen.add(subdir) - subdirs_returned.append(subdir) - return subdirs_returned diff --git a/importlib_resources/docs/_static/.ignoreme b/importlib_resources/docs/_static/.ignoreme deleted file mode 100644 index e69de29b..00000000 diff --git a/importlib_resources/docs/changelog.rst b/importlib_resources/docs/changelog.rst deleted file mode 100644 index 15de06f5..00000000 --- a/importlib_resources/docs/changelog.rst +++ /dev/null @@ -1,61 +0,0 @@ -========================== - importlib_resources NEWS -========================== - -1.0 (2018-06-28) -================ -* Backport fix for test isolation from Python 3.8/3.7. Closes #61 - -0.8 (2018-05-17) -================ -* Strip ``importlib_resources.__version__``. Closes #56 -* Fix a metadata problem with older setuptools. Closes #57 -* Add an ``__all__`` to ``importlib_resources``. Closes #59 - -0.7 (2018-05-15) -================ -* Fix ``setup.cfg`` metadata bug. Closes #55 - -0.6 (2018-05-15) -================ -* Move everything from ``pyproject.toml`` to ``setup.cfg``, with the added - benefit of fixing the PyPI metadata. Closes #54 -* Turn off mypy's ``strict_optional`` setting for now. - -0.5 (2018-05-01) -================ -* Resynchronize with Python 3.7; changes the return type of ``contents()`` to - be an ``Iterable``. Closes #52 - -0.4 (2018-03-27) -================ -* Correctly find resources in subpackages inside a zip file. Closes #51 - -0.3 (2018-02-17) -================ -* The API, implementation, and documentation is synchronized with the Python - 3.7 standard library. Closes #47 -* When run under Python 3.7 this API shadows the stdlib versions. Closes #50 - -0.2 (2017-12-13) -================ -* **Backward incompatible change**. Split the ``open()`` and ``read()`` calls - into separate binary and text versions, i.e. ``open_binary()``, - ``open_text()``, ``read_binary()``, and ``read_text()``. Closes #41 -* Fix a bug where unrelated resources could be returned from ``contents()``. - Closes #44 -* Correctly prevent namespace packages from containing resources. Closes #20 - -0.1 (2017-12-05) -================ -* Initial release. - - -.. - Local Variables: - mode: change-log-mode - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 78 - coding: utf-8 - End: diff --git a/importlib_resources/docs/conf.py b/importlib_resources/docs/conf.py deleted file mode 100644 index 78c41143..00000000 --- a/importlib_resources/docs/conf.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# flake8: noqa -# -# importlib_resources documentation build configuration file, created by -# sphinx-quickstart on Thu Nov 30 10:21:00 2017. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.coverage', - 'sphinx.ext.viewcode'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = 'importlib_resources' -copyright = '2017-2018, Brett Cannon, Barry Warsaw' -author = 'Brett Cannon, Barry Warsaw' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '0.1' -# The full version, including alpha/beta/rc tags. -release = '0.1' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', - ] -} - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'importlib_resourcesdoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'importlib_resources.tex', 'importlib\\_resources Documentation', - 'Brett Cannon, Barry Warsaw', 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'importlib_resources', 'importlib_resources Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'importlib_resources', 'importlib_resources Documentation', - author, 'importlib_resources', 'One line description of project.', - 'Miscellaneous'), -] - - - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = { - 'python': ('https://docs.python.org/3', None), - } diff --git a/importlib_resources/docs/index.rst b/importlib_resources/docs/index.rst deleted file mode 100644 index 981cfa0d..00000000 --- a/importlib_resources/docs/index.rst +++ /dev/null @@ -1,56 +0,0 @@ -================================ - Welcome to importlib_resources -================================ - -``importlib_resources`` is a library which provides for access to *resources* -in Python packages. It provides functionality similar to ``pkg_resources`` -`Basic Resource Access`_ API, but without all of the overhead and performance -problems of ``pkg_resources``. - -In our terminology, a *resource* is a file that is located within an -importable `Python package`_. Resources can live on the file system, in a zip -file, or in any place that has a loader_ supporting the appropriate API for -reading resources. Directories are not resources. - -``importlib_resources`` is a backport of Python 3.7's standard library -`importlib.resources`_ module for Python 2.7, and 3.4 through 3.6. Users of -Python 3.7 and beyond are encouraged to use the standard library module, and -in fact for these versions, ``importlib_resources`` just shadows that module. -Developers looking for detailed API descriptions should refer to the Python -3.7 standard library documentation. - -The documentation here includes a general :ref:`usage ` guide and a -:ref:`migration ` guide for projects that want to adopt -``importlib_resources`` instead of ``pkg_resources``. - - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - using.rst - migration.rst - changelog.rst - - -Project details -=============== - - * Project home: https://gitlab.com/python-devs/importlib_resources - * Report bugs at: https://gitlab.com/python-devs/importlib_resources/issues - * Code hosting: https://gitlab.com/python-devs/importlib_resources.git - * Documentation: http://importlib_resources.readthedocs.io/ - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - - -.. _`importlib.resources`: https://docs.python.org/3.7/library/importlib.html#module-importlib.resources -.. _`Basic Resource Access`: http://setuptools.readthedocs.io/en/latest/pkg_resources.html#basic-resource-access -.. _`Python package`: https://docs.python.org/3/reference/import.html#packages -.. _loader: https://docs.python.org/3/reference/import.html#finders-and-loaders diff --git a/importlib_resources/docs/migration.rst b/importlib_resources/docs/migration.rst deleted file mode 100644 index face2163..00000000 --- a/importlib_resources/docs/migration.rst +++ /dev/null @@ -1,160 +0,0 @@ -.. _migration: - -================= - Migration guide -================= - -The following guide will help you migrate common ``pkg_resources`` APIs to -``importlib_resources``. Only a small number of the most common APIs are -supported by ``importlib_resources``, so projects that use other features -(e.g. entry points) will have to find other solutions. -``importlib_resources`` primarily supports the following `basic resource -access`_ APIs: - -* ``pkg_resources.resource_filename()`` -* ``pkg_resources.resource_stream()`` -* ``pkg_resources.resource_string()`` -* ``pkg_resources.resource_listdir()`` -* ``pkg_resources.resource_isdir()`` - -Keep in mind that ``pkg_resources`` defines *resources* to include -directories. ``importlib_resources`` does not treat directories as resources; -since only files are allowed as resources, file names in the -``importlib_resources`` API may *not* include path separators (e.g. slashes). - - -pkg_resources.resource_filename() -================================= - -``resource_filename()`` is one of the more interesting APIs because it -guarantees that the return value names a file on the file system. This means -that if the resource is in a zip file, ``pkg_resources()`` will extract the -file and return the name of the temporary file it created. The problem is -that ``pkg_resources()`` also *implicitly* cleans up this temporary file, -without control over its lifetime by the programmer. - -``importlib_resources`` takes a different approach. Its equivalent API is the -``path()`` function, which returns a context manager providing a -:py:class:`pathlib.Path` object. This means users have both the flexibility -and responsibility to manage the lifetime of the temporary file. Note though -that if the resource is *already* on the file system, ``importlib_resources`` -still returns a context manager, but nothing needs to get cleaned up. - -Here's an example from ``pkg_resources()``:: - - path = pkg_resources.resource_filename('my.package', 'resource.dat') - -The best way to convert this is with the following idiom:: - - with importlib_resources.path('my.package', 'resource.dat') as path: - # Do something with path. After the with-statement exits, any - # temporary file created will be immediately cleaned up. - -That's all fine if you only need the file temporarily, but what if you need it -to stick around for a while? One way of doing this is to use an -:py:class:`contextlib.ExitStack` instance and manage the resource explicitly:: - - from contextlib import ExitStack - file_manager = ExitStack() - path = file_manager.enter_context( - importlib_resources.path('my.package', 'resource.dat')) - -Now ``path`` will continue to exist until you explicitly call -``file_manager.close()``. What if you want the file to exist until the -process exits, or you can't pass ``file_manager`` around in your code? Use an -:py:mod:`atexit` handler:: - - import atexit - file_manager = ExitStack() - atexit.register(file_manager.close) - path = file_manager.enter_context( - importlib_resources.path('my.package', 'resource.dat')) - -Assuming your Python interpreter exits gracefully, the temporary file will be -cleaned up when Python exits. - - -pkg_resources.resource_stream() -=============================== - -``pkg_resources.resource_stream()`` returns a readable file-like object opened -in binary mode. When you read from the returned file-like object, you get -bytes. E.g.:: - - with pkg_resources.resource_stream('my.package', 'resource.dat') as fp: - my_bytes = fp.read() - -The equivalent code in ``importlib_resources`` is pretty straightforward:: - - with importlib_resources.open_binary('my.package', 'resource.dat') as fp: - my_bytes = fp.read() - - -pkg_resources.resource_string() -=============================== - -In Python 2, ``pkg_resources.resource_string()`` returns the contents of a -resource as a ``str``. In Python 3, this function is a misnomer; it actually -returns the contents of the named resource as ``bytes``. That's why the -following example is often written for clarity as:: - - from pkg_resources import resource_string as resource_bytes - contents = resource_bytes('my.package', 'resource.dat') - -This can be easily rewritten like so:: - - contents = importlib_resources.read_binary('my.package', 'resource.dat') - - -pkg_resources.resource_listdir() -================================ - -This function lists the entries in the package, both files and directories, -but it does not recurse into subdirectories, e.g.:: - - for entry in pkg_resources.listdir('my.package', 'subpackage'): - print(entry) - -This is easily rewritten using the following idiom:: - - for entry in importlib_resources.contents('my.package.subpackage'): - print(entry) - -Note: - -* ``pkg_resources`` does not require ``subpackage`` to be a Python package, - but ``importlib_resources`` does. -* ``importlib_resources.contents()`` returns an iterator, not a concrete - sequence. -* The order in which the elements are returned is undefined. -* ``importlib_resources.contents()`` returns *all* the entries in the - subpackage, i.e. both resources (files) and non-resources (directories). As - with ``pkg_resources.listdir()`` it does not recurse. - - -pkg_resources.resource_isdir() -============================== - -You can ask ``pkg_resources`` to tell you whether a particular resource inside -a package is a directory or not:: - - if pkg_resources.resource_isdir('my.package', 'resource'): - print('A directory') - -Because ``importlib_resources`` explicitly does not define directories as -resources, there's no direct equivalent. However, you can ask whether a -particular resource exists inside a package, and since directories are not -resources you can infer whether the resource is a directory or a file. Here -is a way to do that:: - - from importlib_resources import contents, is_resource - if 'resource' in contents('my.package') and \ - not is_resource('my.package', 'resource'): - print('It must be a directory') - -The reason you have to do it this way and not just call -``not is_resource('my.package', 'resource')`` is because this conditional will -also return False when ``resource`` is not an entry in ``my.package``. - - -.. _`basic resource access`: http://setuptools.readthedocs.io/en/latest/pkg_resources.html#basic-resource-access diff --git a/importlib_resources/docs/using.rst b/importlib_resources/docs/using.rst deleted file mode 100644 index fa744641..00000000 --- a/importlib_resources/docs/using.rst +++ /dev/null @@ -1,175 +0,0 @@ -.. _using: - -=========================== - Using importlib_resources -=========================== - -``importlib_resources`` is a library that leverages Python's import system to -provide access to *resources* within *packages*. Given that this library is -built on top of the import system, it is highly efficient and easy to use. -This library's philosophy is that, if you can import a package, you can access -resources within that package. Resources can be opened or read, in either -binary or text mode. - -What exactly do we mean by "a resource"? It's easiest to think about the -metaphor of files and directories on the file system, though it's important to -keep in mind that this is just a metaphor. Resources and packages **do not** -have to exist as physical files and directories on the file system. - -If you have a file system layout such as:: - - data/ - __init__.py - one/ - __init__.py - resource1.txt - two/ - __init__.py - resource2.txt - -then the directories are ``data``, ``data/one``, and ``data/two``. Each of -these are also Python packages by virtue of the fact that they all contain -``__init__.py`` files [#fn1]_. That means that in Python, all of these import -statements work:: - - import data - import data.one - from data import two - -Each import statement gives you a Python *module* corresponding to the -``__init__.py`` file in each of the respective directories. These modules are -packages since packages are just special module instances that have an -additional attribute, namely a ``__path__`` [#fn2]_. - -In this analogy then, resources are just files within a package directory, so -``data/one/resource1.txt`` and ``data/two/resource2.txt`` are both resources, -as are the ``__init__.py`` files in all the directories. However the package -directories themselves are *not* resources; anything that contains other -things (i.e. directories) are not themselves resources. - -Resources are always accessed relative to the package that they live in. You -cannot access a resource within a subdirectory inside a package. This means -that ``resource1.txt`` is a resource within the ``data.one`` package, but -neither ``resource2.txt`` nor ``two/resource2.txt`` are resources within the -``data`` package. If a directory isn't a package, it can't be imported and -thus can't contain resources. - -Even when this hierarchical structure isn't represented by physical files and -directories, the model still holds. So zip files can contain packages and -resources, as could databases or other storage medium. In fact, while -``importlib_resources`` supports physical file systems and zip files by -default, anything that can be loaded with a Python import system `loader`_ can -provide resources, as long as the loader implements the `ResourceReader`_ -abstract base class. - - -Example -======= - -Let's say you are writing an email parsing library and in your test suite you -have a sample email message in a file called ``message.eml``. You would like -to access the contents of this file for your tests, so you put this in your -project under the ``email/tests/data/message.eml`` path. Let's say your unit -tests live in ``email/tests/test_email.py``. - -Your test could read the data file by doing something like:: - - data_dir = os.path.join(os.path.dirname(__file__), 'tests', 'data') - data_path = os.path.join(data_dir, 'message.eml') - with open(data_path, encoding='utf-8') as fp: - eml = fp.read() - -But there's a problem with this! The use of ``__file__`` doesn't work if your -package lives inside a zip file, since in that case this code does not live on -the file system. - -You could use the `pkg_resources API`_ like so:: - - # In Python 3, resource_string() actually returns bytes! - from pkg_resources import resource_string as resource_bytes - eml = resource_bytes('email.tests.data', 'message.eml').decode('utf-8') - -This requires you to make Python packages of both ``email/tests`` and -``email/tests/data``, by placing an empty ``__init__.py`` files in each of -those directories. - -**This is a requirement for importlib_resources too!** - -The problem with the ``pkg_resources`` approach is that, depending on the -structure of your package, ``pkg_resources`` can be very inefficient even to -just import. ``pkg_resources`` is a sort of grab-bag of APIs and -functionalities, and to support all of this, it sometimes has to do a ton of -work at import time, e.g. to scan every package on your ``sys.path``. This -can have a serious negative impact on things like command line startup time -for Python implement commands. - -``importlib_resources`` solves this by being built entirely on the back of the -stdlib :py:mod:`importlib`. By taking advantage of all the efficiencies in -Python's import system, and the fact that it's built into Python, using -``importlib_resources`` can be much more performant. The equivalent code -using ``importlib_resources`` would look like:: - - from importlib_resources import read_text - # Reads contents with UTF-8 encoding and returns str. - eml = read_text('email.tests.data', 'message.eml') - - -Packages or package names -========================= - -All of the ``importlib_resources`` APIs take a *package* as their first -parameter, but this can either be a package name (as a ``str``) or an actual -module object, though the module *must* be a package [#fn3]_. If a string is -passed in, it must name an importable Python package, and this is first -imported. Thus the above example could also be written as:: - - import email.tests.data - eml = read_text(email.tests.data, 'message.eml') - - -File system or zip file -======================= - -In general you never have to worry whether your package is on the file system -or in a zip file, as the ``importlib_resources`` APIs hide those details from -you. Sometimes though, you need a path to an actual file on the file system. -For example, some SSL APIs require a certificate file to be specified by a -real file system path, and C's ``dlopen()`` function also requires a real file -system path. - -To support this, ``importlib_resources`` provides an API that will extract the -resource from a zip file to a temporary file, and return the file system path -to this temporary file as a :py:class:`pathlib.Path` object. In order to -properly clean up this temporary file, what's actually returned is a context -manager that you can use in a ``with``-statement:: - - from importlib_resources import path - with path(email.tests.data, 'message.eml') as eml: - third_party_api_requiring_file_system_path(eml) - -You can use all the standard :py:mod:`contextlib` APIs to manage this context -manager. - - -.. rubric:: Footnotes - -.. [#fn1] We're ignoring `PEP 420 - `_ style namespace - packages, since ``importlib_resources`` does not support resources - within namespace packages. Also, the example assumes that the - parent directory containing ``data/`` is on ``sys.path``. - -.. [#fn2] As of `PEP 451 `_ this - information is also available on the module's - ``__spec__.submodule_search_locations`` attribute, which will not be - ``None`` for packages. - -.. [#fn3] Specifically, this means that in Python 2, the module object must - have an ``__path__`` attribute, while in Python 3, the module's - ``__spec__.submodule_search_locations`` must not be ``None``. - Otherwise a ``TypeError`` is raised. - - -.. _`pkg_resources API`: http://setuptools.readthedocs.io/en/latest/pkg_resources.html#basic-resource-access -.. _`loader`: https://docs.python.org/3/reference/import.html#finders-and-loaders -.. _`ResourceReader`: https://docs.python.org/3.7/library/importlib.html#importlib.abc.ResourceReader diff --git a/importlib_resources/version.txt b/importlib_resources/version.txt deleted file mode 100644 index d3827e75..00000000 --- a/importlib_resources/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.0 diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 487e88fd..00000000 --- a/pyproject.toml +++ /dev/null @@ -1,2 +0,0 @@ -[build-system] -requires = ["setuptools>=30.3", "wheel"] diff --git a/release.sh b/release.sh deleted file mode 100755 index 26346cc7..00000000 --- a/release.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -source .tox/release/bin/activate -pip wheel -w wheels . -rm -rf artifacts -mkdir artifacts -mv wheels/importlib_resources*.whl artifacts/ -python setup.py sdist -mv dist/importlib_resources*.tar.gz artifacts/ -rm -rf wheels dist diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index cd191658..00000000 --- a/setup.cfg +++ /dev/null @@ -1,47 +0,0 @@ -[metadata] -name = importlib_resources -version = attr: version.__version__ -author = Barry Warsaw -author_email = barry@python.org -url = http://importlib-resources.readthedocs.io/ -description = Read resources from Python packages -long_description = file: README.rst -license = Apache Software License -classifiers = - Development Status :: 4 - Beta - Intended Audience :: Developers - License :: OSI Approved :: Apache Software License - Topic :: Software Development :: Libraries - Programming Language :: Python :: 3 - -[options] -python_requires = >=2.7,!=3.0,!=3.1,!=3.2,!=3.3 -setup_requires = - pathlib2; python_version < '3' - typing; python_version < '3.5' -install_requires = - setuptools - wheel - pathlib2; python_version < '3' - typing; python_version < '3.5' -packages = find: - -[options.package_data] -* = *.zip, *.file, *.txt, *.toml -importlib_resources = - docs/* - docs/_static/* -importlib_resources.tests.data03 = - namespace/* - -[mypy] -ignore_missing_imports = True -# XXX We really should use the default `True` value here, but it causes too -# many warnings, so for now just disable it. E.g. a package's __spec__ is -# defined as Optional[ModuleSpec] so we can't just blindly pull attributes off -# of that attribute. The real fix is to add conditionals or asserts proving -# that package.__spec__ is not None. -strict_optional = False - -[wheel] -universal=1 diff --git a/setup.py b/setup.py deleted file mode 100644 index 8bf1ba93..00000000 --- a/setup.py +++ /dev/null @@ -1,2 +0,0 @@ -from setuptools import setup -setup() diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 32ec1da5..00000000 --- a/tox.ini +++ /dev/null @@ -1,72 +0,0 @@ -[tox] -# Don't cover Python 3.7 since this is just a shim for that version. Do at -# least make sure we don't regress! -envlist = {py27,py34,py35,py36}-{nocov,cov,diffcov},py37-nocov,qa,docs -skip_missing_interpreters = True - - -[testenv] -commands = - nocov: python -m unittest discover - cov,diffcov: python -m coverage run {[coverage]rc} -m unittest discover {posargs} - cov,diffcov: python -m coverage combine {[coverage]rc} - cov: python -m coverage html {[coverage]rc} - cov: python -m coverage xml {[coverage]rc} - cov: python -m coverage report -m {[coverage]rc} --fail-under=100 - diffcov: python -m coverage xml {[coverage]rc} - diffcov: diff-cover coverage.xml --html-report diffcov.html - diffcov: diff-cover coverage.xml --fail-under=100 -usedevelop = True -passenv = - PYTHON* - LANG* - LC_* - OMIT -deps = - cov,diffcov: coverage>=4.5 - diffcov: diff_cover -setenv = - cov: COVERAGE_PROCESS_START={[coverage]rcfile} - cov: COVERAGE_OPTIONS="-p" - cov: COVERAGE_FILE={toxinidir}/.coverage - py27: OMIT=3 - py34,py35,py36,py37: OMIT=2 - - -[testenv:qa] -basepython = python3 -commands = - python -m flake8 importlib_resources - mypy importlib_resources -deps = - mypy - flake8 - - -[testenv:docs] -basepython = python3 -commands = - sphinx-build importlib_resources/docs build/sphinx/html -deps: - sphinx - docutils==0.12 - - -[coverage] -rcfile = {toxinidir}/coverage.ini -rc = --rcfile={[coverage]rcfile} - - -[flake8] -hang-closing = True -jobs = 1 -max-line-length = 79 -exclude = - # Exclude the entire top-level __init__.py file since its only purpose is - # to expose the version string and to handle Python 2/3 compatibility. - importlib_resources/__init__.py - -[testenv:release] -basepython = python3 -whitelist_externals = ./release.sh -commands = ./release.sh diff --git a/update-zips.py b/update-zips.py deleted file mode 100755 index 5777ed57..00000000 --- a/update-zips.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -"""Remake the ziptestdata.zip file. - -Run this to rebuild the importlib_resources/tests/data/ziptestdata.zip file, -e.g. if you want to add a new file to the zip. - -This will replace the file with the new build, but it won't commit anything to -git. -""" - -import os -from zipfile import ZipFile - -RELPATH = 'importlib_resources/tests/data{suffix}' -BASEPATH = 'ziptestdata' -ZF_BASE = 'importlib_resources/tests/zipdata{suffix}/ziptestdata.zip' - -for suffix in ('01', '02'): - zfpath = ZF_BASE.format(suffix=suffix) - with ZipFile(zfpath, 'w') as zf: - relpath = RELPATH.format(suffix=suffix) - for dirpath, dirnames, filenames in os.walk(relpath): - for filename in filenames: - src = os.path.join(dirpath, filename) - if '__pycache__' in src: - continue - if src == zfpath: - continue - commonpath = os.path.commonpath((relpath, dirpath)) - dst = os.path.join( - BASEPATH, dirpath[len(commonpath)+1:], filename) - print(src, '->', dst) - zf.write(src, dst) diff --git a/version.py b/version.py deleted file mode 100644 index 9ef97bd4..00000000 --- a/version.py +++ /dev/null @@ -1,9 +0,0 @@ -# This is a hack until setuptools supports reading a version out of a file -# without importing. Once that bug is fix-released, you can remove this file -# and update the setup.cfg file. -# -# https://github.com/pypa/setuptools/issues/1358 - -# Yes, it's horrible to let reference counting reclaim the open file object, -# but oh well, this will exit soon. -__version__ = open('importlib_resources/version.txt').read().strip() From 1d66835c1b3701e834e136fa9bb12348153a6416 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 1 Mar 2020 11:50:49 -0600 Subject: [PATCH 02/10] Update to match implementation as found in CPython 3.7.0 --- Lib/importlib/abc.py | 400 ++++++++++++++-- Lib/importlib/resources.py | 213 +++++---- Lib/tests/test_importlib/__init__.py | 5 + Lib/tests/test_importlib/test_open.py | 11 +- Lib/tests/test_importlib/test_path.py | 6 +- Lib/tests/test_importlib/test_read.py | 7 +- Lib/tests/test_importlib/test_resource.py | 54 +-- Lib/tests/test_importlib/util.py | 451 ++++++++++++++++-- .../test_importlib/zipdata01/ziptestdata.zip | Bin 876 -> 876 bytes .../test_importlib/zipdata02/ziptestdata.zip | Bin 698 -> 698 bytes 10 files changed, 925 insertions(+), 222 deletions(-) diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index f49e8c70..dbdd5bf6 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -1,58 +1,388 @@ -from __future__ import absolute_import +"""Abstract base classes related to import.""" +from . import _bootstrap +from . import _bootstrap_external +from . import machinery +try: + import _frozen_importlib +except ImportError as exc: + if exc.name != '_frozen_importlib': + raise + _frozen_importlib = None +try: + import _frozen_importlib_external +except ImportError as exc: + _frozen_importlib_external = _bootstrap_external +import abc +import warnings -from ._compat import ABC, FileNotFoundError -from abc import abstractmethod -# We use mypy's comment syntax here since this file must be compatible with -# both Python 2 and 3. -try: - from typing import BinaryIO, Iterable, Text # noqa: F401 -except ImportError: - # Python 2 - pass +def _register(abstract_cls, *classes): + for cls in classes: + abstract_cls.register(cls) + if _frozen_importlib is not None: + try: + frozen_cls = getattr(_frozen_importlib, cls.__name__) + except AttributeError: + frozen_cls = getattr(_frozen_importlib_external, cls.__name__) + abstract_cls.register(frozen_cls) + + +class Finder(metaclass=abc.ABCMeta): + + """Legacy abstract base class for import finders. + + It may be subclassed for compatibility with legacy third party + reimplementations of the import system. Otherwise, finder + implementations should derive from the more specific MetaPathFinder + or PathEntryFinder ABCs. + + Deprecated since Python 3.3 + """ + + @abc.abstractmethod + def find_module(self, fullname, path=None): + """An abstract method that should find a module. + The fullname is a str and the optional path is a str or None. + Returns a Loader object or None. + """ + + +class MetaPathFinder(Finder): + + """Abstract base class for import finders on sys.meta_path.""" + + # We don't define find_spec() here since that would break + # hasattr checks we do to support backward compatibility. + + def find_module(self, fullname, path): + """Return a loader for the module. + + If no module is found, return None. The fullname is a str and + the path is a list of strings or None. + + This method is deprecated since Python 3.4 in favor of + finder.find_spec(). If find_spec() exists then backwards-compatible + functionality is provided for this method. + + """ + warnings.warn("MetaPathFinder.find_module() is deprecated since Python " + "3.4 in favor of MetaPathFinder.find_spec()" + "(available since 3.4)", + DeprecationWarning, + stacklevel=2) + if not hasattr(self, 'find_spec'): + return None + found = self.find_spec(fullname, path) + return found.loader if found is not None else None + + def invalidate_caches(self): + """An optional method for clearing the finder's cache, if any. + This method is used by importlib.invalidate_caches(). + """ + +_register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter, + machinery.PathFinder, machinery.WindowsRegistryFinder) + + +class PathEntryFinder(Finder): + + """Abstract base class for path entry finders used by PathFinder.""" + + # We don't define find_spec() here since that would break + # hasattr checks we do to support backward compatibility. + + def find_loader(self, fullname): + """Return (loader, namespace portion) for the path entry. + + The fullname is a str. The namespace portion is a sequence of + path entries contributing to part of a namespace package. The + sequence may be empty. If loader is not None, the portion will + be ignored. + + The portion will be discarded if another path entry finder + locates the module as a normal module or package. + + This method is deprecated since Python 3.4 in favor of + finder.find_spec(). If find_spec() is provided than backwards-compatible + functionality is provided. + """ + warnings.warn("PathEntryFinder.find_loader() is deprecated since Python " + "3.4 in favor of PathEntryFinder.find_spec() " + "(available since 3.4)", + DeprecationWarning, + stacklevel=2) + if not hasattr(self, 'find_spec'): + return None, [] + found = self.find_spec(fullname) + if found is not None: + if not found.submodule_search_locations: + portions = [] + else: + portions = found.submodule_search_locations + return found.loader, portions + else: + return None, [] + + find_module = _bootstrap_external._find_module_shim + + def invalidate_caches(self): + """An optional method for clearing the finder's cache, if any. + This method is used by PathFinder.invalidate_caches(). + """ + +_register(PathEntryFinder, machinery.FileFinder) + + +class Loader(metaclass=abc.ABCMeta): + + """Abstract base class for import loaders.""" + + def create_module(self, spec): + """Return a module to initialize and into which to load. + + This method should raise ImportError if anything prevents it + from creating a new module. It may return None to indicate + that the spec should create the new module. + """ + # By default, defer to default semantics for the new module. + return None + + # We don't define exec_module() here since that would break + # hasattr checks we do to support backward compatibility. + + def load_module(self, fullname): + """Return the loaded module. + + The module must be added to sys.modules and have import-related + attributes set properly. The fullname is a str. + + ImportError is raised on failure. + + This method is deprecated in favor of loader.exec_module(). If + exec_module() exists then it is used to provide a backwards-compatible + functionality for this method. + + """ + if not hasattr(self, 'exec_module'): + raise ImportError + return _bootstrap._load_module_shim(self, fullname) + + def module_repr(self, module): + """Return a module's repr. + + Used by the module type when the method does not raise + NotImplementedError. + + This method is deprecated. + + """ + # The exception will cause ModuleType.__repr__ to ignore this method. + raise NotImplementedError + + +class ResourceLoader(Loader): + + """Abstract base class for loaders which can return data from their + back-end storage. + + This ABC represents one of the optional protocols specified by PEP 302. + + """ + + @abc.abstractmethod + def get_data(self, path): + """Abstract method which when implemented should return the bytes for + the specified path. The path must be a str.""" + raise OSError -class ResourceReader(ABC): - """Abstract base class for loaders to provide resource reading support.""" +class InspectLoader(Loader): - @abstractmethod + """Abstract base class for loaders which support inspection about the + modules they can load. + + This ABC represents one of the optional protocols specified by PEP 302. + + """ + + def is_package(self, fullname): + """Optional method which when implemented should return whether the + module is a package. The fullname is a str. Returns a bool. + + Raises ImportError if the module cannot be found. + """ + raise ImportError + + def get_code(self, fullname): + """Method which returns the code object for the module. + + The fullname is a str. Returns a types.CodeType if possible, else + returns None if a code object does not make sense + (e.g. built-in module). Raises ImportError if the module cannot be + found. + """ + source = self.get_source(fullname) + if source is None: + return None + return self.source_to_code(source) + + @abc.abstractmethod + def get_source(self, fullname): + """Abstract method which should return the source code for the + module. The fullname is a str. Returns a str. + + Raises ImportError if the module cannot be found. + """ + raise ImportError + + @staticmethod + def source_to_code(data, path=''): + """Compile 'data' into a code object. + + The 'data' argument can be anything that compile() can handle. The'path' + argument should be where the data was retrieved (when applicable).""" + return compile(data, path, 'exec', dont_inherit=True) + + exec_module = _bootstrap_external._LoaderBasics.exec_module + load_module = _bootstrap_external._LoaderBasics.load_module + +_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter) + + +class ExecutionLoader(InspectLoader): + + """Abstract base class for loaders that wish to support the execution of + modules as scripts. + + This ABC represents one of the optional protocols specified in PEP 302. + + """ + + @abc.abstractmethod + def get_filename(self, fullname): + """Abstract method which should return the value that __file__ is to be + set to. + + Raises ImportError if the module cannot be found. + """ + raise ImportError + + def get_code(self, fullname): + """Method to return the code object for fullname. + + Should return None if not applicable (e.g. built-in module). + Raise ImportError if the module cannot be found. + """ + source = self.get_source(fullname) + if source is None: + return None + try: + path = self.get_filename(fullname) + except ImportError: + return self.source_to_code(source) + else: + return self.source_to_code(source, path) + +_register(ExecutionLoader, machinery.ExtensionFileLoader) + + +class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader): + + """Abstract base class partially implementing the ResourceLoader and + ExecutionLoader ABCs.""" + +_register(FileLoader, machinery.SourceFileLoader, + machinery.SourcelessFileLoader) + + +class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLoader): + + """Abstract base class for loading source code (and optionally any + corresponding bytecode). + + To support loading from source code, the abstractmethods inherited from + ResourceLoader and ExecutionLoader need to be implemented. To also support + loading from bytecode, the optional methods specified directly by this ABC + is required. + + Inherited abstractmethods not implemented in this ABC: + + * ResourceLoader.get_data + * ExecutionLoader.get_filename + + """ + + def path_mtime(self, path): + """Return the (int) modification time for the path (str).""" + if self.path_stats.__func__ is SourceLoader.path_stats: + raise OSError + return int(self.path_stats(path)['mtime']) + + def path_stats(self, path): + """Return a metadata dict for the source pointed to by the path (str). + Possible keys: + - 'mtime' (mandatory) is the numeric timestamp of last source + code modification; + - 'size' (optional) is the size in bytes of the source code. + """ + if self.path_mtime.__func__ is SourceLoader.path_mtime: + raise OSError + return {'mtime': self.path_mtime(path)} + + def set_data(self, path, data): + """Write the bytes to the path (if possible). + + Accepts a str path and data as bytes. + + Any needed intermediary directories are to be created. If for some + reason the file cannot be written because of permissions, fail + silently. + """ + +_register(SourceLoader, machinery.SourceFileLoader) + + +class ResourceReader(metaclass=abc.ABCMeta): + + """Abstract base class to provide resource-reading support. + + Loaders that support resource reading are expected to implement + the ``get_resource_reader(fullname)`` method and have it either return None + or an object compatible with this ABC. + """ + + @abc.abstractmethod def open_resource(self, resource): - # type: (Text) -> BinaryIO """Return an opened, file-like object for binary reading. - The 'resource' argument is expected to represent only a file name. + The 'resource' argument is expected to represent only a file name + and thus not contain any subdirectory components. + If the resource cannot be found, FileNotFoundError is raised. """ - # This deliberately raises FileNotFoundError instead of - # NotImplementedError so that if this method is accidentally called, - # it'll still do the right thing. raise FileNotFoundError - @abstractmethod + @abc.abstractmethod def resource_path(self, resource): - # type: (Text) -> Text """Return the file system path to the specified resource. - The 'resource' argument is expected to represent only a file name. + The 'resource' argument is expected to represent only a file name + and thus not contain any subdirectory components. + If the resource does not exist on the file system, raise FileNotFoundError. """ - # This deliberately raises FileNotFoundError instead of - # NotImplementedError so that if this method is accidentally called, - # it'll still do the right thing. raise FileNotFoundError - @abstractmethod - def is_resource(self, path): - # type: (Text) -> bool - """Return True if the named 'path' is a resource. - - Files are resources, directories are not. - """ + @abc.abstractmethod + def is_resource(self, name): + """Return True if the named 'name' is consider a resource.""" raise FileNotFoundError - @abstractmethod + @abc.abstractmethod def contents(self): - # type: () -> Iterable[str] - """Return an iterable of entries in `package`.""" - raise FileNotFoundError + """Return an iterable of strings over the contents of the package.""" + return [] + + +_register(ResourceReader, machinery.SourceFileLoader) diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index 00781bd9..cbefdd54 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -1,5 +1,4 @@ import os -import sys import tempfile from . import abc as resources_abc @@ -12,14 +11,24 @@ from typing import Iterable, Iterator, Optional, Set, Union # noqa: F401 from typing import cast from typing.io import BinaryIO, TextIO -from zipfile import ZipFile +from zipimport import ZipImportError -Package = Union[ModuleType, str] -if sys.version_info >= (3, 6): - Resource = Union[str, os.PathLike] # pragma: <=35 -else: - Resource = str # pragma: >=36 +__all__ = [ + 'Package', + 'Resource', + 'contents', + 'is_resource', + 'open_binary', + 'open_text', + 'path', + 'read_binary', + 'read_text', + ] + + +Package = Union[str, ModuleType] +Resource = Union[str, os.PathLike] def _get_package(package) -> ModuleType: @@ -47,8 +56,7 @@ def _normalize_path(path) -> str: If the resulting string contains path separators, an exception is raised. """ - str_path = str(path) - parent, file_name = os.path.split(str_path) + parent, file_name = os.path.split(path) if parent: raise ValueError('{!r} must be only a file name'.format(path)) else: @@ -63,10 +71,15 @@ def _get_resource_reader( # zipimport.zipimporter does not support weak references, resulting in a # TypeError. That seems terrible. spec = package.__spec__ - reader = getattr(spec.loader, 'get_resource_reader', None) - if reader is None: - return None - return cast(resources_abc.ResourceReader, reader(spec.name)) + if hasattr(spec.loader, 'get_resource_reader'): + return cast(resources_abc.ResourceReader, + spec.loader.get_resource_reader(spec.name)) + return None + + +def _check_location(package): + if package.__spec__.origin is None or not package.__spec__.has_location: + raise FileNotFoundError(f'Package has no location {package!r}') def open_binary(package: Package, resource: Resource) -> BinaryIO: @@ -76,8 +89,7 @@ def open_binary(package: Package, resource: Resource) -> BinaryIO: reader = _get_resource_reader(package) if reader is not None: return reader.open_resource(resource) - # Using pathlib doesn't work well here due to the lack of 'strict' - # argument for pathlib.Path.resolve() prior to Python 3.6. + _check_location(package) absolute_package_path = os.path.abspath(package.__spec__.origin) package_path = os.path.dirname(absolute_package_path) full_path = os.path.join(package_path, resource) @@ -111,8 +123,7 @@ def open_text(package: Package, reader = _get_resource_reader(package) if reader is not None: return TextIOWrapper(reader.open_resource(resource), encoding, errors) - # Using pathlib doesn't work well here due to the lack of 'strict' - # argument for pathlib.Path.resolve() prior to Python 3.6. + _check_location(package) absolute_package_path = os.path.abspath(package.__spec__.origin) package_path = os.path.dirname(absolute_package_path) full_path = os.path.join(package_path, resource) @@ -178,6 +189,8 @@ def path(package: Package, resource: Resource) -> Iterator[Path]: return except FileNotFoundError: pass + else: + _check_location(package) # Fall-through for both the lack of resource_path() *and* if # resource_path() raises FileNotFoundError. package_directory = Path(package.__spec__.origin).parent @@ -203,7 +216,7 @@ def path(package: Package, resource: Resource) -> Iterator[Path]: def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. + """True if 'name' is a resource inside 'package'. Directories are *not* resources. """ @@ -222,42 +235,11 @@ def is_resource(package: Package, name: str) -> bool: # contents doesn't necessarily mean it's a resource. Directories are not # resources, so let's try to find out if it's a directory or not. path = Path(package.__spec__.origin).parent / name - if path.is_file(): - return True - if path.is_dir(): - return False - # If it's not a file and it's not a directory, what is it? Well, this - # means the file doesn't exist on the file system, so it probably lives - # inside a zip file. We have to crack open the zip, look at its table of - # contents, and make sure that this entry doesn't have sub-entries. - archive_path = package.__spec__.loader.archive # type: ignore - package_directory = Path(package.__spec__.origin).parent - with ZipFile(archive_path) as zf: - toc = zf.namelist() - relpath = package_directory.relative_to(archive_path) - candidate_path = relpath / name - for entry in toc: # pragma: nobranch - try: - relative_to_candidate = Path(entry).relative_to(candidate_path) - except ValueError: - # The two paths aren't relative to each other so we can ignore it. - continue - # Since directories aren't explicitly listed in the zip file, we must - # infer their 'directory-ness' by looking at the number of path - # components in the path relative to the package resource we're - # looking up. If there are zero additional parts, it's a file, i.e. a - # resource. If there are more than zero it's a directory, i.e. not a - # resource. It has to be one of these two cases. - return len(relative_to_candidate.parts) == 0 - # I think it's impossible to get here. It would mean that we are looking - # for a resource in a zip file, there's an entry matching it in the return - # value of contents(), but we never actually found it in the zip's table of - # contents. - raise AssertionError('Impossible situation') + return path.is_file() def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. + """Return an iterable of entries in 'package'. Note that not all entries are resources. Specifically, directories are not considered resources. Use `is_resource()` on each entry returned here @@ -268,45 +250,94 @@ def contents(package: Package) -> Iterable[str]: if reader is not None: return reader.contents() # Is the package a namespace package? By definition, namespace packages - # cannot have resources. - if (package.__spec__.origin == 'namespace' and - not package.__spec__.has_location): + # cannot have resources. We could use _check_location() and catch the + # exception, but that's extra work, so just inline the check. + elif package.__spec__.origin is None or not package.__spec__.has_location: return () - package_directory = Path(package.__spec__.origin).parent - try: - return os.listdir(str(package_directory)) - except (NotADirectoryError, FileNotFoundError): - # The package is probably in a zip file. - archive_path = getattr(package.__spec__.loader, 'archive', None) - if archive_path is None: - raise - relpath = package_directory.relative_to(archive_path) - with ZipFile(archive_path) as zf: - toc = zf.namelist() - subdirs_seen = set() # type: Set - subdirs_returned = [] - for filename in toc: - path = Path(filename) - # Strip off any path component parts that are in common with the - # package directory, relative to the zip archive's file system - # path. This gives us all the parts that live under the named - # package inside the zip file. If the length of these subparts is - # exactly 1, then it is situated inside the package. The resulting - # length will be 0 if it's above the package, and it will be - # greater than 1 if it lives in a subdirectory of the package - # directory. - # - # However, since directories themselves don't appear in the zip - # archive as a separate entry, we need to return the first path - # component for any case that has > 1 subparts -- but only once! - if path.parts[:len(relpath.parts)] != relpath.parts: + else: + package_directory = Path(package.__spec__.origin).parent + return os.listdir(package_directory) + + +# Private implementation of ResourceReader and get_resource_reader() called +# from zipimport.c. Don't use these directly! We're implementing these in +# Python because 1) it's easier, 2) zipimport may get rewritten in Python +# itself at some point, so doing this all in C would difficult and a waste of +# effort. + +class _ZipImportResourceReader(resources_abc.ResourceReader): + """Private class used to support ZipImport.get_resource_reader(). + + This class is allowed to reference all the innards and private parts of + the zipimporter. + """ + + def __init__(self, zipimporter, fullname): + self.zipimporter = zipimporter + self.fullname = fullname + + def open_resource(self, resource): + fullname_as_path = self.fullname.replace('.', '/') + path = f'{fullname_as_path}/{resource}' + try: + return BytesIO(self.zipimporter.get_data(path)) + except OSError: + raise FileNotFoundError(path) + + def resource_path(self, resource): + # All resources are in the zip file, so there is no path to the file. + # Raising FileNotFoundError tells the higher level API to extract the + # binary data and create a temporary file. + raise FileNotFoundError + + def is_resource(self, name): + # Maybe we could do better, but if we can get the data, it's a + # resource. Otherwise it isn't. + fullname_as_path = self.fullname.replace('.', '/') + path = f'{fullname_as_path}/{name}' + try: + self.zipimporter.get_data(path) + except OSError: + return False + return True + + def contents(self): + # This is a bit convoluted, because fullname will be a module path, + # but _files is a list of file names relative to the top of the + # archive's namespace. We want to compare file paths to find all the + # names of things inside the module represented by fullname. So we + # turn the module path of fullname into a file path relative to the + # top of the archive, and then we iterate through _files looking for + # names inside that "directory". + fullname_path = Path(self.zipimporter.get_filename(self.fullname)) + relative_path = fullname_path.relative_to(self.zipimporter.archive) + # Don't forget that fullname names a package, so its path will include + # __init__.py, which we want to ignore. + assert relative_path.name == '__init__.py' + package_path = relative_path.parent + subdirs_seen = set() + for filename in self.zipimporter._files: + try: + relative = Path(filename).relative_to(package_path) + except ValueError: continue - subparts = path.parts[len(relpath.parts):] - if len(subparts) == 1: - subdirs_returned.append(subparts[0]) - elif len(subparts) > 1: # pragma: nobranch - subdir = subparts[0] - if subdir not in subdirs_seen: - subdirs_seen.add(subdir) - subdirs_returned.append(subdir) - return subdirs_returned + # If the path of the file (which is relative to the top of the zip + # namespace), relative to the package given when the resource + # reader was created, has a parent, then it's a name in a + # subdirectory and thus we skip it. + parent_name = relative.parent.name + if len(parent_name) == 0: + yield relative.name + elif parent_name not in subdirs_seen: + subdirs_seen.add(parent_name) + yield parent_name + + +# Called from zipimport.c +def _zipimport_get_resource_reader(zipimporter, fullname): + try: + if not zipimporter.is_package(fullname): + return None + except ZipImportError: + return None + return _ZipImportResourceReader(zipimporter, fullname) diff --git a/Lib/tests/test_importlib/__init__.py b/Lib/tests/test_importlib/__init__.py index e69de29b..4b16ecc3 100644 --- a/Lib/tests/test_importlib/__init__.py +++ b/Lib/tests/test_importlib/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/tests/test_importlib/test_open.py b/Lib/tests/test_importlib/test_open.py index 8a3429f2..fd6e84b7 100644 --- a/Lib/tests/test_importlib/test_open.py +++ b/Lib/tests/test_importlib/test_open.py @@ -1,18 +1,17 @@ import unittest -import importlib_resources as resources +from importlib import resources from . import data01 from . import util -from .._compat import FileNotFoundError -class CommonBinaryTests(util.CommonTests, unittest.TestCase): +class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase): def execute(self, package, path): with resources.open_binary(package, path): pass -class CommonTextTests(util.CommonTests, unittest.TestCase): +class CommonTextTests(util.CommonResourceTests, unittest.TestCase): def execute(self, package, path): with resources.open_text(package, path): pass @@ -20,9 +19,9 @@ def execute(self, package, path): class OpenTests: def test_open_binary(self): - with resources.open_binary(self.data, 'utf-8.file') as fp: + with resources.open_binary(self.data, 'binary.file') as fp: result = fp.read() - self.assertEqual(result, b'Hello, UTF-8 world!\n') + self.assertEqual(result, b'\x00\x01\x02\x03') def test_open_text_default_encoding(self): with resources.open_text(self.data, 'utf-8.file') as fp: diff --git a/Lib/tests/test_importlib/test_path.py b/Lib/tests/test_importlib/test_path.py index 51ba185d..2d3dcda7 100644 --- a/Lib/tests/test_importlib/test_path.py +++ b/Lib/tests/test_importlib/test_path.py @@ -1,19 +1,17 @@ import unittest -import importlib_resources as resources +from importlib import resources from . import data01 from . import util -class CommonTests(util.CommonTests, unittest.TestCase): - +class CommonTests(util.CommonResourceTests, unittest.TestCase): def execute(self, package, path): with resources.path(package, path): pass class PathTests: - def test_reading(self): # Path should be readable. # Test also implicitly verifies the returned object is a pathlib.Path diff --git a/Lib/tests/test_importlib/test_read.py b/Lib/tests/test_importlib/test_read.py index ee94d8ad..ff78d0b2 100644 --- a/Lib/tests/test_importlib/test_read.py +++ b/Lib/tests/test_importlib/test_read.py @@ -1,17 +1,16 @@ import unittest -import importlib_resources as resources +from importlib import import_module, resources from . import data01 from . import util -from importlib import import_module -class CommonBinaryTests(util.CommonTests, unittest.TestCase): +class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase): def execute(self, package, path): resources.read_binary(package, path) -class CommonTextTests(util.CommonTests, unittest.TestCase): +class CommonTextTests(util.CommonResourceTests, unittest.TestCase): def execute(self, package, path): resources.read_text(package, path) diff --git a/Lib/tests/test_importlib/test_resource.py b/Lib/tests/test_importlib/test_resource.py index 8c5a72cb..f88d92d1 100644 --- a/Lib/tests/test_importlib/test_resource.py +++ b/Lib/tests/test_importlib/test_resource.py @@ -1,11 +1,10 @@ import sys import unittest -import importlib_resources as resources from . import data01 from . import zipdata01, zipdata02 from . import util -from importlib import import_module +from importlib import resources, import_module class ResourceTests: @@ -49,7 +48,6 @@ class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): pass -@unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') class ResourceLoaderTests(unittest.TestCase): def test_resource_contents(self): package = util.create_package( @@ -95,7 +93,24 @@ def test_package_has_no_reader_fallback(self): self.assertFalse(resources.is_resource(module, 'A')) -class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): +class ResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase): + ZIP_MODULE = zipdata02 # type: ignore + + def test_unrelated_contents(self): + # https://gitlab.com/python-devs/importlib_resources/issues/44 + # + # Here we have a zip file with two unrelated subpackages. The bug + # reports that getting the contents of a resource returns unrelated + # files. + self.assertEqual( + set(resources.contents('ziptestdata.one')), + {'__init__.py', 'resource1.txt'}) + self.assertEqual( + set(resources.contents('ziptestdata.two')), + {'__init__.py', 'resource2.txt'}) + + +class SubdirectoryResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase): ZIP_MODULE = zipdata01 # type: ignore def test_is_submodule_resource(self): @@ -119,51 +134,32 @@ def test_submodule_contents_by_name(self): {'__init__.py', 'binary.file'}) -class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore - - def test_unrelated_contents(self): - # https://gitlab.com/python-devs/importlib_resources/issues/44 - # - # Here we have a zip file with two unrelated subpackages. The bug - # reports that getting the contents of a resource returns unrelated - # files. - self.assertEqual( - set(resources.contents('ziptestdata.one')), - {'__init__.py', 'resource1.txt'}) - self.assertEqual( - set(resources.contents('ziptestdata.two')), - {'__init__.py', 'resource2.txt'}) - - -@unittest.skipIf(sys.version_info < (3,), 'No namespace packages in Python 2') class NamespaceTest(unittest.TestCase): def test_namespaces_cannot_have_resources(self): - contents = resources.contents( - 'importlib_resources.tests.data03.namespace') + contents = resources.contents('test.test_importlib.data03.namespace') self.assertFalse(list(contents)) # Even though there is a file in the namespace directory, it is not # considered a resource, since namespace packages can't have them. self.assertFalse(resources.is_resource( - 'importlib_resources.tests.data03.namespace', + 'test.test_importlib.data03.namespace', 'resource1.txt')) # We should get an exception if we try to read it or open it. self.assertRaises( FileNotFoundError, resources.open_text, - 'importlib_resources.tests.data03.namespace', 'resource1.txt') + 'test.test_importlib.data03.namespace', 'resource1.txt') self.assertRaises( FileNotFoundError, resources.open_binary, - 'importlib_resources.tests.data03.namespace', 'resource1.txt') + 'test.test_importlib.data03.namespace', 'resource1.txt') self.assertRaises( FileNotFoundError, resources.read_text, - 'importlib_resources.tests.data03.namespace', 'resource1.txt') + 'test.test_importlib.data03.namespace', 'resource1.txt') self.assertRaises( FileNotFoundError, resources.read_binary, - 'importlib_resources.tests.data03.namespace', 'resource1.txt') + 'test.test_importlib.data03.namespace', 'resource1.txt') if __name__ == '__main__': diff --git a/Lib/tests/test_importlib/util.py b/Lib/tests/test_importlib/util.py index 8c26496d..b0badebc 100644 --- a/Lib/tests/test_importlib/util.py +++ b/Lib/tests/test_importlib/util.py @@ -1,44 +1,398 @@ import abc +import builtins +import contextlib +import errno +import functools import importlib +from importlib import machinery, util, invalidate_caches +from importlib.abc import ResourceReader import io +import os +import os.path +from pathlib import Path, PurePath +from test import support +import unittest import sys +import tempfile import types -import unittest from . import data01 from . import zipdata01 -from .._compat import ABC, Path, PurePath, FileNotFoundError -from ..abc import ResourceReader - -try: - from test.support import modules_setup, modules_cleanup -except ImportError: - # Python 2.7. - def modules_setup(): - return sys.modules.copy(), - - def modules_cleanup(oldmodules): - # Encoders/decoders are registered permanently within the internal - # codec cache. If we destroy the corresponding modules their - # globals will be set to None which will trip up the cached functions. - encodings = [(k, v) for k, v in sys.modules.items() - if k.startswith('encodings.')] - sys.modules.clear() - sys.modules.update(encodings) - # XXX: This kind of problem can affect more than just encodings. In - # particular extension modules (such as _ssl) don't cope with reloading - # properly. Really, test modules should be cleaning out the test - # specific modules they know they added (ala test_runpy) rather than - # relying on this function (as test_importhooks and test_pkg do - # currently). Implicitly imported *real* modules should be left alone - # (see issue 10556). - sys.modules.update(oldmodules) - - -try: - from importlib.machinery import ModuleSpec -except ImportError: - ModuleSpec = None # type: ignore + + +BUILTINS = types.SimpleNamespace() +BUILTINS.good_name = None +BUILTINS.bad_name = None +if 'errno' in sys.builtin_module_names: + BUILTINS.good_name = 'errno' +if 'importlib' not in sys.builtin_module_names: + BUILTINS.bad_name = 'importlib' + +EXTENSIONS = types.SimpleNamespace() +EXTENSIONS.path = None +EXTENSIONS.ext = None +EXTENSIONS.filename = None +EXTENSIONS.file_path = None +EXTENSIONS.name = '_testcapi' + +def _extension_details(): + global EXTENSIONS + for path in sys.path: + for ext in machinery.EXTENSION_SUFFIXES: + filename = EXTENSIONS.name + ext + file_path = os.path.join(path, filename) + if os.path.exists(file_path): + EXTENSIONS.path = path + EXTENSIONS.ext = ext + EXTENSIONS.filename = filename + EXTENSIONS.file_path = file_path + return + +_extension_details() + + +def import_importlib(module_name): + """Import a module from importlib both w/ and w/o _frozen_importlib.""" + fresh = ('importlib',) if '.' in module_name else () + frozen = support.import_fresh_module(module_name) + source = support.import_fresh_module(module_name, fresh=fresh, + blocked=('_frozen_importlib', '_frozen_importlib_external')) + return {'Frozen': frozen, 'Source': source} + + +def specialize_class(cls, kind, base=None, **kwargs): + # XXX Support passing in submodule names--load (and cache) them? + # That would clean up the test modules a bit more. + if base is None: + base = unittest.TestCase + elif not isinstance(base, type): + base = base[kind] + name = '{}_{}'.format(kind, cls.__name__) + bases = (cls, base) + specialized = types.new_class(name, bases) + specialized.__module__ = cls.__module__ + specialized._NAME = cls.__name__ + specialized._KIND = kind + for attr, values in kwargs.items(): + value = values[kind] + setattr(specialized, attr, value) + return specialized + + +def split_frozen(cls, base=None, **kwargs): + frozen = specialize_class(cls, 'Frozen', base, **kwargs) + source = specialize_class(cls, 'Source', base, **kwargs) + return frozen, source + + +def test_both(test_class, base=None, **kwargs): + return split_frozen(test_class, base, **kwargs) + + +CASE_INSENSITIVE_FS = True +# Windows is the only OS that is *always* case-insensitive +# (OS X *can* be case-sensitive). +if sys.platform not in ('win32', 'cygwin'): + changed_name = __file__.upper() + if changed_name == __file__: + changed_name = __file__.lower() + if not os.path.exists(changed_name): + CASE_INSENSITIVE_FS = False + +source_importlib = import_importlib('importlib')['Source'] +__import__ = {'Frozen': staticmethod(builtins.__import__), + 'Source': staticmethod(source_importlib.__import__)} + + +def case_insensitive_tests(test): + """Class decorator that nullifies tests requiring a case-insensitive + file system.""" + return unittest.skipIf(not CASE_INSENSITIVE_FS, + "requires a case-insensitive filesystem")(test) + + +def submodule(parent, name, pkg_dir, content=''): + path = os.path.join(pkg_dir, name + '.py') + with open(path, 'w') as subfile: + subfile.write(content) + return '{}.{}'.format(parent, name), path + + +@contextlib.contextmanager +def uncache(*names): + """Uncache a module from sys.modules. + + A basic sanity check is performed to prevent uncaching modules that either + cannot/shouldn't be uncached. + + """ + for name in names: + if name in ('sys', 'marshal', 'imp'): + raise ValueError( + "cannot uncache {0}".format(name)) + try: + del sys.modules[name] + except KeyError: + pass + try: + yield + finally: + for name in names: + try: + del sys.modules[name] + except KeyError: + pass + + +@contextlib.contextmanager +def temp_module(name, content='', *, pkg=False): + conflicts = [n for n in sys.modules if n.partition('.')[0] == name] + with support.temp_cwd(None) as cwd: + with uncache(name, *conflicts): + with support.DirsOnSysPath(cwd): + invalidate_caches() + + location = os.path.join(cwd, name) + if pkg: + modpath = os.path.join(location, '__init__.py') + os.mkdir(name) + else: + modpath = location + '.py' + if content is None: + # Make sure the module file gets created. + content = '' + if content is not None: + # not a namespace package + with open(modpath, 'w') as modfile: + modfile.write(content) + yield location + + +@contextlib.contextmanager +def import_state(**kwargs): + """Context manager to manage the various importers and stored state in the + sys module. + + The 'modules' attribute is not supported as the interpreter state stores a + pointer to the dict that the interpreter uses internally; + reassigning to sys.modules does not have the desired effect. + + """ + originals = {} + try: + for attr, default in (('meta_path', []), ('path', []), + ('path_hooks', []), + ('path_importer_cache', {})): + originals[attr] = getattr(sys, attr) + if attr in kwargs: + new_value = kwargs[attr] + del kwargs[attr] + else: + new_value = default + setattr(sys, attr, new_value) + if len(kwargs): + raise ValueError( + 'unrecognized arguments: {0}'.format(kwargs.keys())) + yield + finally: + for attr, value in originals.items(): + setattr(sys, attr, value) + + +class _ImporterMock: + + """Base class to help with creating importer mocks.""" + + def __init__(self, *names, module_code={}): + self.modules = {} + self.module_code = {} + for name in names: + if not name.endswith('.__init__'): + import_name = name + else: + import_name = name[:-len('.__init__')] + if '.' not in name: + package = None + elif import_name == name: + package = name.rsplit('.', 1)[0] + else: + package = import_name + module = types.ModuleType(import_name) + module.__loader__ = self + module.__file__ = '' + module.__package__ = package + module.attr = name + if import_name != name: + module.__path__ = [''] + self.modules[import_name] = module + if import_name in module_code: + self.module_code[import_name] = module_code[import_name] + + def __getitem__(self, name): + return self.modules[name] + + def __enter__(self): + self._uncache = uncache(*self.modules.keys()) + self._uncache.__enter__() + return self + + def __exit__(self, *exc_info): + self._uncache.__exit__(None, None, None) + + +class mock_modules(_ImporterMock): + + """Importer mock using PEP 302 APIs.""" + + def find_module(self, fullname, path=None): + if fullname not in self.modules: + return None + else: + return self + + def load_module(self, fullname): + if fullname not in self.modules: + raise ImportError + else: + sys.modules[fullname] = self.modules[fullname] + if fullname in self.module_code: + try: + self.module_code[fullname]() + except Exception: + del sys.modules[fullname] + raise + return self.modules[fullname] + + +class mock_spec(_ImporterMock): + + """Importer mock using PEP 451 APIs.""" + + def find_spec(self, fullname, path=None, parent=None): + try: + module = self.modules[fullname] + except KeyError: + return None + spec = util.spec_from_file_location( + fullname, module.__file__, loader=self, + submodule_search_locations=getattr(module, '__path__', None)) + return spec + + def create_module(self, spec): + if spec.name not in self.modules: + raise ImportError + return self.modules[spec.name] + + def exec_module(self, module): + try: + self.module_code[module.__spec__.name]() + except KeyError: + pass + + +def writes_bytecode_files(fxn): + """Decorator to protect sys.dont_write_bytecode from mutation and to skip + tests that require it to be set to False.""" + if sys.dont_write_bytecode: + return lambda *args, **kwargs: None + @functools.wraps(fxn) + def wrapper(*args, **kwargs): + original = sys.dont_write_bytecode + sys.dont_write_bytecode = False + try: + to_return = fxn(*args, **kwargs) + finally: + sys.dont_write_bytecode = original + return to_return + return wrapper + + +def ensure_bytecode_path(bytecode_path): + """Ensure that the __pycache__ directory for PEP 3147 pyc file exists. + + :param bytecode_path: File system path to PEP 3147 pyc file. + """ + try: + os.mkdir(os.path.dirname(bytecode_path)) + except OSError as error: + if error.errno != errno.EEXIST: + raise + + +@contextlib.contextmanager +def create_modules(*names): + """Temporarily create each named module with an attribute (named 'attr') + that contains the name passed into the context manager that caused the + creation of the module. + + All files are created in a temporary directory returned by + tempfile.mkdtemp(). This directory is inserted at the beginning of + sys.path. When the context manager exits all created files (source and + bytecode) are explicitly deleted. + + No magic is performed when creating packages! This means that if you create + a module within a package you must also create the package's __init__ as + well. + + """ + source = 'attr = {0!r}' + created_paths = [] + mapping = {} + state_manager = None + uncache_manager = None + try: + temp_dir = tempfile.mkdtemp() + mapping['.root'] = temp_dir + import_names = set() + for name in names: + if not name.endswith('__init__'): + import_name = name + else: + import_name = name[:-len('.__init__')] + import_names.add(import_name) + if import_name in sys.modules: + del sys.modules[import_name] + name_parts = name.split('.') + file_path = temp_dir + for directory in name_parts[:-1]: + file_path = os.path.join(file_path, directory) + if not os.path.exists(file_path): + os.mkdir(file_path) + created_paths.append(file_path) + file_path = os.path.join(file_path, name_parts[-1] + '.py') + with open(file_path, 'w') as file: + file.write(source.format(name)) + created_paths.append(file_path) + mapping[name] = file_path + uncache_manager = uncache(*import_names) + uncache_manager.__enter__() + state_manager = import_state(path=[temp_dir]) + state_manager.__enter__() + yield mapping + finally: + if state_manager is not None: + state_manager.__exit__(None, None, None) + if uncache_manager is not None: + uncache_manager.__exit__(None, None, None) + support.rmtree(temp_dir) + + +def mock_path_hook(*entries, importer): + """A mock sys.path_hooks entry.""" + def hook(entry): + if entry not in entries: + raise ImportError + return importer + return hook + + +class CASEOKTestBase: + + def caseok_env_changed(self, *, should_exist): + possibilities = b'PYTHONCASEOK', 'PYTHONCASEOK' + if any(x in self.importlib._bootstrap_external._os.environ + for x in possibilities) != should_exist: + self.skipTest('os.environ changes not reflected in _os.environ') def create_package(file, path, is_package=True, contents=()): @@ -81,26 +435,17 @@ def contents(self): # Unforunately importlib.util.module_from_spec() was not introduced until # Python 3.5. module = types.ModuleType(name) - if ModuleSpec is None: - # Python 2. - module.__name__ = name - module.__file__ = 'does-not-exist' - if is_package: - module.__path__ = [] - else: - # Python 3. - loader = Reader() - spec = ModuleSpec( - name, loader, - origin='does-not-exist', - is_package=is_package) - module.__spec__ = spec - module.__loader__ = loader + loader = Reader() + spec = machinery.ModuleSpec( + name, loader, + origin='does-not-exist', + is_package=is_package) + module.__spec__ = spec + module.__loader__ = loader return module -class CommonTests(ABC): - +class CommonResourceTests(abc.ABC): @abc.abstractmethod def execute(self, package, path): raise NotImplementedError @@ -149,7 +494,7 @@ def test_non_package_by_name(self): def test_non_package_by_package(self): # The anchor package cannot be a module. with self.assertRaises(TypeError): - module = sys.modules['importlib_resources.tests.util'] + module = sys.modules['test.test_importlib.util'] self.execute(module, 'utf-8.file') @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') @@ -205,8 +550,8 @@ def tearDownClass(cls): pass def setUp(self): - modules = modules_setup() - self.addCleanup(modules_cleanup, *modules) + modules = support.modules_setup() + self.addCleanup(support.modules_cleanup, *modules) class ZipSetup(ZipSetupBase): diff --git a/Lib/tests/test_importlib/zipdata01/ziptestdata.zip b/Lib/tests/test_importlib/zipdata01/ziptestdata.zip index ddcccfb37368b3d80af8a5f5ebdd04c2a887c8c0..8d8fa97f199cc29f6905404ea05f88926658ee2b 100644 GIT binary patch delta 176 zcmaFE_J)loz?+#xgaHKF$|i0#W&)D0%t0hiJc!J7<83SJ@dhgZ5t9oT6(;*JiUQ>( zr!t~T?qF1$oWvv!lB{NuVKkjQgGmjcN;|-tkx7IZW?zr@=^uw?fp{P+ a1~MNCwltof%+I6>(tCg@07MBfM*#qvNh@*y delta 177 zcmaFE_J)loz?+#xgaHJ)GbUEWGXcrlZXlA!7(_an^LA(Sc!L#yh{*+v3X}a9MS*gY zQyJaBlAF7gY;=HfI+J%WDo##j5(i1vFv&0`PM*o63gWC~lwn*sQCdcmkx7IZW?PT9 laNEvq7NABD76)P&*wT1!Vt~|SO(q_Y<^xOtAWDci3IO^uEBpWe diff --git a/Lib/tests/test_importlib/zipdata02/ziptestdata.zip b/Lib/tests/test_importlib/zipdata02/ziptestdata.zip index 93f4ede5e9573b0c64b0a6f7727c8fa3ed318d12..6f348899a80491ee0d9b095dffebf7e5a0ae8859 100644 GIT binary patch delta 100 zcmdnRx{H-3z?+#xgaHKF$|mv{GXY6QbB?yM9`Cb1|E!$M$S5)SKcnEpmEvF-xrq-X jL5$xXKytDxqcez-W0IM?j8PlJyv*nc5pf4m=}f@@X%-=c delta 100 zcmdnRx{H-3z?+#xgaHJ)GbZvFGXY6QbB^we9`Cb1|E!$M$S5)SKcnEpmEvF-xrq-X jL5$xXKytDxqcez-W0IM?j8PlJyv*nc5pf4m=}f@@S{NZK From c5e9b5cbf1485d2d99954d2d0e0bed2771131456 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 6 Mar 2020 14:49:21 -0500 Subject: [PATCH 03/10] Update to match implementation as found at CPython master@e59334ebc9 --- Lib/importlib/abc.py | 4 +- Lib/importlib/resources.py | 84 -------------------------------------- 2 files changed, 2 insertions(+), 86 deletions(-) diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index dbdd5bf6..b1b5ccce 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -10,7 +10,7 @@ _frozen_importlib = None try: import _frozen_importlib_external -except ImportError as exc: +except ImportError: _frozen_importlib_external = _bootstrap_external import abc import warnings @@ -66,7 +66,7 @@ def find_module(self, fullname, path): """ warnings.warn("MetaPathFinder.find_module() is deprecated since Python " - "3.4 in favor of MetaPathFinder.find_spec()" + "3.4 in favor of MetaPathFinder.find_spec() " "(available since 3.4)", DeprecationWarning, stacklevel=2) diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index cbefdd54..fc3a1c9c 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -257,87 +257,3 @@ def contents(package: Package) -> Iterable[str]: else: package_directory = Path(package.__spec__.origin).parent return os.listdir(package_directory) - - -# Private implementation of ResourceReader and get_resource_reader() called -# from zipimport.c. Don't use these directly! We're implementing these in -# Python because 1) it's easier, 2) zipimport may get rewritten in Python -# itself at some point, so doing this all in C would difficult and a waste of -# effort. - -class _ZipImportResourceReader(resources_abc.ResourceReader): - """Private class used to support ZipImport.get_resource_reader(). - - This class is allowed to reference all the innards and private parts of - the zipimporter. - """ - - def __init__(self, zipimporter, fullname): - self.zipimporter = zipimporter - self.fullname = fullname - - def open_resource(self, resource): - fullname_as_path = self.fullname.replace('.', '/') - path = f'{fullname_as_path}/{resource}' - try: - return BytesIO(self.zipimporter.get_data(path)) - except OSError: - raise FileNotFoundError(path) - - def resource_path(self, resource): - # All resources are in the zip file, so there is no path to the file. - # Raising FileNotFoundError tells the higher level API to extract the - # binary data and create a temporary file. - raise FileNotFoundError - - def is_resource(self, name): - # Maybe we could do better, but if we can get the data, it's a - # resource. Otherwise it isn't. - fullname_as_path = self.fullname.replace('.', '/') - path = f'{fullname_as_path}/{name}' - try: - self.zipimporter.get_data(path) - except OSError: - return False - return True - - def contents(self): - # This is a bit convoluted, because fullname will be a module path, - # but _files is a list of file names relative to the top of the - # archive's namespace. We want to compare file paths to find all the - # names of things inside the module represented by fullname. So we - # turn the module path of fullname into a file path relative to the - # top of the archive, and then we iterate through _files looking for - # names inside that "directory". - fullname_path = Path(self.zipimporter.get_filename(self.fullname)) - relative_path = fullname_path.relative_to(self.zipimporter.archive) - # Don't forget that fullname names a package, so its path will include - # __init__.py, which we want to ignore. - assert relative_path.name == '__init__.py' - package_path = relative_path.parent - subdirs_seen = set() - for filename in self.zipimporter._files: - try: - relative = Path(filename).relative_to(package_path) - except ValueError: - continue - # If the path of the file (which is relative to the top of the zip - # namespace), relative to the package given when the resource - # reader was created, has a parent, then it's a name in a - # subdirectory and thus we skip it. - parent_name = relative.parent.name - if len(parent_name) == 0: - yield relative.name - elif parent_name not in subdirs_seen: - subdirs_seen.add(parent_name) - yield parent_name - - -# Called from zipimport.c -def _zipimport_get_resource_reader(zipimporter, fullname): - try: - if not zipimporter.is_package(fullname): - return None - except ZipImportError: - return None - return _ZipImportResourceReader(zipimporter, fullname) From 926a05ec2312c6a1cf296599c7e51469cfa09773 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 6 Mar 2020 14:58:23 -0500 Subject: [PATCH 04/10] Update references for CPython codebase --- Lib/importlib/_common.py | 24 ++++++++++-------------- Lib/importlib/resources.py | 7 ++++--- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/Lib/importlib/_common.py b/Lib/importlib/_common.py index 1689f357..d7c79296 100644 --- a/Lib/importlib/_common.py +++ b/Lib/importlib/_common.py @@ -1,21 +1,17 @@ -from __future__ import absolute_import - import os +import pathlib +import zipfile import tempfile +import functools import contextlib -from ._compat import ( - Path, package_spec, FileNotFoundError, ZipPath, - singledispatch, suppress, - ) - def from_package(package): """ Return a Traversable object for the given package. """ - spec = package_spec(package) + spec = package.__spec__ return from_traversable_resources(spec) or fallback_resources(spec) @@ -24,16 +20,16 @@ def from_traversable_resources(spec): If the spec.loader implements TraversableResources, directly or implicitly, it will have a ``files()`` method. """ - with suppress(AttributeError): + with contextlib.suppress(AttributeError): return spec.loader.files() def fallback_resources(spec): - package_directory = Path(spec.origin).parent + package_directory = pathlib.Path(spec.origin).parent try: archive_path = spec.loader.archive rel_path = package_directory.relative_to(archive_path) - return ZipPath(archive_path, str(rel_path) + '/') + return zipfile.Path(archive_path, str(rel_path) + '/') except Exception: pass return package_directory @@ -48,7 +44,7 @@ def _tempfile(reader): try: os.write(fd, reader()) os.close(fd) - yield Path(raw_path) + yield pathlib.Path(raw_path) finally: try: os.remove(raw_path) @@ -56,7 +52,7 @@ def _tempfile(reader): pass -@singledispatch +@functools.singledispatch @contextlib.contextmanager def as_file(path): """ @@ -67,7 +63,7 @@ def as_file(path): yield local -@as_file.register(Path) +@as_file.register(pathlib.Path) @contextlib.contextmanager def _(path): """ diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index 71570c51..e570401c 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -2,23 +2,24 @@ from . import abc as resources_abc from . import _common +from ._common import as_file from contextlib import contextmanager, suppress from importlib import import_module from importlib.abc import ResourceLoader from io import BytesIO, TextIOWrapper from pathlib import Path from types import ModuleType -from typing import Iterable, Iterator, Optional, Set, Union # noqa: F401 +from typing import ContextManager, Iterable, Optional, Union from typing import cast from typing.io import BinaryIO, TextIO -if False: # TYPE_CHECKING - from typing import ContextManager __all__ = [ 'Package', 'Resource', + 'as_file', 'contents', + 'files', 'is_resource', 'open_binary', 'open_text', From 03fd6f67a23fa5016c2ffccd9f43f2687aa94532 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Apr 2020 19:55:00 -0400 Subject: [PATCH 05/10] Tests are in Lib/test --- Lib/{tests => test}/test_importlib/__init__.py | 0 .../test_importlib/data01/__init__.py | 0 .../test_importlib/data01/binary.file | Bin .../test_importlib/data01/subdirectory/__init__.py | 0 .../test_importlib/data01/subdirectory/binary.file | Bin .../test_importlib/data01/utf-16.file | Bin .../test_importlib/data01/utf-8.file | 0 .../test_importlib/data02/__init__.py | 0 .../test_importlib/data02/one/__init__.py | 0 .../test_importlib/data02/one/resource1.txt | 0 .../test_importlib/data02/two/__init__.py | 0 .../test_importlib/data02/two/resource2.txt | 0 .../test_importlib/data03/__init__.py | 0 .../data03/namespace/portion1/__init__.py | 0 .../data03/namespace/portion2/__init__.py | 0 .../test_importlib/data03/namespace/resource1.txt | 0 Lib/{tests => test}/test_importlib/test_files.py | 0 Lib/{tests => test}/test_importlib/test_open.py | 0 Lib/{tests => test}/test_importlib/test_path.py | 0 Lib/{tests => test}/test_importlib/test_read.py | 0 Lib/{tests => test}/test_importlib/test_resource.py | 0 Lib/{tests => test}/test_importlib/util.py | 0 .../test_importlib/zipdata01/__init__.py | 0 .../test_importlib/zipdata01/ziptestdata.zip | Bin .../test_importlib/zipdata02/__init__.py | 0 .../test_importlib/zipdata02/ziptestdata.zip | Bin 26 files changed, 0 insertions(+), 0 deletions(-) rename Lib/{tests => test}/test_importlib/__init__.py (100%) rename Lib/{tests => test}/test_importlib/data01/__init__.py (100%) rename Lib/{tests => test}/test_importlib/data01/binary.file (100%) rename Lib/{tests => test}/test_importlib/data01/subdirectory/__init__.py (100%) rename Lib/{tests => test}/test_importlib/data01/subdirectory/binary.file (100%) rename Lib/{tests => test}/test_importlib/data01/utf-16.file (100%) rename Lib/{tests => test}/test_importlib/data01/utf-8.file (100%) rename Lib/{tests => test}/test_importlib/data02/__init__.py (100%) rename Lib/{tests => test}/test_importlib/data02/one/__init__.py (100%) rename Lib/{tests => test}/test_importlib/data02/one/resource1.txt (100%) rename Lib/{tests => test}/test_importlib/data02/two/__init__.py (100%) rename Lib/{tests => test}/test_importlib/data02/two/resource2.txt (100%) rename Lib/{tests => test}/test_importlib/data03/__init__.py (100%) rename Lib/{tests => test}/test_importlib/data03/namespace/portion1/__init__.py (100%) rename Lib/{tests => test}/test_importlib/data03/namespace/portion2/__init__.py (100%) rename Lib/{tests => test}/test_importlib/data03/namespace/resource1.txt (100%) rename Lib/{tests => test}/test_importlib/test_files.py (100%) rename Lib/{tests => test}/test_importlib/test_open.py (100%) rename Lib/{tests => test}/test_importlib/test_path.py (100%) rename Lib/{tests => test}/test_importlib/test_read.py (100%) rename Lib/{tests => test}/test_importlib/test_resource.py (100%) rename Lib/{tests => test}/test_importlib/util.py (100%) rename Lib/{tests => test}/test_importlib/zipdata01/__init__.py (100%) rename Lib/{tests => test}/test_importlib/zipdata01/ziptestdata.zip (100%) rename Lib/{tests => test}/test_importlib/zipdata02/__init__.py (100%) rename Lib/{tests => test}/test_importlib/zipdata02/ziptestdata.zip (100%) diff --git a/Lib/tests/test_importlib/__init__.py b/Lib/test/test_importlib/__init__.py similarity index 100% rename from Lib/tests/test_importlib/__init__.py rename to Lib/test/test_importlib/__init__.py diff --git a/Lib/tests/test_importlib/data01/__init__.py b/Lib/test/test_importlib/data01/__init__.py similarity index 100% rename from Lib/tests/test_importlib/data01/__init__.py rename to Lib/test/test_importlib/data01/__init__.py diff --git a/Lib/tests/test_importlib/data01/binary.file b/Lib/test/test_importlib/data01/binary.file similarity index 100% rename from Lib/tests/test_importlib/data01/binary.file rename to Lib/test/test_importlib/data01/binary.file diff --git a/Lib/tests/test_importlib/data01/subdirectory/__init__.py b/Lib/test/test_importlib/data01/subdirectory/__init__.py similarity index 100% rename from Lib/tests/test_importlib/data01/subdirectory/__init__.py rename to Lib/test/test_importlib/data01/subdirectory/__init__.py diff --git a/Lib/tests/test_importlib/data01/subdirectory/binary.file b/Lib/test/test_importlib/data01/subdirectory/binary.file similarity index 100% rename from Lib/tests/test_importlib/data01/subdirectory/binary.file rename to Lib/test/test_importlib/data01/subdirectory/binary.file diff --git a/Lib/tests/test_importlib/data01/utf-16.file b/Lib/test/test_importlib/data01/utf-16.file similarity index 100% rename from Lib/tests/test_importlib/data01/utf-16.file rename to Lib/test/test_importlib/data01/utf-16.file diff --git a/Lib/tests/test_importlib/data01/utf-8.file b/Lib/test/test_importlib/data01/utf-8.file similarity index 100% rename from Lib/tests/test_importlib/data01/utf-8.file rename to Lib/test/test_importlib/data01/utf-8.file diff --git a/Lib/tests/test_importlib/data02/__init__.py b/Lib/test/test_importlib/data02/__init__.py similarity index 100% rename from Lib/tests/test_importlib/data02/__init__.py rename to Lib/test/test_importlib/data02/__init__.py diff --git a/Lib/tests/test_importlib/data02/one/__init__.py b/Lib/test/test_importlib/data02/one/__init__.py similarity index 100% rename from Lib/tests/test_importlib/data02/one/__init__.py rename to Lib/test/test_importlib/data02/one/__init__.py diff --git a/Lib/tests/test_importlib/data02/one/resource1.txt b/Lib/test/test_importlib/data02/one/resource1.txt similarity index 100% rename from Lib/tests/test_importlib/data02/one/resource1.txt rename to Lib/test/test_importlib/data02/one/resource1.txt diff --git a/Lib/tests/test_importlib/data02/two/__init__.py b/Lib/test/test_importlib/data02/two/__init__.py similarity index 100% rename from Lib/tests/test_importlib/data02/two/__init__.py rename to Lib/test/test_importlib/data02/two/__init__.py diff --git a/Lib/tests/test_importlib/data02/two/resource2.txt b/Lib/test/test_importlib/data02/two/resource2.txt similarity index 100% rename from Lib/tests/test_importlib/data02/two/resource2.txt rename to Lib/test/test_importlib/data02/two/resource2.txt diff --git a/Lib/tests/test_importlib/data03/__init__.py b/Lib/test/test_importlib/data03/__init__.py similarity index 100% rename from Lib/tests/test_importlib/data03/__init__.py rename to Lib/test/test_importlib/data03/__init__.py diff --git a/Lib/tests/test_importlib/data03/namespace/portion1/__init__.py b/Lib/test/test_importlib/data03/namespace/portion1/__init__.py similarity index 100% rename from Lib/tests/test_importlib/data03/namespace/portion1/__init__.py rename to Lib/test/test_importlib/data03/namespace/portion1/__init__.py diff --git a/Lib/tests/test_importlib/data03/namespace/portion2/__init__.py b/Lib/test/test_importlib/data03/namespace/portion2/__init__.py similarity index 100% rename from Lib/tests/test_importlib/data03/namespace/portion2/__init__.py rename to Lib/test/test_importlib/data03/namespace/portion2/__init__.py diff --git a/Lib/tests/test_importlib/data03/namespace/resource1.txt b/Lib/test/test_importlib/data03/namespace/resource1.txt similarity index 100% rename from Lib/tests/test_importlib/data03/namespace/resource1.txt rename to Lib/test/test_importlib/data03/namespace/resource1.txt diff --git a/Lib/tests/test_importlib/test_files.py b/Lib/test/test_importlib/test_files.py similarity index 100% rename from Lib/tests/test_importlib/test_files.py rename to Lib/test/test_importlib/test_files.py diff --git a/Lib/tests/test_importlib/test_open.py b/Lib/test/test_importlib/test_open.py similarity index 100% rename from Lib/tests/test_importlib/test_open.py rename to Lib/test/test_importlib/test_open.py diff --git a/Lib/tests/test_importlib/test_path.py b/Lib/test/test_importlib/test_path.py similarity index 100% rename from Lib/tests/test_importlib/test_path.py rename to Lib/test/test_importlib/test_path.py diff --git a/Lib/tests/test_importlib/test_read.py b/Lib/test/test_importlib/test_read.py similarity index 100% rename from Lib/tests/test_importlib/test_read.py rename to Lib/test/test_importlib/test_read.py diff --git a/Lib/tests/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py similarity index 100% rename from Lib/tests/test_importlib/test_resource.py rename to Lib/test/test_importlib/test_resource.py diff --git a/Lib/tests/test_importlib/util.py b/Lib/test/test_importlib/util.py similarity index 100% rename from Lib/tests/test_importlib/util.py rename to Lib/test/test_importlib/util.py diff --git a/Lib/tests/test_importlib/zipdata01/__init__.py b/Lib/test/test_importlib/zipdata01/__init__.py similarity index 100% rename from Lib/tests/test_importlib/zipdata01/__init__.py rename to Lib/test/test_importlib/zipdata01/__init__.py diff --git a/Lib/tests/test_importlib/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/zipdata01/ziptestdata.zip similarity index 100% rename from Lib/tests/test_importlib/zipdata01/ziptestdata.zip rename to Lib/test/test_importlib/zipdata01/ziptestdata.zip diff --git a/Lib/tests/test_importlib/zipdata02/__init__.py b/Lib/test/test_importlib/zipdata02/__init__.py similarity index 100% rename from Lib/tests/test_importlib/zipdata02/__init__.py rename to Lib/test/test_importlib/zipdata02/__init__.py diff --git a/Lib/tests/test_importlib/zipdata02/ziptestdata.zip b/Lib/test/test_importlib/zipdata02/ziptestdata.zip similarity index 100% rename from Lib/tests/test_importlib/zipdata02/ziptestdata.zip rename to Lib/test/test_importlib/zipdata02/ziptestdata.zip From 38504ca06ba746ac94ddb84c709d9b3e31a581ee Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Apr 2020 20:22:48 -0400 Subject: [PATCH 06/10] Sync with upstream --- Lib/test/test_importlib/util.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index b0badebc..de6e0b02 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -7,6 +7,7 @@ from importlib import machinery, util, invalidate_caches from importlib.abc import ResourceReader import io +import marshal import os import os.path from pathlib import Path, PurePath @@ -118,6 +119,16 @@ def submodule(parent, name, pkg_dir, content=''): return '{}.{}'.format(parent, name), path +def get_code_from_pyc(pyc_path): + """Reads a pyc file and returns the unmarshalled code object within. + + No header validation is performed. + """ + with open(pyc_path, 'rb') as pyc_f: + pyc_f.seek(16) + return marshal.load(pyc_f) + + @contextlib.contextmanager def uncache(*names): """Uncache a module from sys.modules. @@ -319,6 +330,17 @@ def ensure_bytecode_path(bytecode_path): raise +@contextlib.contextmanager +def temporary_pycache_prefix(prefix): + """Adjust and restore sys.pycache_prefix.""" + _orig_prefix = sys.pycache_prefix + sys.pycache_prefix = prefix + try: + yield + finally: + sys.pycache_prefix = _orig_prefix + + @contextlib.contextmanager def create_modules(*names): """Temporarily create each named module with an attribute (named 'attr') @@ -432,7 +454,7 @@ def contents(self): yield entry name = 'testingpackage' - # Unforunately importlib.util.module_from_spec() was not introduced until + # Unfortunately importlib.util.module_from_spec() was not introduced until # Python 3.5. module = types.ModuleType(name) loader = Reader() @@ -477,7 +499,7 @@ def test_absolute_path(self): self.execute(data01, full_path) def test_relative_path(self): - # A reative path is a ValueError. + # A relative path is a ValueError. with self.assertRaises(ValueError): self.execute(data01, '../data01/utf-8.file') From 523969f64ee89c00151ae702c651c83a46beede3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 22 Oct 2020 22:46:03 -0400 Subject: [PATCH 07/10] Sync changes from CPython --- Lib/importlib/abc.py | 48 +-------------------------------- Lib/test/test_importlib/util.py | 16 ++++++----- 2 files changed, 10 insertions(+), 54 deletions(-) diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 0b20e7c1..97d5afa3 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -12,6 +12,7 @@ import _frozen_importlib_external except ImportError: _frozen_importlib_external = _bootstrap_external +from ._abc import Loader import abc import warnings from typing import Protocol, runtime_checkable @@ -134,53 +135,6 @@ def invalidate_caches(self): _register(PathEntryFinder, machinery.FileFinder) -class Loader(metaclass=abc.ABCMeta): - - """Abstract base class for import loaders.""" - - def create_module(self, spec): - """Return a module to initialize and into which to load. - - This method should raise ImportError if anything prevents it - from creating a new module. It may return None to indicate - that the spec should create the new module. - """ - # By default, defer to default semantics for the new module. - return None - - # We don't define exec_module() here since that would break - # hasattr checks we do to support backward compatibility. - - def load_module(self, fullname): - """Return the loaded module. - - The module must be added to sys.modules and have import-related - attributes set properly. The fullname is a str. - - ImportError is raised on failure. - - This method is deprecated in favor of loader.exec_module(). If - exec_module() exists then it is used to provide a backwards-compatible - functionality for this method. - - """ - if not hasattr(self, 'exec_module'): - raise ImportError - return _bootstrap._load_module_shim(self, fullname) - - def module_repr(self, module): - """Return a module's repr. - - Used by the module type when the method does not raise - NotImplementedError. - - This method is deprecated. - - """ - # The exception will cause ModuleType.__repr__ to ignore this method. - raise NotImplementedError - - class ResourceLoader(Loader): """Abstract base class for loaders which can return data from their diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index de6e0b02..5c0375e0 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -12,6 +12,8 @@ import os.path from pathlib import Path, PurePath from test import support +from test.support import import_helper +from test.support import os_helper import unittest import sys import tempfile @@ -55,8 +57,8 @@ def _extension_details(): def import_importlib(module_name): """Import a module from importlib both w/ and w/o _frozen_importlib.""" fresh = ('importlib',) if '.' in module_name else () - frozen = support.import_fresh_module(module_name) - source = support.import_fresh_module(module_name, fresh=fresh, + frozen = import_helper.import_fresh_module(module_name) + source = import_helper.import_fresh_module(module_name, fresh=fresh, blocked=('_frozen_importlib', '_frozen_importlib_external')) return {'Frozen': frozen, 'Source': source} @@ -158,9 +160,9 @@ def uncache(*names): @contextlib.contextmanager def temp_module(name, content='', *, pkg=False): conflicts = [n for n in sys.modules if n.partition('.')[0] == name] - with support.temp_cwd(None) as cwd: + with os_helper.temp_cwd(None) as cwd: with uncache(name, *conflicts): - with support.DirsOnSysPath(cwd): + with import_helper.DirsOnSysPath(cwd): invalidate_caches() location = os.path.join(cwd, name) @@ -396,7 +398,7 @@ def create_modules(*names): state_manager.__exit__(None, None, None) if uncache_manager is not None: uncache_manager.__exit__(None, None, None) - support.rmtree(temp_dir) + os_helper.rmtree(temp_dir) def mock_path_hook(*entries, importer): @@ -572,8 +574,8 @@ def tearDownClass(cls): pass def setUp(self): - modules = support.modules_setup() - self.addCleanup(support.modules_cleanup, *modules) + modules = import_helper.modules_setup() + self.addCleanup(import_helper.modules_cleanup, *modules) class ZipSetup(ZipSetupBase): From 8529e15f885aafdbbba260a648f139af73fc531c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 22 Oct 2020 22:50:50 -0400 Subject: [PATCH 08/10] Fixup imports --- Lib/test/test_importlib/test_resource.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py index d3c4ef1f..1d1bdad1 100644 --- a/Lib/test/test_importlib/test_resource.py +++ b/Lib/test/test_importlib/test_resource.py @@ -7,7 +7,8 @@ from . import zipdata01, zipdata02 from . import util from importlib import resources, import_module -from test.support import import_helper, unlink +from test.support import import_helper +from test.support.os_helper import unlink class ResourceTests: @@ -175,10 +176,10 @@ def setUp(self): modules = import_helper.modules_setup() self.addCleanup(import_helper.modules_cleanup, *modules) - data_path = Path(self.ZIP_MODULE.__file__) + data_path = pathlib.Path(self.ZIP_MODULE.__file__) data_dir = data_path.parent self.source_zip_path = data_dir / 'ziptestdata.zip' - self.zip_path = Path('{}.zip'.format(uuid.uuid4())).absolute() + self.zip_path = pathlib.Path('{}.zip'.format(uuid.uuid4())).absolute() self.zip_path.write_bytes(self.source_zip_path.read_bytes()) sys.path.append(str(self.zip_path)) self.data = import_module('ziptestdata') From cf54b86f494f973073883f82af2819b564b5a062 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 10 Jan 2021 09:48:19 -0500 Subject: [PATCH 09/10] Apply python/cpython@7d9d25dbedfffce61fc76bc7ccbfa9ae901bf56f --- Lib/importlib/abc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 97d5afa3..55e70889 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -1,5 +1,4 @@ """Abstract base classes related to import.""" -from . import _bootstrap from . import _bootstrap_external from . import machinery try: From 71693694c262e7dce3d8d192ab41f141079f7840 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 21 May 2021 12:09:52 -0400 Subject: [PATCH 10/10] Sync upstream changes from python/cpython@b51081c1a8. --- Lib/importlib/abc.py | 20 ++++++++++++++++---- Lib/test/test_importlib/test_files.py | 2 +- Lib/test/test_importlib/test_path.py | 8 +++----- Lib/test/test_importlib/test_reader.py | 1 - Lib/test/test_importlib/util.py | 6 +++--- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 4be51e23..0b4a3f80 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -41,15 +41,27 @@ class Finder(metaclass=abc.ABCMeta): Deprecated since Python 3.3 """ + def __init__(self): + warnings.warn("the Finder ABC is deprecated and " + "slated for removal in Python 3.12; use MetaPathFinder " + "or PathEntryFinder instead", + DeprecationWarning) + @abc.abstractmethod def find_module(self, fullname, path=None): """An abstract method that should find a module. The fullname is a str and the optional path is a str or None. Returns a Loader object or None. """ + warnings.warn("importlib.abc.Finder along with its find_module() " + "method are deprecated and " + "slated for removal in Python 3.12; use " + "MetaPathFinder.find_spec() or " + "PathEntryFinder.find_spec() instead", + DeprecationWarning) -class MetaPathFinder(Finder): +class MetaPathFinder(metaclass=abc.ABCMeta): """Abstract base class for import finders on sys.meta_path.""" @@ -68,8 +80,8 @@ def find_module(self, fullname, path): """ warnings.warn("MetaPathFinder.find_module() is deprecated since Python " - "3.4 in favor of MetaPathFinder.find_spec() " - "(available since 3.4)", + "3.4 in favor of MetaPathFinder.find_spec() and is " + "slated for removal in Python 3.12", DeprecationWarning, stacklevel=2) if not hasattr(self, 'find_spec'): @@ -86,7 +98,7 @@ def invalidate_caches(self): machinery.PathFinder, machinery.WindowsRegistryFinder) -class PathEntryFinder(Finder): +class PathEntryFinder(metaclass=abc.ABCMeta): """Abstract base class for path entry finders used by PathFinder.""" diff --git a/Lib/test/test_importlib/test_files.py b/Lib/test/test_importlib/test_files.py index 1e7a1f3c..481829b7 100644 --- a/Lib/test/test_importlib/test_files.py +++ b/Lib/test/test_importlib/test_files.py @@ -15,7 +15,7 @@ def test_read_bytes(self): def test_read_text(self): files = resources.files(self.data) - actual = files.joinpath('utf-8.file').read_text() + actual = files.joinpath('utf-8.file').read_text(encoding='utf-8') assert actual == 'Hello, UTF-8 world!\n' @unittest.skipUnless( diff --git a/Lib/test/test_importlib/test_path.py b/Lib/test/test_importlib/test_path.py index 2110770e..d6ed09a9 100644 --- a/Lib/test/test_importlib/test_path.py +++ b/Lib/test/test_importlib/test_path.py @@ -29,11 +29,9 @@ class PathDiskTests(PathTests, unittest.TestCase): data = data01 def test_natural_path(self): - """ - Guarantee the internal implementation detail that - file-system-backed resources do not get the tempdir - treatment. - """ + # Guarantee the internal implementation detail that + # file-system-backed resources do not get the tempdir + # treatment. with resources.path(self.data, 'utf-8.file') as path: assert 'data' in str(path) diff --git a/Lib/test/test_importlib/test_reader.py b/Lib/test/test_importlib/test_reader.py index 905d4fcd..b0bf49b8 100644 --- a/Lib/test/test_importlib/test_reader.py +++ b/Lib/test/test_importlib/test_reader.py @@ -60,7 +60,6 @@ def test_open_file(self): path.open() def test_join_path(self): - print('test_join_path') prefix = os.path.abspath(os.path.join(__file__, '..')) data01 = os.path.join(prefix, 'data01') path = MultiplexedPath(self.folder, data01) diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 5c0375e0..ca0d8c9b 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -116,7 +116,7 @@ def case_insensitive_tests(test): def submodule(parent, name, pkg_dir, content=''): path = os.path.join(pkg_dir, name + '.py') - with open(path, 'w') as subfile: + with open(path, 'w', encoding='utf-8') as subfile: subfile.write(content) return '{}.{}'.format(parent, name), path @@ -176,7 +176,7 @@ def temp_module(name, content='', *, pkg=False): content = '' if content is not None: # not a namespace package - with open(modpath, 'w') as modfile: + with open(modpath, 'w', encoding='utf-8') as modfile: modfile.write(content) yield location @@ -384,7 +384,7 @@ def create_modules(*names): os.mkdir(file_path) created_paths.append(file_path) file_path = os.path.join(file_path, name_parts[-1] + '.py') - with open(file_path, 'w') as file: + with open(file_path, 'w', encoding='utf-8') as file: file.write(source.format(name)) created_paths.append(file_path) mapping[name] = file_path