diff --git a/.github/workflows/linux_meson.yml b/.github/workflows/linux_meson.yml index 8ef0e5752119..b489c9e3f12f 100644 --- a/.github/workflows/linux_meson.yml +++ b/.github/workflows/linux_meson.yml @@ -68,5 +68,6 @@ jobs: TERM: xterm-256color LD_LIBRARY_PATH: "/usr/local/lib/" # to find libopenblas.so.0 run: | + export NPY_RUN_MYPY_IN_TESTSUITE=1 pip install pytest pytest-xdist hypothesis typing_extensions spin test -j auto diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 5a33200089ab..506a6d59f3eb 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -78,7 +78,7 @@ jobs: - [ubuntu-20.04, musllinux_x86_64] - [macos-12, macosx_x86_64] - [windows-2019, win_amd64] - python: ["cp39", "cp310", "cp311", "cp312"] # "pp39" + python: ["cp39", "cp310", "cp311", "cp312", "pp39"] exclude: # Don't build PyPy 32-bit windows - buildplat: [windows-2019, win32] diff --git a/numpy/__init__.py b/numpy/__init__.py index b4b33320b9f9..cf852aeadd14 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -66,7 +66,7 @@ NumPy testing tools distutils Enhancements to distutils with support for - Fortran compilers support and more. + Fortran compilers support and more (for Python <= 3.11). Utilities --------- diff --git a/numpy/_pytesttester.py b/numpy/_pytesttester.py index 01ddaaf98834..1c38291ae331 100644 --- a/numpy/_pytesttester.py +++ b/numpy/_pytesttester.py @@ -135,12 +135,13 @@ def __call__(self, label='fast', verbose=1, extra_argv=None, # offset verbosity. The "-q" cancels a "-v". pytest_args += ["-q"] - with warnings.catch_warnings(): - warnings.simplefilter("always") - # Filter out distutils cpu warnings (could be localized to - # distutils tests). ASV has problems with top level import, - # so fetch module for suppression here. - from numpy.distutils import cpuinfo + if sys.version_info < (3, 12): + with warnings.catch_warnings(): + warnings.simplefilter("always") + # Filter out distutils cpu warnings (could be localized to + # distutils tests). ASV has problems with top level import, + # so fetch module for suppression here. + from numpy.distutils import cpuinfo with warnings.catch_warnings(record=True): # Ignore the warning from importing the array_api submodule. This diff --git a/numpy/core/meson.build b/numpy/core/meson.build index ab773c56d735..17760efa2fc0 100644 --- a/numpy/core/meson.build +++ b/numpy/core/meson.build @@ -50,7 +50,10 @@ C_API_VERSION = '0x00000011' # Check whether we have a mismatch between the set C API VERSION and the # actual C API VERSION. Will raise a MismatchCAPIError if so. -r = run_command('code_generators/verify_c_api_version.py', '--api-version', C_API_VERSION) +r = run_command( + 'code_generators/verify_c_api_version.py', '--api-version', C_API_VERSION, + check: true +) if r.returncode() != 0 error('verify_c_api_version.py failed with output:\n' + r.stderr().strip()) diff --git a/numpy/core/tests/test_array_interface.py b/numpy/core/tests/test_array_interface.py index 8b1ab27c5cd3..16c719c5a5b9 100644 --- a/numpy/core/tests/test_array_interface.py +++ b/numpy/core/tests/test_array_interface.py @@ -128,6 +128,9 @@ def get_module(tmp_path): more_init=more_init) +# FIXME: numpy.testing.extbuild uses `numpy.distutils`, so this won't work on +# Python 3.12 and up. +@pytest.mark.skipif(sys.version_info >= (3, 12), reason="no numpy.distutils") @pytest.mark.slow def test_cstruct(get_module): diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 57831f46f431..81692015fc5e 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -756,6 +756,11 @@ def iter_struct_object_dtypes(): yield pytest.param(dt, p, 12, obj, id="") +@pytest.mark.skipif( + sys.version_info >= (3, 12), + reason="Python 3.12 has immortal refcounts, this test will no longer " + "work. See gh-23986" +) @pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") class TestStructuredObjectRefcounting: """These tests cover various uses of complicated structured types which diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 841144790e31..678c727db479 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -1464,6 +1464,10 @@ def test_structured_arrays_with_objects1(self): x[x.nonzero()] = x.ravel()[:1] assert_(x[0, 1] == x[0, 0]) + @pytest.mark.skipif( + sys.version_info >= (3, 12), + reason="Python 3.12 has immortal refcounts, this test no longer works." + ) @pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") def test_structured_arrays_with_objects2(self): # Ticket #1299 second test diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 701eebbf0023..3bbbb215bb77 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -527,6 +527,7 @@ def test_load_padded_dtype(tmpdir, dt): assert_array_equal(arr, arr1) +@pytest.mark.skipif(sys.version_info >= (3, 12), reason="see gh-23988") @pytest.mark.xfail(IS_WASM, reason="Emscripten NODEFS has a buggy dup") def test_python2_python3_interoperability(): fname = 'win64python2.npy' diff --git a/numpy/tests/test_ctypeslib.py b/numpy/tests/test_ctypeslib.py index 1ea0837008b7..63906b0f41cb 100644 --- a/numpy/tests/test_ctypeslib.py +++ b/numpy/tests/test_ctypeslib.py @@ -1,11 +1,12 @@ import sys -import pytest +import sysconfig import weakref from pathlib import Path +import pytest + import numpy as np from numpy.ctypeslib import ndpointer, load_library, as_array -from numpy.distutils.misc_util import get_shared_lib_extension from numpy.testing import assert_, assert_array_equal, assert_raises, assert_equal try: @@ -52,12 +53,9 @@ def test_basic2(self): # Regression for #801: load_library with a full library name # (including extension) does not work. try: - try: - so = get_shared_lib_extension(is_python_ext=True) - # Should succeed - load_library('_multiarray_umath%s' % so, np.core._multiarray_umath.__file__) - except ImportError: - print("No distutils available, skipping test.") + so_ext = sysconfig.get_config_var('EXT_SUFFIX') + load_library('_multiarray_umath%s' % so_ext, + np.core._multiarray_umath.__file__) except ImportError as e: msg = ("ctypes is not available on this python: skipping the test" " (import error was: %s)" % str(e)) diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index eaa89aa6f749..555f1638413d 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -127,12 +127,6 @@ def test_NPY_NO_EXPORT(): "array_api", "array_api.linalg", "ctypeslib", - "distutils", - "distutils.cpuinfo", - "distutils.exec_command", - "distutils.misc_util", - "distutils.log", - "distutils.system_info", "doc", "doc.constants", "doc.ufuncs", @@ -165,6 +159,18 @@ def test_NPY_NO_EXPORT(): "typing.mypy_plugin", "version", ]] +if sys.version_info < (3, 12): + PUBLIC_MODULES += [ + 'numpy.' + s for s in [ + "distutils", + "distutils.cpuinfo", + "distutils.exec_command", + "distutils.misc_util", + "distutils.log", + "distutils.system_info", + ] + ] + PUBLIC_ALIASED_MODULES = [ @@ -193,62 +199,6 @@ def test_NPY_NO_EXPORT(): "core.records", "core.shape_base", "core.umath", - "core.umath_tests", - "distutils.armccompiler", - "distutils.fujitsuccompiler", - "distutils.ccompiler", - 'distutils.ccompiler_opt', - "distutils.command", - "distutils.command.autodist", - "distutils.command.bdist_rpm", - "distutils.command.build", - "distutils.command.build_clib", - "distutils.command.build_ext", - "distutils.command.build_py", - "distutils.command.build_scripts", - "distutils.command.build_src", - "distutils.command.config", - "distutils.command.config_compiler", - "distutils.command.develop", - "distutils.command.egg_info", - "distutils.command.install", - "distutils.command.install_clib", - "distutils.command.install_data", - "distutils.command.install_headers", - "distutils.command.sdist", - "distutils.conv_template", - "distutils.core", - "distutils.extension", - "distutils.fcompiler", - "distutils.fcompiler.absoft", - "distutils.fcompiler.arm", - "distutils.fcompiler.compaq", - "distutils.fcompiler.environment", - "distutils.fcompiler.g95", - "distutils.fcompiler.gnu", - "distutils.fcompiler.hpux", - "distutils.fcompiler.ibm", - "distutils.fcompiler.intel", - "distutils.fcompiler.lahey", - "distutils.fcompiler.mips", - "distutils.fcompiler.nag", - "distutils.fcompiler.none", - "distutils.fcompiler.pathf95", - "distutils.fcompiler.pg", - "distutils.fcompiler.nv", - "distutils.fcompiler.sun", - "distutils.fcompiler.vast", - "distutils.fcompiler.fujitsu", - "distutils.from_template", - "distutils.intelccompiler", - "distutils.lib2def", - "distutils.line_endings", - "distutils.mingw32ccompiler", - "distutils.msvccompiler", - "distutils.npy_pkg_config", - "distutils.numpy_distribution", - "distutils.pathccompiler", - "distutils.unixccompiler", "f2py.auxfuncs", "f2py.capi_maps", "f2py.cb_rules", @@ -290,6 +240,66 @@ def test_NPY_NO_EXPORT(): "random.bit_generator", "testing.print_coercion_tables", ]] +if sys.version_info < (3, 12): + PRIVATE_BUT_PRESENT_MODULES += [ + 'numpy.' + s for s in [ + "distutils.armccompiler", + "distutils.fujitsuccompiler", + "distutils.ccompiler", + 'distutils.ccompiler_opt', + "distutils.command", + "distutils.command.autodist", + "distutils.command.bdist_rpm", + "distutils.command.build", + "distutils.command.build_clib", + "distutils.command.build_ext", + "distutils.command.build_py", + "distutils.command.build_scripts", + "distutils.command.build_src", + "distutils.command.config", + "distutils.command.config_compiler", + "distutils.command.develop", + "distutils.command.egg_info", + "distutils.command.install", + "distutils.command.install_clib", + "distutils.command.install_data", + "distutils.command.install_headers", + "distutils.command.sdist", + "distutils.conv_template", + "distutils.core", + "distutils.extension", + "distutils.fcompiler", + "distutils.fcompiler.absoft", + "distutils.fcompiler.arm", + "distutils.fcompiler.compaq", + "distutils.fcompiler.environment", + "distutils.fcompiler.g95", + "distutils.fcompiler.gnu", + "distutils.fcompiler.hpux", + "distutils.fcompiler.ibm", + "distutils.fcompiler.intel", + "distutils.fcompiler.lahey", + "distutils.fcompiler.mips", + "distutils.fcompiler.nag", + "distutils.fcompiler.none", + "distutils.fcompiler.pathf95", + "distutils.fcompiler.pg", + "distutils.fcompiler.nv", + "distutils.fcompiler.sun", + "distutils.fcompiler.vast", + "distutils.fcompiler.fujitsu", + "distutils.from_template", + "distutils.intelccompiler", + "distutils.lib2def", + "distutils.line_endings", + "distutils.mingw32ccompiler", + "distutils.msvccompiler", + "distutils.npy_pkg_config", + "distutils.numpy_distribution", + "distutils.pathccompiler", + "distutils.unixccompiler", + ] + ] def is_unexpected(name): @@ -323,10 +333,14 @@ def is_unexpected(name): "numpy.core.code_generators.verify_c_api_version", "numpy.core.cversions", "numpy.core.generate_numpy_api", - "numpy.distutils.msvc9compiler", + "numpy.core.umath_tests", ] +if sys.version_info < (3, 12): + SKIP_LIST += ["numpy.distutils.msvc9compiler"] +# suppressing warnings from deprecated modules +@pytest.mark.filterwarnings("ignore:.*np.compat.*:DeprecationWarning") def test_all_modules_are_expected(): """ Test that we don't add anything that looks like a new public module by @@ -351,9 +365,6 @@ def test_all_modules_are_expected(): # below SKIP_LIST_2 = [ 'numpy.math', - 'numpy.distutils.log.sys', - 'numpy.distutils.log.logging', - 'numpy.distutils.log.warnings', 'numpy.doc.constants.re', 'numpy.doc.constants.textwrap', 'numpy.lib.emath', @@ -369,6 +380,12 @@ def test_all_modules_are_expected(): 'numpy.matlib.ctypeslib', 'numpy.matlib.ma', ] +if sys.version_info < (3, 12): + SKIP_LIST_2 += [ + 'numpy.distutils.log.sys', + 'numpy.distutils.log.logging', + 'numpy.distutils.log.warnings', + ] def test_all_modules_are_expected_2(): @@ -472,11 +489,7 @@ def check_importable(module_name): @pytest.mark.xfail( - hasattr(np.__config__, "_built_with_meson"), - reason = "Meson does not yet support entry points via pyproject.toml", -) -@pytest.mark.xfail( - sysconfig.get_config_var("Py_DEBUG") is not None, + sysconfig.get_config_var("Py_DEBUG") not in (None, 0, "0"), reason=( "NumPy possibly built with `USE_DEBUG=True ./tools/travis-test.sh`, " "which does not expose the `array_api` entry point. " @@ -488,6 +501,11 @@ def test_array_api_entry_point(): Entry point for Array API implementation can be found with importlib and returns the numpy.array_api namespace. """ + # For a development install that did not go through meson-python, + # the entrypoint will not have been installed. So ensure this test fails + # only if numpy is inside site-packages. + numpy_in_sitepackages = sysconfig.get_path('platlib') in np.__file__ + eps = importlib.metadata.entry_points() try: xp_eps = eps.select(group="array_api") @@ -497,12 +515,19 @@ def test_array_api_entry_point(): # Array API entry points so that running this test in <=3.9 will # still work - see https://github.com/numpy/numpy/pull/19800. xp_eps = eps.get("array_api", []) - assert len(xp_eps) > 0, "No entry points for 'array_api' found" + if len(xp_eps) == 0: + if numpy_in_sitepackages: + msg = "No entry points for 'array_api' found" + raise AssertionError(msg) from None + return try: ep = next(ep for ep in xp_eps if ep.name == "numpy") except StopIteration: - raise AssertionError("'numpy' not in array_api entry points") from None + if numpy_in_sitepackages: + msg = "'numpy' not in array_api entry points" + raise AssertionError(msg) from None + return xp = ep.load() msg = ( diff --git a/numpy/typing/tests/test_isfile.py b/numpy/typing/tests/test_isfile.py index a898b3e285b9..2ca2c9b21f94 100644 --- a/numpy/typing/tests/test_isfile.py +++ b/numpy/typing/tests/test_isfile.py @@ -1,4 +1,5 @@ import os +import sys from pathlib import Path import numpy as np @@ -10,7 +11,6 @@ ROOT / "__init__.pyi", ROOT / "ctypeslib.pyi", ROOT / "core" / "__init__.pyi", - ROOT / "distutils" / "__init__.pyi", ROOT / "f2py" / "__init__.pyi", ROOT / "fft" / "__init__.pyi", ROOT / "lib" / "__init__.pyi", @@ -21,6 +21,8 @@ ROOT / "random" / "__init__.pyi", ROOT / "testing" / "__init__.pyi", ] +if sys.version_info < (3, 12): + FILES += [ROOT / "distutils" / "__init__.pyi"] class TestIsFile: diff --git a/numpy/typing/tests/test_typing.py b/numpy/typing/tests/test_typing.py index bcaaf5250c9d..491431a86351 100644 --- a/numpy/typing/tests/test_typing.py +++ b/numpy/typing/tests/test_typing.py @@ -18,6 +18,21 @@ _C_INTP, ) + +# Only trigger a full `mypy` run if this environment variable is set +# Note that these tests tend to take over a minute even on a macOS M1 CPU, +# and more than that in CI. +RUN_MYPY = "NPY_RUN_MYPY_IN_TESTSUITE" in os.environ +if RUN_MYPY and RUN_MYPY not in ('0', '', 'false'): + RUN_MYPY = True + +# Skips all functions in this file +pytestmark = pytest.mark.skipif( + not RUN_MYPY, + reason="`NPY_RUN_MYPY_IN_TESTSUITE` not set" +) + + try: from mypy import api except ImportError: diff --git a/pyproject.toml b/pyproject.toml index 47fa0959bd3b..68361be2a881 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,7 +139,7 @@ tracker = "https://github.com/numpy/numpy/issues" # Note: the below skip command doesn't do much currently, the platforms to # build wheels for in CI are controlled in `.github/workflows/wheels.yml` and # `tools/ci/cirrus_wheels.yml`. -skip = "cp36-* cp37-* cp-38* pp37-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux_aarch64 *-win32" +skip = "cp36-* cp37-* cp-38* pp37-* pp38-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux_aarch64 *-win32" build-verbosity = "3" before-build = "bash {project}/tools/wheels/cibw_before_build.sh {project}" # meson has a hard dependency on ninja, and we need meson to build diff --git a/tools/wheels/check_license.py b/tools/wheels/check_license.py index 8ced317d674c..7d0ef7921a4e 100644 --- a/tools/wheels/check_license.py +++ b/tools/wheels/check_license.py @@ -7,10 +7,10 @@ distribution. """ -import os import sys import re import argparse +import pathlib def check_text(text): @@ -33,8 +33,12 @@ def main(): __import__(args.module) mod = sys.modules[args.module] + # LICENSE.txt is installed in the .dist-info directory, so find it there + sitepkgs = pathlib.Path(mod.__file__).parent.parent + distinfo_path = [s for s in sitepkgs.glob("numpy-*.dist-info")][0] + # Check license text - license_txt = os.path.join(os.path.dirname(mod.__file__), "LICENSE.txt") + license_txt = distinfo_path / "LICENSE.txt" with open(license_txt, encoding="utf-8") as f: text = f.read() diff --git a/tools/wheels/cibw_before_build.sh b/tools/wheels/cibw_before_build.sh index 493cceeae4b1..372b25af09b8 100644 --- a/tools/wheels/cibw_before_build.sh +++ b/tools/wheels/cibw_before_build.sh @@ -29,15 +29,32 @@ if [[ $RUNNER_OS == "Linux" || $RUNNER_OS == "macOS" ]] ; then cp $basedir/include/* /usr/local/include fi elif [[ $RUNNER_OS == "Windows" ]]; then + # delvewheel is the equivalent of delocate/auditwheel for windows. + python -m pip install delvewheel + + # make the DLL available for tools/wheels/repair_windows.sh. If you change + # this location you need to alter that script. + mkdir -p /c/opt/openblas/openblas_dll + PYTHONPATH=tools python -c "import openblas_support; openblas_support.make_init('numpy')" - target=$(python tools/openblas_support.py) - ls /tmp - mkdir -p openblas - # bash on windows does not like cp -r $target/* openblas - for f in $(ls $target); do - cp -r $target/$f openblas - done - ls openblas + mkdir -p /c/opt/32/lib/pkgconfig + mkdir -p /c/opt/64/lib/pkgconfig + target=$(python -c "import tools.openblas_support as obs; plat=obs.get_plat(); ilp64=obs.get_ilp64(); target=f'openblas_{plat}.zip'; obs.download_openblas(target, plat, ilp64);print(target)") + if [[ $PLATFORM == 'win-32' ]]; then + # 32-bit openBLAS + # Download 32 bit openBLAS and put it into c/opt/32/lib + unzip -o -d /c/opt/ $target + cp /c/opt/32/bin/*.dll /c/opt/openblas/openblas_dll + else + # 64-bit openBLAS + unzip -o -d /c/opt/ $target + if [[ -f /c/opt/64/lib/pkgconfig/openblas64.pc ]]; then + # As of v0.3.23, the 64-bit interface has a openblas64.pc file, + # but this is wrong. It should be openblas.pc + cp /c/opt/64/lib/pkgconfig/openblas{64,}.pc + fi + cp /c/opt/64/bin/*.dll /c/opt/openblas/openblas_dll + fi fi if [[ $RUNNER_OS == "macOS" ]]; then diff --git a/tools/wheels/cibw_test_command.sh b/tools/wheels/cibw_test_command.sh index 36c275f32d91..c35fb56832e5 100644 --- a/tools/wheels/cibw_test_command.sh +++ b/tools/wheels/cibw_test_command.sh @@ -26,10 +26,7 @@ fi # Set available memory value to avoid OOM problems on aarch64. # See gh-22418. export NPY_AVAILABLE_MEM="4 GB" -if [[ $(python -c "import sys; print(sys.implementation.name)") == "pypy" ]]; then - # make PyPy more verbose, try to catch a segfault - python -c "import sys; import numpy; sys.exit(not numpy.test(label='full', verbose=2))" -else - python -c "import sys; import numpy; sys.exit(not numpy.test(label='full'))" -fi +# Run full tests with -n=auto. This makes pytest-xdist distribute tests across +# the available N CPU cores: 2 by default for Linux instances and 4 for macOS arm64 +python -c "import sys; import numpy; sys.exit(not numpy.test(label='full', extra_argv=['-n=auto']))" python $PROJECT_DIR/tools/wheels/check_license.py