numpy · rgommers · Oct 21, 2023 · Oct 26, 2023 · Oct 26, 2023 · Oct 26, 2023
diff --git a/.cirrus.star b/.cirrus.star
@@ -51,4 +51,4 @@ def main(ctx):
     if int(pr_number) < 0:
         return []
 
-    return fs.read("tools/ci/cirrus_arm.yml")
+    # DISABLED: return fs.read("tools/ci/cirrus_arm.yml")
diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml
@@ -73,6 +73,8 @@ jobs:
           source .venv-pyodide/bin/activate
           pip install dist/*.whl
           python -c "import sys; print(sys.platform)"
+          # TODO: when re-enabled this workflow, install test deps differently
+          # since ninja isn't installable with pyodide
           pip install -r test_requirements.txt
       - name: Test
         run: |

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -202,7 +202,6 @@ jobs:
     - name: Install test dependencies
       run: |
         pip install -r test_requirements.txt
-        pip install ninja
     - name: Run test suite
       run: |
         cd tools

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -196,7 +196,6 @@ jobs:
           # TODO: Don't run test suite, and instead build wheels from sdist
           # Depends on pypa/cibuildwheel#1020
           python -m pip install dist/*.gz -Csetup-args=-Dallow-noblas=true
-          pip install ninja
           pip install -r test_requirements.txt
           cd .. # Can't import numpy within numpy src directory
           python -c "import numpy, sys; print(numpy.__version__); sys.exit(numpy.test() is False)"

diff --git a/numpy/f2py/tests/util.py b/numpy/f2py/tests/util.py
@@ -21,7 +21,8 @@
 
 from pathlib import Path
 from numpy._utils import asunicode
-from numpy.testing import temppath, IS_WASM
+from numpy.exceptions import VisibleDeprecationWarning
+from numpy.testing import temppath, IS_WASM, suppress_warnings
 from importlib import import_module
 
 #
@@ -409,13 +410,16 @@ def setup_method(self):
             )
 
         if self.sources is not None:
-            self.module = build_module(
-                self.sources,
-                options=self.options,
-                skip=self.skip,
-                only=self.only,
-                module_name=self.module_name,
-            )
+            with suppress_warnings() as sup:
+                sup.filter(VisibleDeprecationWarning,
+                           "distutils has been deprecated since")
+                self.module = build_module(
+                    self.sources,
+                    options=self.options,
+                    skip=self.skip,
+                    only=self.only,
+                    module_name=self.module_name,
+                )
 
 
 #

diff --git a/pyproject.toml b/pyproject.toml
@@ -140,34 +140,28 @@ tracker = "https://github.com/numpy/numpy/issues"
 # Note: the below skip command doesn't do much currently, the platforms to
 # build wheels for in CI are controlled in `.github/workflows/wheels.yml` and
 # `tools/ci/cirrus_wheels.yml`.
+build-frontend = "build"
 skip = "cp36-* cp37-* cp-38* pp37-* *-manylinux_i686 *_ppc64le *_s390x"
 build-verbosity = "3"
 before-build = "bash {project}/tools/wheels/cibw_before_build.sh {project}"
-config-settings = "setup-args=-Duse-ilp64=true setup-args=-Dblas=openblas setup-args=-Dlapack=openblas setup-args=-Dblas-symbol-suffix=64_ setup-args=-Dallow-noblas=false"
-# meson has a hard dependency on ninja, and we need meson to build
-# c-extensions in tests. There is a ninja PyPI package used in
-# build_requirements.txt for macOS, windows, linux but it cannot be in
-# test_requirements.txt since pyodide, which uses test_requirements.txt, does
-# not have it.
-before-test = "pip install ninja && pip install -r {project}/test_requirements.txt"
+# The build will use openblas64 everywhere, except on arm64 macOS >=14.0 (uses Accelerate)
+config-settings = "setup-args=-Duse-ilp64=true setup-args=-Dallow-noblas=false build-dir=build"
+before-test = "pip install -r {project}/test_requirements.txt"
 test-command = "bash {project}/tools/wheels/cibw_test_command.sh {project}"
 
 [tool.cibuildwheel.linux]
 manylinux-x86_64-image = "manylinux2014"
 manylinux-aarch64-image = "manylinux2014"
 musllinux-x86_64-image = "musllinux_1_1"
-environment = {CFLAGS="-fno-strict-aliasing", LDFLAGS="-Wl,--strip-debug", RUNNER_OS="Linux"}
+# RUNNER_OS is a GitHub Actions specific env var; define it here so it works on Cirrus CI too
+environment = {RUNNER_OS="Linux"}
 
 [tool.cibuildwheel.macos]
-# For universal2 wheels, we will need to fuse them manually
-# instead of going through cibuildwheel
-# This is because cibuildwheel tries to make a fat wheel
-# https://github.com/multi-build/multibuild/blame/devel/README.rst#L541-L565
-# for more info
+# universal2 wheels are not supported (see gh-21233), use `delocate-fuse` if you need them
 archs = "x86_64 arm64"
 test-skip = "*_universal2:arm64"
 # MACOS linker doesn't support stripping symbols.
-environment = {CFLAGS="-fno-strict-aliasing", RUNNER_OS="macOS"}
+environment = {RUNNER_OS="macOS"}
 
 [tool.cibuildwheel.windows]
 environment = {PKG_CONFIG_PATH="C:/opt/64/lib/pkgconfig"}

diff --git a/test_requirements.txt b/test_requirements.txt
@@ -7,6 +7,7 @@ pytest==7.4.0
 pytz==2023.3
 pytest-cov==4.1.0
 meson
+ninja
 pytest-xdist
 # for numpy.random.test.test_extending
 cffi; python_version < '3.10'

diff --git a/tools/ci/cirrus_arm.yml b/tools/ci/cirrus_arm.yml
@@ -95,7 +95,6 @@ macos_arm64_test_task:
     python --version
 
     RUNNER_OS="macOS"
-    SDKROOT=/Applications/Xcode-14.0.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk
 
     # NOTE: OpenBLAS is not used in this job; if that's done in the future, ensure
     # PKG_CONFIG_PATH points to the directory containing the openblas.pc file

diff --git a/tools/ci/cirrus_wheels.yml b/tools/ci/cirrus_wheels.yml
@@ -1,52 +1,13 @@
 build_and_store_wheels: &BUILD_AND_STORE_WHEELS
   install_cibuildwheel_script:
-    - python -m pip install cibuildwheel
+    - python3 -m pip install cibuildwheel
   cibuildwheel_script:
     - cibuildwheel
+  always:
+    show_meson_log_script: cat build/meson-logs/meson-log.txt
   wheels_artifacts:
     path: "wheelhouse/*"
 
-######################################################################
-# Build linux_aarch64 natively
-######################################################################
-
-linux_aarch64_task:
-  use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
-  compute_engine_instance:
-    image_project: cirrus-images
-    image: family/docker-builder-arm64
-    architecture: arm64
-    platform: linux
-    cpu: 1
-    memory: 8G
-  matrix:
-    # build in a matrix because building and testing all four wheels in a
-    # single task takes longer than 60 mins (the default time limit for a
-    # cirrus-ci task).
-    - env:
-        CIRRUS_CLONE_SUBMODULES: true
-        CIBW_BUILD: cp39-*
-        EXPECT_CPU_FEATURES: NEON NEON_FP16 NEON_VFPV4 ASIMD ASIMDHP ASIMDDP ASIMDFHM
-    - env:
-        CIRRUS_CLONE_SUBMODULES: true
-        CIBW_BUILD: cp310-*
-    - env:
-        CIRRUS_CLONE_SUBMODULES: true
-        CIBW_BUILD: cp311-*
-    - env:
-        CIRRUS_CLONE_SUBMODULES: true
-        CIBW_PRERELEASE_PYTHONS: True
-        CIBW_BUILD: cp312-*
-
-  build_script: |
-    apt update
-    apt install -y python3-venv python-is-python3 gfortran libatlas-base-dev libgfortran5 eatmydata
-    git fetch origin
-    bash ./tools/wheels/cibw_before_build.sh ${PWD}
-    which python
-    echo $CIRRUS_CHANGE_MESSAGE
-  <<: *BUILD_AND_STORE_WHEELS
-
 
 ######################################################################
 # Build macosx_arm64 natively
@@ -55,112 +16,32 @@ linux_aarch64_task:
 macosx_arm64_task:
   use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-monterey-xcode:14
+    matrix:
+      - image: ghcr.io/cirruslabs/macos-monterey-xcode:14
+      #- image: ghcr.io/cirruslabs/macos-sonoma-xcode:latest
   matrix:
     - env:
         CIRRUS_CLONE_SUBMODULES: true
-        CIBW_BUILD: cp39-*
-    - env:
-        CIRRUS_CLONE_SUBMODULES: true
-        CIBW_BUILD: cp310-* cp311-*
-    - env:
-        CIRRUS_CLONE_SUBMODULES: true
-        CIBW_PRERELEASE_PYTHONS: True
-        CIBW_BUILD: cp312-*
+        CIBW_BUILD: cp310-*
+        # Specifying CIBW_ENVIRONMENT_MACOS overrides pyproject.toml, so include
+        # all the settings from there, otherwise they're lost.
   env:
     PATH: /opt/homebrew/opt/[email protected]/bin:/usr/local/lib:/usr/local/include:$PATH
     CIBW_ARCHS: arm64
-    # Specifying CIBW_ENVIRONMENT_MACOS overrides pyproject.toml, so include
-    # all the settings from there, otherwise they're lost.
-    # SDKROOT needs to be set for repackaged conda-forge gfortran compilers
-    # supplied by isuruf.
-    # Find out SDKROOT via `xcrun --sdk macosx --show-sdk-path`
     CIBW_ENVIRONMENT_MACOS: >
       RUNNER_OS=macOS
-      SDKROOT=/Applications/Xcode-14.0.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk
+      MACOSX_DEPLOYMENT_TARGET=11.0  # 14.0
       LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
 
   build_script:
     - brew install [email protected]
     - ln -s python3 /opt/homebrew/opt/[email protected]/bin/python
-    - which python
+    - which python3
     # needed for submodules
     - git submodule update --init
     # need to obtain all the tags so setup.py can determine FULLVERSION
     - git fetch origin
     - uname -m
-    - python -c "import platform;print(platform.python_version());print(platform.system());print(platform.machine())"
+    - python3 -c "import platform; print(platform.python_version()); print(platform.system()); print(platform.machine())"
     - clang --version
   <<: *BUILD_AND_STORE_WHEELS
-
-
-######################################################################
-# Upload all wheels
-######################################################################
-
-wheels_upload_task:
-  use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
-  # Artifacts don't seem to be persistent from task to task.
-  # Rather than upload wheels at the end of each cibuildwheel run we do a
-  # final upload here. This is because a run may be on different OS for
-  # which bash, etc, may not be present.
-  depends_on:
-    - linux_aarch64
-    - macosx_arm64
-  compute_engine_instance:
-    image_project: cirrus-images
-    image: family/docker-builder
-    platform: linux
-    cpu: 1
-
-  env:
-    NUMPY_STAGING_UPLOAD_TOKEN: ENCRYPTED[!5a69522ae0c2af9edb2bc1cdfeaca6292fb3666d9ecd82dca0615921834a6ce3b702352835d8bde4ea2a9ed5ef8424ac!]
-    NUMPY_NIGHTLY_UPLOAD_TOKEN: ENCRYPTED[ef04347663cfcb58d121385707e55951dc8e03b009edeed988aa4a33ba8205c54ca9980ac4da88e1adfdebff8b9d7ed4]
-
-  upload_script: |
-    apt-get update
-    apt-get install -y curl wget
-    export IS_SCHEDULE_DISPATCH="false"
-    export IS_PUSH="false"
-
-    # cron job
-    if [[ "$CIRRUS_CRON" == "nightly" ]]; then
-      export IS_SCHEDULE_DISPATCH="true"
-    fi
-
-    # a manual build was started
-    if [[ "$CIRRUS_BUILD_SOURCE" == "api" && "$CIRRUS_COMMIT_MESSAGE" == "API build for null" ]]; then
-      export IS_SCHEDULE_DISPATCH="true"
-    fi
-
-    # only upload wheels to staging if it's a tag beginning with 'v' and you're
-    # on a maintenance branch
-    if [[ "$CIRRUS_TAG" == v* ]] && [[ $CIRRUS_TAG != *"dev0"* ]]; then
-      export IS_PUSH="true"
-    fi
-
-    if [[ $IS_PUSH == "true" ]] || [[ $IS_SCHEDULE_DISPATCH == "true" ]]; then
-        # install miniconda in the home directory. For some reason HOME isn't set by Cirrus
-        export HOME=$PWD
-
-        # install miniconda for uploading to anaconda
-        wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-        bash miniconda.sh -b -p $HOME/miniconda3
-        $HOME/miniconda3/bin/conda init bash
-        source $HOME/miniconda3/bin/activate
-        conda install -y anaconda-client
-
-        # The name of the zip file is derived from the `wheels_artifact` line.
-        # If you change the artifact line to `myfile_artifact` then it would be
-        # called myfile.zip
-
-        curl https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/wheels.zip --output wheels.zip
-        unzip wheels.zip
-
-        source ./tools/wheels/upload_wheels.sh
-        # IS_PUSH takes precedence over IS_SCHEDULE_DISPATCH
-        set_upload_vars
-
-        # Will be skipped if not a push/tag/scheduled build
-        upload_wheels
-    fi
diff --git a/tools/wheels/cibw_before_build.sh b/tools/wheels/cibw_before_build.sh
@@ -16,8 +16,19 @@ elif [[ $RUNNER_OS == "Windows" ]]; then
     cat $PROJECT_DIR/tools/wheels/LICENSE_win32.txt >> $PROJECT_DIR/LICENSE.txt
 fi
 
+install_openblas=true
+if [[ $RUNNER_OS == "macOS" ]]; then
+    if [[ $MACOSX_DEPLOYMENT_VERSION == "14.0" ]]; then
+        # This is the wheel build with Accelerate
+        install_openblas=false
+    else
+        # Done in gfortran_utils.sh
+        echo "deployment target determined from Python interpreter"
+    fi
+fi
+
 # Install Openblas
-if [[ $RUNNER_OS == "Linux" || $RUNNER_OS == "macOS" ]] ; then
+if [[ $RUNNER_OS == "Linux" || ($RUNNER_OS == "macOS" && $install_openblas) ]] ; then
     basedir=$(python tools/openblas_support.py --use-ilp64)
     if [[ $RUNNER_OS == "macOS" && $PLATFORM == "macosx-arm64" ]]; then
         # /usr/local/lib doesn't exist on cirrus-ci runners
@@ -52,6 +63,7 @@ if [[ $RUNNER_OS == "macOS" ]]; then
     if [[ $PLATFORM == "macosx-arm64" ]]; then
         PLAT="arm64"
     fi
+
     source $PROJECT_DIR/tools/wheels/gfortran_utils.sh
     install_gfortran
     pip install "delocate==0.10.4"

diff --git a/tools/wheels/gfortran_utils.sh b/tools/wheels/gfortran_utils.sh
@@ -109,12 +109,15 @@ function get_gf_lib_for_suf {
 }
 
 if [ "$(uname)" == "Darwin" ]; then
+    # Set deployment target to match the one Python was built for, if the env
+    # var wasn't explicitly set already.
     mac_target=${MACOSX_DEPLOYMENT_TARGET:-$(get_macosx_target)}
     export MACOSX_DEPLOYMENT_TARGET=$mac_target
     # Keep this for now as some builds might depend on this being
     # available before install_gfortran is called
     export GFORTRAN_SHA=c469a420d2d003112749dcdcbe3c684eef42127e
-    # Set SDKROOT env variable if not set
+    # SDKROOT needs to be set for repackaged conda-forge gfortran compilers
+    # supplied by isuruf. So set it if it's not already set.
     export SDKROOT=${SDKROOT:-$(xcrun --show-sdk-path)}
 
     function download_and_unpack_gfortran {
@@ -144,6 +147,7 @@ if [ "$(uname)" == "Darwin" ]; then
         pushd /opt
             sudo tar -xvf gfortran-darwin-${arch}-${type}.tar.gz
             sudo rm gfortran-darwin-${arch}-${type}.tar.gz
+            find gfortran-darwin-${arch}-${type} -name "libgfortran.spec" -exec sed -i.bak "s@\-lm@-lm -isysroot ${SDKROOT}@g" {} +
-            find gfortran-darwin-${arch}-${type} -name "libgfortran.spec" -exec sed -i.bak "s@\-lm@-lm -isysroot ${SDKROOT}@g" {} +
+            find gfortran-darwin-${arch}-${type}/lib/gcc -name "libgfortran.spec" -exec sed -i.bak "s@\-lm@-lm -isysroot ${SDKROOT}@g" {} +
-            find gfortran-darwin-${arch}-${type} -name "libgfortran.spec" -exec sed -i.bak "s@\-lm@-lm -isysroot ${SDKROOT}@g" {} +
+            find gfortran-darwin-${arch}-${type}/lib/gcc -name "libgfortran.spec" -exec sed -i.bak "s@\-lm@-lm -isysroot ${SDKROOT}@g" {} +
         popd
 	if [[ "${type}" == "native" ]]; then
 	    # Link these into /usr/local so that there's no need to add rpath or -L