From fff5a54f9273416b30100b6fa2a70cb951a52b7f Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Wed, 15 Mar 2023 16:35:46 -0400 Subject: [PATCH] ci: Merge sdist and wheel building workflows This allows verifying that wheels can be installed after building, and that the sdist can be built into a wheel without a second workflow. Also simplify some of the checking scripts. --- .github/workflows/cibuildsdist.yml | 74 ----------------------- .github/workflows/cibuildwheel.yml | 95 ++++++++++++++++++++++++------ ci/check_version_number.py | 8 +-- ci/check_wheel_licenses.py | 23 ++++---- ci/export_sdist_name.py | 21 +++++++ ci/silence | 14 ----- 6 files changed, 111 insertions(+), 124 deletions(-) delete mode 100644 .github/workflows/cibuildsdist.yml create mode 100644 ci/export_sdist_name.py delete mode 100755 ci/silence diff --git a/.github/workflows/cibuildsdist.yml b/.github/workflows/cibuildsdist.yml deleted file mode 100644 index 14c283ab1313..000000000000 --- a/.github/workflows/cibuildsdist.yml +++ /dev/null @@ -1,74 +0,0 @@ ---- -name: Build CI sdist and wheel - -on: - # Save CI by only running this on release branches or tags. - push: - branches: - - main - - v[0-9]+.[0-9]+.x - tags: - - v* - # Also allow running this action on PRs if requested by applying the - # "Run cibuildwheel" label. - pull_request: - types: - - opened - - synchronize - - reopened - - labeled - -permissions: - contents: read - -jobs: - build_sdist: - if: | - github.event_name == 'push' || - github.event_name == 'pull_request' && ( - ( - github.event.action == 'labeled' && - github.event.label.name == 'CI: Run cibuildwheel' - ) || - contains(github.event.pull_request.labels.*.name, 'CI: Run cibuildwheel') - ) - name: Build sdist and wheel on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04] - python-version: ['3.9', '3.10', '3.11'] - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - uses: actions/setup-python@v4 - name: Install Python - with: - python-version: ${{ matrix.python-version }} - - - name: Install build - run: pip install build - - - name: Build sdist and wheel - run: python -m build . - - - name: Install built matplotlib sdist - run: pip install dist/matplotlib*.tar.gz - - - name: Check version number is not 0 - run: python ./ci/check_version_number.py - - - name: Install built matplotlib wheel - run: pip install dist/matplotlib*.whl --force-reinstall - - - name: Check version number is not 0 - run: python ./ci/check_version_number.py - - - name: Install twine - run: pip install twine - - - name: Check README rendering for PyPI - run: twine check dist/* diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index b5c0e984b46a..bf84c99fe3bc 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -22,6 +22,59 @@ permissions: contents: read jobs: + build_sdist: + if: | + github.event_name == 'push' || + github.event_name == 'pull_request' && ( + ( + github.event.action == 'labeled' && + github.event.label.name == 'CI: Run cibuildwheel' + ) || + contains(github.event.pull_request.labels.*.name, + 'CI: Run cibuildwheel') + ) + name: Build sdist + runs-on: ubuntu-20.04 + outputs: + SDIST_NAME: ${{ steps.sdist.outputs.SDIST_NAME }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v4 + name: Install Python + with: + python-version: 3.9 + + # Something changed somewhere that prevents the downloaded-at-build-time + # licenses from being included in built wheels, so pre-download them so + # that they exist before the build and are included. + - name: Pre-download bundled licenses + run: > + curl -Lo LICENSE/LICENSE_QHULL + https://github.com/qhull/qhull/raw/2020.2/COPYING.txt + + - name: Install dependencies + run: python -m pip install build twine + + - name: Build sdist + id: sdist + run: | + python -m build --sdist + python ci/export_sdist_name.py + + - name: Check README rendering for PyPI + run: twine check dist/* + + - name: Upload sdist result + uses: actions/upload-artifact@v3 + with: + name: sdist + path: dist/*.tar.gz + if-no-files-found: error + build_wheels: if: | github.event_name == 'push' || @@ -30,21 +83,31 @@ jobs: github.event.action == 'labeled' && github.event.label.name == 'CI: Run cibuildwheel' ) || - contains(github.event.pull_request.labels.*.name, 'CI: Run cibuildwheel') + contains(github.event.pull_request.labels.*.name, + 'CI: Run cibuildwheel') ) + needs: build_sdist name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} env: CIBW_BEFORE_BUILD: >- pip install certifi oldest-supported-numpy && - git clean -fxd build + rm -rf {package}/build CIBW_BEFORE_BUILD_WINDOWS: >- pip install certifi delvewheel oldest-supported-numpy && - git clean -fxd build + rm -rf {package}/build CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >- delvewheel repair -w {dest_dir} {wheel} + CIBW_AFTER_BUILD: >- + twine check {wheel} && + python {package}/ci/check_wheel_licenses.py {wheel} CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 CIBW_SKIP: "*-musllinux*" + CIBW_TEST_COMMAND: >- + python {package}/ci/check_version_number.py + # Apple Silicon machines are not available for testing, so silence the + # warning from cibuildwheel. Remove the skip when they're available. + CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64" MACOSX_DEPLOYMENT_TARGET: "10.12" MPL_DISABLE_FH4: "yes" strategy: @@ -66,47 +129,45 @@ jobs: with: platforms: arm64 - - uses: actions/checkout@v3 + - name: Download sdist + uses: actions/download-artifact@v3 with: - fetch-depth: 0 - - # Something changed somewhere that prevents the downloaded-at-build-time - # licenses from being included in built wheels, so pre-download them so - # that they exist before the build and are included. - - name: Pre-download bundled licenses - run: > - curl -Lo LICENSE/LICENSE_QHULL - https://github.com/qhull/qhull/raw/2020.2/COPYING.txt + name: sdist + path: dist/ - name: Build wheels for CPython 3.11 uses: pypa/cibuildwheel@v2.13.0 + with: + package-dir: dist/${{ needs.build_sdist.outputs.SDIST_NAME }} env: CIBW_BUILD: "cp311-*" CIBW_ARCHS: ${{ matrix.cibw_archs }} - name: Build wheels for CPython 3.10 uses: pypa/cibuildwheel@v2.13.0 + with: + package-dir: dist/${{ needs.build_sdist.outputs.SDIST_NAME }} env: CIBW_BUILD: "cp310-*" CIBW_ARCHS: ${{ matrix.cibw_archs }} - name: Build wheels for CPython 3.9 uses: pypa/cibuildwheel@v2.13.0 + with: + package-dir: dist/${{ needs.build_sdist.outputs.SDIST_NAME }} env: CIBW_BUILD: "cp39-*" CIBW_ARCHS: ${{ matrix.cibw_archs }} - name: Build wheels for PyPy uses: pypa/cibuildwheel@v2.13.0 + with: + package-dir: dist/${{ needs.build_sdist.outputs.SDIST_NAME }} env: CIBW_BUILD: "pp39-*" CIBW_ARCHS: ${{ matrix.cibw_archs }} if: matrix.cibw_archs != 'aarch64' - - name: Validate that LICENSE files are included in wheels - run: | - python3 ./ci/check_wheel_licenses.py - - uses: actions/upload-artifact@v3 with: name: wheels diff --git a/ci/check_version_number.py b/ci/check_version_number.py index 123eb721fb59..8902fb0806c7 100644 --- a/ci/check_version_number.py +++ b/ci/check_version_number.py @@ -9,15 +9,11 @@ $ pip install dist/matplotlib*.whl for wheel $ ./ci/check_version_number.py """ -import matplotlib - import sys -EXIT_SUCCESS = 0 -EXIT_FAILURE = 1 +import matplotlib print(f"Version {matplotlib.__version__} installed") if matplotlib.__version__[0] == "0": - sys.exit(EXIT_FAILURE) -sys.exit(EXIT_SUCCESS) + sys.exit("Version incorrectly starts with 0") diff --git a/ci/check_wheel_licenses.py b/ci/check_wheel_licenses.py index 1b77108c83e1..13470dc692bd 100644 --- a/ci/check_wheel_licenses.py +++ b/ci/check_wheel_licenses.py @@ -1,27 +1,26 @@ #!/usr/bin/env python3 """ -Check that all .whl files in the dist folder have the correct LICENSE files -included. +Check that all specified .whl files have the correct LICENSE files included. To run: - $ python3 setup.py bdist_wheel - $ ./ci/check_wheel_licenses.py + $ python3 -m build --wheel + $ ./ci/check_wheel_licenses.py dist/*.whl """ from pathlib import Path import sys import zipfile -EXIT_SUCCESS = 0 -EXIT_FAILURE = 1 + +if len(sys.argv) <= 1: + sys.exit('At least one wheel must be specified in command-line arguments.') project_dir = Path(__file__).parent.resolve().parent -dist_dir = project_dir / 'dist' license_dir = project_dir / 'LICENSE' license_file_names = {path.name for path in sorted(license_dir.glob('*'))} -for wheel in dist_dir.glob('*.whl'): +for wheel in sys.argv[1:]: print(f'Checking LICENSE files in: {wheel}') with zipfile.ZipFile(wheel) as f: wheel_license_file_names = {Path(path).name @@ -29,8 +28,6 @@ if '.dist-info/LICENSE' in path} if not (len(wheel_license_file_names) and wheel_license_file_names.issuperset(license_file_names)): - print(f'LICENSE file(s) missing:\n' - f'{wheel_license_file_names} !=\n' - f'{license_file_names}') - sys.exit(EXIT_FAILURE) -sys.exit(EXIT_SUCCESS) + sys.exit(f'LICENSE file(s) missing:\n' + f'{wheel_license_file_names} !=\n' + f'{license_file_names}') diff --git a/ci/export_sdist_name.py b/ci/export_sdist_name.py new file mode 100644 index 000000000000..632c11a15f83 --- /dev/null +++ b/ci/export_sdist_name.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +""" +Determine the name of the sdist and export to GitHub output named SDIST_NAME. + +To run: + $ python3 -m build --sdist + $ ./ci/determine_sdist_name.py +""" +import os +from pathlib import Path +import sys + + +paths = [p.name for p in Path("dist").glob("*.tar.gz")] +if len(paths) != 1: + sys.exit(f"Only a single sdist is supported, but found: {paths}") + +print(paths[0]) +with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"SDIST_NAME={paths[0]}\n") diff --git a/ci/silence b/ci/silence deleted file mode 100755 index 4889e5d1bd58..000000000000 --- a/ci/silence +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# Run a command, hiding its standard output and error if its exit -# status is zero. - -stdout=$(mktemp -t stdout) || exit 1 -stderr=$(mktemp -t stderr) || exit 1 -"$@" >$stdout 2>$stderr -code=$? -if [[ $code != 0 ]]; then - cat $stdout - cat $stderr >&2 - exit $code -fi