From 06ab1886da877001b1e1a0f6ff742f1000879836 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 21 Jan 2021 13:49:52 -0500 Subject: [PATCH 01/19] CI Adds manylinux1 [cd build] --- .github/workflows/wheels.yml | 12 ++++++++++++ build_tools/github/check_wheels.py | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 17726ec9a112b..2bc47c07540e6 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -61,9 +61,19 @@ jobs: - os: ubuntu-latest bitness: 64 platform_id: manylinux_x86_64 + manylinux_image: manylinux2010 + - os: ubuntu-latest + bitness: 64 + platform_id: manylinux_x86_64 + manylinux_image: manylinux1 + - os: ubuntu-latest + bitness: 32 + platform_id: manylinux_i686 + manylinux_image: manylinux2010 - os: ubuntu-latest bitness: 32 platform_id: manylinux_i686 + manylinux_image: manylinux1 - os: macos-latest bitness: 64 platform_id: macosx_x86_64 @@ -88,6 +98,8 @@ jobs: SKLEARN_BUILD_PARALLEL=3 MACOSX_DEPLOYMENT_TARGET=10.13 CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl diff --git a/build_tools/github/check_wheels.py b/build_tools/github/check_wheels.py index 64cebe3b6b0c4..9268877248e1e 100644 --- a/build_tools/github/check_wheels.py +++ b/build_tools/github/check_wheels.py @@ -13,9 +13,9 @@ # For each python version we have: 5 wheels # 1 osx wheel (x86_64) -# 2 linux wheel (i686 + x86_64) +# 4 linux wheel (i686 + x86_64) * (manylinux1 + manylinux2010) # 2 windows wheel (win32 + wind_amd64) -n_wheels = 5 * n_python_versions +n_wheels = 7 * n_python_versions # plus one more for the sdist n_wheels += 1 From cafc8886d74f0a14a9314cde253f4784f00add61 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 21 Jan 2021 14:20:42 -0500 Subject: [PATCH 02/19] WIP Check build matrix [cd build] --- .github/workflows/wheels.yml | 198 ++++++++++++++++++----------------- 1 file changed, 100 insertions(+), 98 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2bc47c07540e6..577129d4a33a4 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -38,7 +38,7 @@ jobs: # Build the wheels for Linux, Windows and macOS for Python 3.6 and newer build_wheels: - name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }} + name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} runs-on: ${{ matrix.os }} needs: check_build_trigger if: needs.check_build_trigger.outputs.build @@ -47,17 +47,19 @@ jobs: # Ensure that a wheel builder finishes even if another fails fail-fast: false matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - python: [36, 37, 38, 39] + # os: [windows-latest, ubuntu-latest, macos-latest] + os: [ubuntu-latest] + # python: [36, 37, 38, 39] + python: [39] bitness: [32, 64] include: # Run 32 and 64 bit version in parallel for Linux and Windows - - os: windows-latest - bitness: 64 - platform_id: win_amd64 - - os: windows-latest - bitness: 32 - platform_id: win32 + # - os: windows-latest + # bitness: 64 + # platform_id: win_amd64 + # - os: windows-latest + # bitness: 32 + # platform_id: win32 - os: ubuntu-latest bitness: 64 platform_id: manylinux_x86_64 @@ -74,100 +76,100 @@ jobs: bitness: 32 platform_id: manylinux_i686 manylinux_image: manylinux1 - - os: macos-latest - bitness: 64 - platform_id: macosx_x86_64 - exclude: - - os: macos-latest - bitness: 32 - - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v1 - - - name: Setup Python - uses: actions/setup-python@v2 - - - name: Build and test wheels - env: - CONFTEST_PATH: ${{ github.workspace }}/conftest.py - CONFTEST_NAME: conftest.py - CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 - OPENBLAS_NUM_THREADS=2 - SKLEARN_SKIP_NETWORK_TESTS=1 - SKLEARN_BUILD_PARALLEL=3 - MACOSX_DEPLOYMENT_TARGET=10.13 - CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} - CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} - CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} - CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} - CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} - CIBW_TEST_REQUIRES: pytest pandas threadpoolctl - CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh - CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} - - run: bash build_tools/github/build_wheels.sh - - - name: Store artifacts - uses: actions/upload-artifact@v2 - with: - path: wheelhouse/*.whl - - # Build the source distribution under Linux - build_sdist: - name: Source distribution - runs-on: ubuntu-latest - needs: check_build_trigger - if: needs.check_build_trigger.outputs.build - - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v1 - - - name: Setup Python - uses: actions/setup-python@v2 - - - name: Build source distribution - run: bash build_tools/github/build_source.sh - env: - SKLEARN_BUILD_PARALLEL: 3 - - - name: Test source distribution - run: bash build_tools/github/test_source.sh - env: - OMP_NUM_THREADS: 2 - OPENBLAS_NUM_THREADS: 2 - SKLEARN_SKIP_NETWORK_TESTS: 1 - - - name: Store artifacts - uses: actions/upload-artifact@v2 - with: - path: dist/*.tar.gz - - # Upload the wheels and the source distribution - upload_anaconda: - name: Upload to Anaconda - runs-on: ubuntu-latest - needs: [build_wheels, build_sdist] - # The artifacts cannot be uploaded on PRs - if: github.event_name != 'pull_request' + # - os: macos-latest + # bitness: 64 + # platform_id: macosx_x86_64 + # exclude: + # - os: macos-latest + # bitness: 32 steps: - name: Checkout scikit-learn uses: actions/checkout@v1 - - name: Download artifacts - uses: actions/download-artifact@v2 - with: - path: dist - - name: Setup Python uses: actions/setup-python@v2 - - name: Upload artifacts - env: - # Secret variables need to be mapped to environment variables explicitly - SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} - SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} - # Force a replacement if the remote file already exists - run: bash build_tools/github/upload_anaconda.sh + # - name: Build and test wheels + # env: + # CONFTEST_PATH: ${{ github.workspace }}/conftest.py + # CONFTEST_NAME: conftest.py + # CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 + # OPENBLAS_NUM_THREADS=2 + # SKLEARN_SKIP_NETWORK_TESTS=1 + # SKLEARN_BUILD_PARALLEL=3 + # MACOSX_DEPLOYMENT_TARGET=10.13 + # CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + # CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} + # CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} + # CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} + # CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} + # CIBW_TEST_REQUIRES: pytest pandas threadpoolctl + # CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh + # CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} + + # run: bash build_tools/github/build_wheels.sh + + # - name: Store artifacts + # uses: actions/upload-artifact@v2 + # with: + # path: wheelhouse/*.whl + + # # Build the source distribution under Linux + # build_sdist: + # name: Source distribution + # runs-on: ubuntu-latest + # needs: check_build_trigger + # if: needs.check_build_trigger.outputs.build + + # steps: + # - name: Checkout scikit-learn + # uses: actions/checkout@v1 + + # - name: Setup Python + # uses: actions/setup-python@v2 + + # - name: Build source distribution + # run: bash build_tools/github/build_source.sh + # env: + # SKLEARN_BUILD_PARALLEL: 3 + + # - name: Test source distribution + # run: bash build_tools/github/test_source.sh + # env: + # OMP_NUM_THREADS: 2 + # OPENBLAS_NUM_THREADS: 2 + # SKLEARN_SKIP_NETWORK_TESTS: 1 + + # - name: Store artifacts + # uses: actions/upload-artifact@v2 + # with: + # path: dist/*.tar.gz + + # # Upload the wheels and the source distribution + # upload_anaconda: + # name: Upload to Anaconda + # runs-on: ubuntu-latest + # needs: [build_wheels, build_sdist] + # # The artifacts cannot be uploaded on PRs + # if: github.event_name != 'pull_request' + + # steps: + # - name: Checkout scikit-learn + # uses: actions/checkout@v1 + + # - name: Download artifacts + # uses: actions/download-artifact@v2 + # with: + # path: dist + + # - name: Setup Python + # uses: actions/setup-python@v2 + + # - name: Upload artifacts + # env: + # # Secret variables need to be mapped to environment variables explicitly + # SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} + # SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} + # # Force a replacement if the remote file already exists + # run: bash build_tools/github/upload_anaconda.sh From ae2273f6f91b22fe553472aa125d0dfc08071214 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 21 Jan 2021 14:35:10 -0500 Subject: [PATCH 03/19] WIP Check build matrix [cd build] --- .github/workflows/wheels.yml | 107 ++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 53 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 577129d4a33a4..0645d374939e8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -36,9 +36,9 @@ jobs: name: Check build trigger run: bash build_tools/github/check_build_trigger.sh - # Build the wheels for Linux, Windows and macOS for Python 3.6 and newer - build_wheels: - name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} + # Build the wheels for Windows and macOS for Python 3.6 and newer + build_wheels_windows_osx: + name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }} runs-on: ${{ matrix.os }} needs: check_build_trigger if: needs.check_build_trigger.outputs.build @@ -47,74 +47,75 @@ jobs: # Ensure that a wheel builder finishes even if another fails fail-fast: false matrix: - # os: [windows-latest, ubuntu-latest, macos-latest] - os: [ubuntu-latest] - # python: [36, 37, 38, 39] - python: [39] + os: [windows-latest, macos-latest] + python: [36, 37, 38, 39] bitness: [32, 64] include: - # Run 32 and 64 bit version in parallel for Linux and Windows - # - os: windows-latest - # bitness: 64 - # platform_id: win_amd64 - # - os: windows-latest - # bitness: 32 - # platform_id: win32 - - os: ubuntu-latest - bitness: 64 - platform_id: manylinux_x86_64 - manylinux_image: manylinux2010 - - os: ubuntu-latest + # Run 32 and 64 bit version in parallel for Windows + - os: windows-latest bitness: 64 - platform_id: manylinux_x86_64 - manylinux_image: manylinux1 - - os: ubuntu-latest + platform_id: win_amd64 + - os: windows-latest bitness: 32 - platform_id: manylinux_i686 - manylinux_image: manylinux2010 - - os: ubuntu-latest + platform_id: win32 + - os: macos-latest + bitness: 64 + platform_id: macosx_x86_64 + exclude: + - os: macos-latest bitness: 32 - platform_id: manylinux_i686 - manylinux_image: manylinux1 - # - os: macos-latest - # bitness: 64 - # platform_id: macosx_x86_64 - # exclude: - # - os: macos-latest - # bitness: 32 - steps: + steps: &build_wheel_steps - name: Checkout scikit-learn uses: actions/checkout@v1 - name: Setup Python uses: actions/setup-python@v2 - # - name: Build and test wheels - # env: - # CONFTEST_PATH: ${{ github.workspace }}/conftest.py - # CONFTEST_NAME: conftest.py - # CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 - # OPENBLAS_NUM_THREADS=2 - # SKLEARN_SKIP_NETWORK_TESTS=1 - # SKLEARN_BUILD_PARALLEL=3 - # MACOSX_DEPLOYMENT_TARGET=10.13 - # CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} - # CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} - # CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} - # CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} - # CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} - # CIBW_TEST_REQUIRES: pytest pandas threadpoolctl - # CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh - # CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} - - # run: bash build_tools/github/build_wheels.sh + - name: Build and test wheels + env: + CONFTEST_PATH: ${{ github.workspace }}/conftest.py + CONFTEST_NAME: conftest.py + CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 + OPENBLAS_NUM_THREADS=2 + SKLEARN_SKIP_NETWORK_TESTS=1 + SKLEARN_BUILD_PARALLEL=3 + MACOSX_DEPLOYMENT_TARGET=10.13 + CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} + CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} + CIBW_TEST_REQUIRES: pytest pandas threadpoolctl + CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh + CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} + + # run: bash build_tools/github/build_wheels.sh + run: echo "Ran" # - name: Store artifacts # uses: actions/upload-artifact@v2 # with: # path: wheelhouse/*.whl + # Build the wheels for Linux + build_wheels_linux: + name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} + runs-on: ${{ matrix.os }} + needs: check_build_trigger + if: needs.check_build_trigger.outputs.build + + strategy: + # Ensure that a wheel builder finishes even if another fails + fail-fast: false + matrix: + os: [ubuntu-latest] + python: [36, 37, 38, 39] + bitness: [32, 64] + manylinux_image: [manylinux1, manylinux2010] + + <<: *build_wheel_steps + # # Build the source distribution under Linux # build_sdist: # name: Source distribution From 716cc79fa0607960ea0aeb66e909352601478bf3 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 21 Jan 2021 14:44:57 -0500 Subject: [PATCH 04/19] WIP Check build matrix [cd build] [ci skip] --- .github/workflows/wheels.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 0645d374939e8..02226ce62c57b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -99,22 +99,22 @@ jobs: # path: wheelhouse/*.whl # Build the wheels for Linux - build_wheels_linux: - name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} - runs-on: ${{ matrix.os }} - needs: check_build_trigger - if: needs.check_build_trigger.outputs.build + # build_wheels_linux: + # name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} + # runs-on: ${{ matrix.os }} + # needs: check_build_trigger + # if: needs.check_build_trigger.outputs.build - strategy: - # Ensure that a wheel builder finishes even if another fails - fail-fast: false - matrix: - os: [ubuntu-latest] - python: [36, 37, 38, 39] - bitness: [32, 64] - manylinux_image: [manylinux1, manylinux2010] + # strategy: + # # Ensure that a wheel builder finishes even if another fails + # fail-fast: false + # matrix: + # os: [ubuntu-latest] + # python: [36, 37, 38, 39] + # bitness: [32, 64] + # manylinux_image: [manylinux1, manylinux2010] - <<: *build_wheel_steps + # <<: *build_wheel_steps # # Build the source distribution under Linux # build_sdist: From c3cced03cb20d713f032e1674263e32195ada1ce Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 21 Jan 2021 14:48:25 -0500 Subject: [PATCH 05/19] WIP Check build matrix [cd build] [ci skip] --- .github/workflows/wheels.yml | 56 +++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 02226ce62c57b..bc84f4cc509cb 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -65,7 +65,7 @@ jobs: - os: macos-latest bitness: 32 - steps: &build_wheel_steps + steps: - name: Checkout scikit-learn uses: actions/checkout@v1 @@ -82,8 +82,6 @@ jobs: SKLEARN_BUILD_PARALLEL=3 MACOSX_DEPLOYMENT_TARGET=10.13 CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} - CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} - CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl @@ -99,22 +97,46 @@ jobs: # path: wheelhouse/*.whl # Build the wheels for Linux - # build_wheels_linux: - # name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} - # runs-on: ${{ matrix.os }} - # needs: check_build_trigger - # if: needs.check_build_trigger.outputs.build + build_wheels_linux: + name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} + runs-on: ${{ matrix.os }} + needs: check_build_trigger + if: needs.check_build_trigger.outputs.build - # strategy: - # # Ensure that a wheel builder finishes even if another fails - # fail-fast: false - # matrix: - # os: [ubuntu-latest] - # python: [36, 37, 38, 39] - # bitness: [32, 64] - # manylinux_image: [manylinux1, manylinux2010] + strategy: + # Ensure that a wheel builder finishes even if another fails + fail-fast: false + matrix: + os: [ubuntu-latest] + python: [36, 37, 38, 39] + bitness: [32, 64] + manylinux_image: [manylinux1, manylinux2010] + + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v1 + + - name: Setup Python + uses: actions/setup-python@v2 - # <<: *build_wheel_steps + - name: Build and test wheels + env: + CONFTEST_PATH: ${{ github.workspace }}/conftest.py + CONFTEST_NAME: conftest.py + CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 + OPENBLAS_NUM_THREADS=2 + SKLEARN_SKIP_NETWORK_TESTS=1 + SKLEARN_BUILD_PARALLEL=3 + MACOSX_DEPLOYMENT_TARGET=10.13 + CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} + CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} + CIBW_TEST_REQUIRES: pytest pandas threadpoolctl + CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh + CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} + + # run: bash build_tools/github/build_wheels.sh + run: echo "Ran" # # Build the source distribution under Linux # build_sdist: From 5fb3dc2c8101094a1fff617fd072f162b0360f71 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 21 Jan 2021 14:52:39 -0500 Subject: [PATCH 06/19] WIP Check build matrix [cd build] [ci skip] --- .github/workflows/wheels.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index bc84f4cc509cb..3c0d8e7ab3320 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -111,6 +111,13 @@ jobs: python: [36, 37, 38, 39] bitness: [32, 64] manylinux_image: [manylinux1, manylinux2010] + include: + - os: ubuntu-latest + bitness: 64 + platform_id: manylinux_x86_64 + - os: ubuntu-latest + bitness: 32 + platform_id: manylinux_i686 steps: - name: Checkout scikit-learn @@ -129,6 +136,8 @@ jobs: SKLEARN_BUILD_PARALLEL=3 MACOSX_DEPLOYMENT_TARGET=10.13 CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl From c59e62e525fe01499b44802e71fa7ad335397049 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 21 Jan 2021 15:08:54 -0500 Subject: [PATCH 07/19] ENH Adds manylinux1 wheels [cd build] [ci skip] --- .github/workflows/wheels.yml | 146 ++++++++++++++--------------- build_tools/github/check_wheels.py | 5 +- 2 files changed, 76 insertions(+), 75 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 3c0d8e7ab3320..65602178612a9 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -88,18 +88,17 @@ jobs: CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} - # run: bash build_tools/github/build_wheels.sh - run: echo "Ran" + run: bash build_tools/github/build_wheels.sh - # - name: Store artifacts - # uses: actions/upload-artifact@v2 - # with: - # path: wheelhouse/*.whl + - name: Store artifacts + uses: actions/upload-artifact@v2 + with: + path: wheelhouse/*.whl # Build the wheels for Linux build_wheels_linux: name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest needs: check_build_trigger if: needs.check_build_trigger.outputs.build @@ -107,16 +106,13 @@ jobs: # Ensure that a wheel builder finishes even if another fails fail-fast: false matrix: - os: [ubuntu-latest] python: [36, 37, 38, 39] bitness: [32, 64] manylinux_image: [manylinux1, manylinux2010] include: - - os: ubuntu-latest - bitness: 64 + - bitness: 64 platform_id: manylinux_x86_64 - - os: ubuntu-latest - bitness: 32 + - bitness: 32 platform_id: manylinux_i686 steps: @@ -144,64 +140,68 @@ jobs: CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} - # run: bash build_tools/github/build_wheels.sh - run: echo "Ran" - - # # Build the source distribution under Linux - # build_sdist: - # name: Source distribution - # runs-on: ubuntu-latest - # needs: check_build_trigger - # if: needs.check_build_trigger.outputs.build - - # steps: - # - name: Checkout scikit-learn - # uses: actions/checkout@v1 - - # - name: Setup Python - # uses: actions/setup-python@v2 - - # - name: Build source distribution - # run: bash build_tools/github/build_source.sh - # env: - # SKLEARN_BUILD_PARALLEL: 3 - - # - name: Test source distribution - # run: bash build_tools/github/test_source.sh - # env: - # OMP_NUM_THREADS: 2 - # OPENBLAS_NUM_THREADS: 2 - # SKLEARN_SKIP_NETWORK_TESTS: 1 - - # - name: Store artifacts - # uses: actions/upload-artifact@v2 - # with: - # path: dist/*.tar.gz - - # # Upload the wheels and the source distribution - # upload_anaconda: - # name: Upload to Anaconda - # runs-on: ubuntu-latest - # needs: [build_wheels, build_sdist] - # # The artifacts cannot be uploaded on PRs - # if: github.event_name != 'pull_request' - - # steps: - # - name: Checkout scikit-learn - # uses: actions/checkout@v1 - - # - name: Download artifacts - # uses: actions/download-artifact@v2 - # with: - # path: dist - - # - name: Setup Python - # uses: actions/setup-python@v2 - - # - name: Upload artifacts - # env: - # # Secret variables need to be mapped to environment variables explicitly - # SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} - # SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} - # # Force a replacement if the remote file already exists - # run: bash build_tools/github/upload_anaconda.sh + run: bash build_tools/github/build_wheels.sh + + - name: Store artifacts + uses: actions/upload-artifact@v2 + with: + path: wheelhouse/*.whl + + # Build the source distribution under Linux + build_sdist: + name: Source distribution + runs-on: ubuntu-latest + needs: check_build_trigger + if: needs.check_build_trigger.outputs.build + + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v1 + + - name: Setup Python + uses: actions/setup-python@v2 + + - name: Build source distribution + run: bash build_tools/github/build_source.sh + env: + SKLEARN_BUILD_PARALLEL: 3 + + - name: Test source distribution + run: bash build_tools/github/test_source.sh + env: + OMP_NUM_THREADS: 2 + OPENBLAS_NUM_THREADS: 2 + SKLEARN_SKIP_NETWORK_TESTS: 1 + + - name: Store artifacts + uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz + + # Upload the wheels and the source distribution + upload_anaconda: + name: Upload to Anaconda + runs-on: ubuntu-latest + needs: [build_wheels_windows_osx, build_wheels_linux, build_sdist] + # The artifacts cannot be uploaded on PRs + if: github.event_name != 'pull_request' + + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v1 + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + path: dist + + - name: Setup Python + uses: actions/setup-python@v2 + + - name: Upload artifacts + env: + # Secret variables need to be mapped to environment variables explicitly + SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} + SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} + # Force a replacement if the remote file already exists + run: bash build_tools/github/upload_anaconda.sh diff --git a/build_tools/github/check_wheels.py b/build_tools/github/check_wheels.py index 9268877248e1e..a0668270a43cf 100644 --- a/build_tools/github/check_wheels.py +++ b/build_tools/github/check_wheels.py @@ -8,10 +8,11 @@ with gh_wheel_path.open('r') as f: wheel_config = yaml.safe_load(f) -build_matrix = wheel_config['jobs']['build_wheels']['strategy']['matrix'] +build_matrix = wheel_config['jobs'][ + 'build_wheels_windows_osx']['strategy']['matrix'] n_python_versions = len(build_matrix['python']) -# For each python version we have: 5 wheels +# For each python version we have: 7 wheels # 1 osx wheel (x86_64) # 4 linux wheel (i686 + x86_64) * (manylinux1 + manylinux2010) # 2 windows wheel (win32 + wind_amd64) From b97bcc1549ad2037dddf5d239c680d488b185a2e Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 22 Jan 2021 08:51:54 -0500 Subject: [PATCH 08/19] ENH Uses suggestion --- .github/workflows/wheels.yml | 65 +++++++----------------------------- 1 file changed, 12 insertions(+), 53 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 65602178612a9..689bd09e28de8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -37,8 +37,8 @@ jobs: run: bash build_tools/github/check_build_trigger.sh # Build the wheels for Windows and macOS for Python 3.6 and newer - build_wheels_windows_osx: - name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }} + build_wheels: + name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} runs-on: ${{ matrix.os }} needs: check_build_trigger if: needs.check_build_trigger.outputs.build @@ -50,6 +50,7 @@ jobs: os: [windows-latest, macos-latest] python: [36, 37, 38, 39] bitness: [32, 64] + manylinux_image: [manylinux1, manylinux2010] include: # Run 32 and 64 bit version in parallel for Windows - os: windows-latest @@ -64,56 +65,14 @@ jobs: exclude: - os: macos-latest bitness: 32 - - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v1 - - - name: Setup Python - uses: actions/setup-python@v2 - - - name: Build and test wheels - env: - CONFTEST_PATH: ${{ github.workspace }}/conftest.py - CONFTEST_NAME: conftest.py - CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 - OPENBLAS_NUM_THREADS=2 - SKLEARN_SKIP_NETWORK_TESTS=1 - SKLEARN_BUILD_PARALLEL=3 - MACOSX_DEPLOYMENT_TARGET=10.13 - CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} - CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} - CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} - CIBW_TEST_REQUIRES: pytest pandas threadpoolctl - CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh - CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} - - run: bash build_tools/github/build_wheels.sh - - - name: Store artifacts - uses: actions/upload-artifact@v2 - with: - path: wheelhouse/*.whl - - # Build the wheels for Linux - build_wheels_linux: - name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} - runs-on: ubuntu-latest - needs: check_build_trigger - if: needs.check_build_trigger.outputs.build - - strategy: - # Ensure that a wheel builder finishes even if another fails - fail-fast: false - matrix: - python: [36, 37, 38, 39] - bitness: [32, 64] - manylinux_image: [manylinux1, manylinux2010] - include: - - bitness: 64 - platform_id: manylinux_x86_64 - - bitness: 32 - platform_id: manylinux_i686 + - os: windows-latest + manylinux_image: manylinux_1 + - os: windows-latest + manylinux_image: manylinux_2010 + - os: macos-latest + manylinux_image: manylinux_1 + - os: macos-latest + manylinux_image: manylinux_2010 steps: - name: Checkout scikit-learn @@ -182,7 +141,7 @@ jobs: upload_anaconda: name: Upload to Anaconda runs-on: ubuntu-latest - needs: [build_wheels_windows_osx, build_wheels_linux, build_sdist] + needs: [build_wheels, build_sdist] # The artifacts cannot be uploaded on PRs if: github.event_name != 'pull_request' From b6a1e52d83d9fae8d7794eeb9171bbb1e1709613 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 22 Jan 2021 08:53:39 -0500 Subject: [PATCH 09/19] WIP Check build matrix [cd build] [ci skip] --- .github/workflows/wheels.yml | 141 ++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 67 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 689bd09e28de8..0050f4f299fe7 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -36,7 +36,7 @@ jobs: name: Check build trigger run: bash build_tools/github/check_build_trigger.sh - # Build the wheels for Windows and macOS for Python 3.6 and newer + # Build the wheels for Linux, Windows and macOS for Python 3.6 and newer build_wheels: name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} runs-on: ${{ matrix.os }} @@ -47,7 +47,7 @@ jobs: # Ensure that a wheel builder finishes even if another fails fail-fast: false matrix: - os: [windows-latest, macos-latest] + os: [windows-latest, ubuntu-latest, macos-latest] python: [36, 37, 38, 39] bitness: [32, 64] manylinux_image: [manylinux1, manylinux2010] @@ -59,6 +59,12 @@ jobs: - os: windows-latest bitness: 32 platform_id: win32 + - os: ubuntu-latest + bitness: 64 + platform_id: manylinux_x86_64 + - os: ubuntu-latest + bitness: 32 + platform_id: manylinux_i686 - os: macos-latest bitness: 64 platform_id: macosx_x86_64 @@ -99,68 +105,69 @@ jobs: CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} - run: bash build_tools/github/build_wheels.sh - - - name: Store artifacts - uses: actions/upload-artifact@v2 - with: - path: wheelhouse/*.whl - - # Build the source distribution under Linux - build_sdist: - name: Source distribution - runs-on: ubuntu-latest - needs: check_build_trigger - if: needs.check_build_trigger.outputs.build - - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v1 - - - name: Setup Python - uses: actions/setup-python@v2 - - - name: Build source distribution - run: bash build_tools/github/build_source.sh - env: - SKLEARN_BUILD_PARALLEL: 3 - - - name: Test source distribution - run: bash build_tools/github/test_source.sh - env: - OMP_NUM_THREADS: 2 - OPENBLAS_NUM_THREADS: 2 - SKLEARN_SKIP_NETWORK_TESTS: 1 - - - name: Store artifacts - uses: actions/upload-artifact@v2 - with: - path: dist/*.tar.gz - - # Upload the wheels and the source distribution - upload_anaconda: - name: Upload to Anaconda - runs-on: ubuntu-latest - needs: [build_wheels, build_sdist] - # The artifacts cannot be uploaded on PRs - if: github.event_name != 'pull_request' - - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v1 - - - name: Download artifacts - uses: actions/download-artifact@v2 - with: - path: dist - - - name: Setup Python - uses: actions/setup-python@v2 - - - name: Upload artifacts - env: - # Secret variables need to be mapped to environment variables explicitly - SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} - SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} - # Force a replacement if the remote file already exists - run: bash build_tools/github/upload_anaconda.sh + # run: bash build_tools/github/build_wheels.sh + run: Echo "ran" + + # - name: Store artifacts + # uses: actions/upload-artifact@v2 + # with: + # path: wheelhouse/*.whl + + # # Build the source distribution under Linux + # build_sdist: + # name: Source distribution + # runs-on: ubuntu-latest + # needs: check_build_trigger + # if: needs.check_build_trigger.outputs.build + + # steps: + # - name: Checkout scikit-learn + # uses: actions/checkout@v1 + + # - name: Setup Python + # uses: actions/setup-python@v2 + + # - name: Build source distribution + # run: bash build_tools/github/build_source.sh + # env: + # SKLEARN_BUILD_PARALLEL: 3 + + # - name: Test source distribution + # run: bash build_tools/github/test_source.sh + # env: + # OMP_NUM_THREADS: 2 + # OPENBLAS_NUM_THREADS: 2 + # SKLEARN_SKIP_NETWORK_TESTS: 1 + + # - name: Store artifacts + # uses: actions/upload-artifact@v2 + # with: + # path: dist/*.tar.gz + + # # Upload the wheels and the source distribution + # upload_anaconda: + # name: Upload to Anaconda + # runs-on: ubuntu-latest + # needs: [build_wheels, build_sdist] + # # The artifacts cannot be uploaded on PRs + # if: github.event_name != 'pull_request' + + # steps: + # - name: Checkout scikit-learn + # uses: actions/checkout@v1 + + # - name: Download artifacts + # uses: actions/download-artifact@v2 + # with: + # path: dist + + # - name: Setup Python + # uses: actions/setup-python@v2 + + # - name: Upload artifacts + # env: + # # Secret variables need to be mapped to environment variables explicitly + # SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} + # SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} + # # Force a replacement if the remote file already exists + # run: bash build_tools/github/upload_anaconda.sh From f3102207406d7dbbd0517c7e06b9d057df42062f Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 22 Jan 2021 08:58:40 -0500 Subject: [PATCH 10/19] WIP Check build matrix [cd build] [ci skip] --- .github/workflows/wheels.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 0050f4f299fe7..b133f69c03c4d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -71,14 +71,6 @@ jobs: exclude: - os: macos-latest bitness: 32 - - os: windows-latest - manylinux_image: manylinux_1 - - os: windows-latest - manylinux_image: manylinux_2010 - - os: macos-latest - manylinux_image: manylinux_1 - - os: macos-latest - manylinux_image: manylinux_2010 steps: - name: Checkout scikit-learn From 74382acdc9f1ace615c66ceb8cdadd1909c4a3d6 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 22 Jan 2021 09:01:10 -0500 Subject: [PATCH 11/19] WIP Check build matrix [cd build] [ci skip] --- .github/workflows/wheels.yml | 68 ++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b133f69c03c4d..79be419891881 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -71,6 +71,14 @@ jobs: exclude: - os: macos-latest bitness: 32 + - os: windows-latest + manylinux_image: manylinux_1 + - os: windows-latest + manylinux_image: manylinux_2010 + - os: macos-latest + manylinux_image: manylinux_1 + - os: macos-latest + manylinux_image: manylinux_2010 steps: - name: Checkout scikit-learn @@ -100,43 +108,43 @@ jobs: # run: bash build_tools/github/build_wheels.sh run: Echo "ran" - # - name: Store artifacts - # uses: actions/upload-artifact@v2 - # with: - # path: wheelhouse/*.whl + # - name: Store artifacts + # uses: actions/upload-artifact@v2 + # with: + # path: wheelhouse/*.whl - # # Build the source distribution under Linux - # build_sdist: - # name: Source distribution - # runs-on: ubuntu-latest - # needs: check_build_trigger - # if: needs.check_build_trigger.outputs.build + # Build the source distribution under Linux + build_sdist: + name: Source distribution + runs-on: ubuntu-latest + needs: check_build_trigger + if: needs.check_build_trigger.outputs.build - # steps: - # - name: Checkout scikit-learn - # uses: actions/checkout@v1 + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v1 - # - name: Setup Python - # uses: actions/setup-python@v2 + - name: Setup Python + uses: actions/setup-python@v2 - # - name: Build source distribution - # run: bash build_tools/github/build_source.sh - # env: - # SKLEARN_BUILD_PARALLEL: 3 + - name: Build source distribution + run: bash build_tools/github/build_source.sh + env: + SKLEARN_BUILD_PARALLEL: 3 - # - name: Test source distribution - # run: bash build_tools/github/test_source.sh - # env: - # OMP_NUM_THREADS: 2 - # OPENBLAS_NUM_THREADS: 2 - # SKLEARN_SKIP_NETWORK_TESTS: 1 + - name: Test source distribution + run: bash build_tools/github/test_source.sh + env: + OMP_NUM_THREADS: 2 + OPENBLAS_NUM_THREADS: 2 + SKLEARN_SKIP_NETWORK_TESTS: 1 - # - name: Store artifacts - # uses: actions/upload-artifact@v2 - # with: - # path: dist/*.tar.gz + - name: Store artifacts + uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz - # # Upload the wheels and the source distribution + # Upload the wheels and the source distribution # upload_anaconda: # name: Upload to Anaconda # runs-on: ubuntu-latest From 6aced2fe8b3662879574b19c84e331e7a217cbbb Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 22 Jan 2021 09:08:46 -0500 Subject: [PATCH 12/19] WIP Check build matrix [cd build] [ci skip] --- .github/workflows/wheels.yml | 54 ++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 79be419891881..27f90ce8cd637 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -52,7 +52,7 @@ jobs: bitness: [32, 64] manylinux_image: [manylinux1, manylinux2010] include: - # Run 32 and 64 bit version in parallel for Windows + # Run 32 and 64 bit version in parallel for Linux and Windows - os: windows-latest bitness: 64 platform_id: win_amd64 @@ -145,29 +145,29 @@ jobs: path: dist/*.tar.gz # Upload the wheels and the source distribution - # upload_anaconda: - # name: Upload to Anaconda - # runs-on: ubuntu-latest - # needs: [build_wheels, build_sdist] - # # The artifacts cannot be uploaded on PRs - # if: github.event_name != 'pull_request' - - # steps: - # - name: Checkout scikit-learn - # uses: actions/checkout@v1 - - # - name: Download artifacts - # uses: actions/download-artifact@v2 - # with: - # path: dist - - # - name: Setup Python - # uses: actions/setup-python@v2 - - # - name: Upload artifacts - # env: - # # Secret variables need to be mapped to environment variables explicitly - # SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} - # SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} - # # Force a replacement if the remote file already exists - # run: bash build_tools/github/upload_anaconda.sh + upload_anaconda: + name: Upload to Anaconda + runs-on: ubuntu-latest + needs: [build_wheels, build_sdist] + # The artifacts cannot be uploaded on PRs + if: github.event_name != 'pull_request' + + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v1 + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + path: dist + + - name: Setup Python + uses: actions/setup-python@v2 + + - name: Upload artifacts + env: + # Secret variables need to be mapped to environment variables explicitly + SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} + SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} + # Force a replacement if the remote file already exists + run: bash build_tools/github/upload_anaconda.sh From d8deff4774a8894f2a2ba2bfe3d6381781e82624 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 22 Jan 2021 09:11:37 -0500 Subject: [PATCH 13/19] WIP Check build matrix [cd build] [ci skip] --- .github/workflows/wheels.yml | 167 +++++++++++++++++------------------ 1 file changed, 83 insertions(+), 84 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 27f90ce8cd637..4cfbb6ca14f21 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -87,87 +87,86 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 - - name: Build and test wheels - env: - CONFTEST_PATH: ${{ github.workspace }}/conftest.py - CONFTEST_NAME: conftest.py - CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 - OPENBLAS_NUM_THREADS=2 - SKLEARN_SKIP_NETWORK_TESTS=1 - SKLEARN_BUILD_PARALLEL=3 - MACOSX_DEPLOYMENT_TARGET=10.13 - CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} - CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} - CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} - CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} - CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} - CIBW_TEST_REQUIRES: pytest pandas threadpoolctl - CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh - CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} - - # run: bash build_tools/github/build_wheels.sh - run: Echo "ran" - - # - name: Store artifacts - # uses: actions/upload-artifact@v2 - # with: - # path: wheelhouse/*.whl - - # Build the source distribution under Linux - build_sdist: - name: Source distribution - runs-on: ubuntu-latest - needs: check_build_trigger - if: needs.check_build_trigger.outputs.build - - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v1 - - - name: Setup Python - uses: actions/setup-python@v2 - - - name: Build source distribution - run: bash build_tools/github/build_source.sh - env: - SKLEARN_BUILD_PARALLEL: 3 - - - name: Test source distribution - run: bash build_tools/github/test_source.sh - env: - OMP_NUM_THREADS: 2 - OPENBLAS_NUM_THREADS: 2 - SKLEARN_SKIP_NETWORK_TESTS: 1 - - - name: Store artifacts - uses: actions/upload-artifact@v2 - with: - path: dist/*.tar.gz - - # Upload the wheels and the source distribution - upload_anaconda: - name: Upload to Anaconda - runs-on: ubuntu-latest - needs: [build_wheels, build_sdist] - # The artifacts cannot be uploaded on PRs - if: github.event_name != 'pull_request' - - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v1 - - - name: Download artifacts - uses: actions/download-artifact@v2 - with: - path: dist - - - name: Setup Python - uses: actions/setup-python@v2 - - - name: Upload artifacts - env: - # Secret variables need to be mapped to environment variables explicitly - SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} - SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} - # Force a replacement if the remote file already exists - run: bash build_tools/github/upload_anaconda.sh + # - name: Build and test wheels + # env: + # CONFTEST_PATH: ${{ github.workspace }}/conftest.py + # CONFTEST_NAME: conftest.py + # CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 + # OPENBLAS_NUM_THREADS=2 + # SKLEARN_SKIP_NETWORK_TESTS=1 + # SKLEARN_BUILD_PARALLEL=3 + # MACOSX_DEPLOYMENT_TARGET=10.13 + # CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + # CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} + # CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} + # CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} + # CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} + # CIBW_TEST_REQUIRES: pytest pandas threadpoolctl + # CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh + # CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} + + # run: bash build_tools/github/build_wheels.sh + + # - name: Store artifacts + # uses: actions/upload-artifact@v2 + # with: + # path: wheelhouse/*.whl + + # # Build the source distribution under Linux + # build_sdist: + # name: Source distribution + # runs-on: ubuntu-latest + # needs: check_build_trigger + # if: needs.check_build_trigger.outputs.build + + # steps: + # - name: Checkout scikit-learn + # uses: actions/checkout@v1 + + # - name: Setup Python + # uses: actions/setup-python@v2 + + # - name: Build source distribution + # run: bash build_tools/github/build_source.sh + # env: + # SKLEARN_BUILD_PARALLEL: 3 + + # - name: Test source distribution + # run: bash build_tools/github/test_source.sh + # env: + # OMP_NUM_THREADS: 2 + # OPENBLAS_NUM_THREADS: 2 + # SKLEARN_SKIP_NETWORK_TESTS: 1 + + # - name: Store artifacts + # uses: actions/upload-artifact@v2 + # with: + # path: dist/*.tar.gz + + # # Upload the wheels and the source distribution + # upload_anaconda: + # name: Upload to Anaconda + # runs-on: ubuntu-latest + # needs: [build_wheels, build_sdist] + # # The artifacts cannot be uploaded on PRs + # if: github.event_name != 'pull_request' + + # steps: + # - name: Checkout scikit-learn + # uses: actions/checkout@v1 + + # - name: Download artifacts + # uses: actions/download-artifact@v2 + # with: + # path: dist + + # - name: Setup Python + # uses: actions/setup-python@v2 + + # - name: Upload artifacts + # env: + # # Secret variables need to be mapped to environment variables explicitly + # SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} + # SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} + # # Force a replacement if the remote file already exists + # run: bash build_tools/github/upload_anaconda.sh From 9f341af924c8688fdf0dd7cb2a86da294f577910 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 22 Jan 2021 09:19:00 -0500 Subject: [PATCH 14/19] WIP Check build matrix [cd build] [ci skip] --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4cfbb6ca14f21..b1ebd85a9d3c3 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -38,7 +38,7 @@ jobs: # Build the wheels for Linux, Windows and macOS for Python 3.6 and newer build_wheels: - name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} + name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }} runs-on: ${{ matrix.os }} needs: check_build_trigger if: needs.check_build_trigger.outputs.build From 8a0f17c3f98c9aae2481a605ddb4916e31589b2f Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 22 Jan 2021 09:22:14 -0500 Subject: [PATCH 15/19] WIP Check build matrix [cd build] --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b1ebd85a9d3c3..4cfbb6ca14f21 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -38,7 +38,7 @@ jobs: # Build the wheels for Linux, Windows and macOS for Python 3.6 and newer build_wheels: - name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }} + name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} runs-on: ${{ matrix.os }} needs: check_build_trigger if: needs.check_build_trigger.outputs.build From 6f5990aa0990f2507ba3759098abdbcf225b064c Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 22 Jan 2021 09:24:24 -0500 Subject: [PATCH 16/19] WIP Check build matrix [cd build] --- .github/workflows/wheels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4cfbb6ca14f21..b95c93ea0946e 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -72,13 +72,13 @@ jobs: - os: macos-latest bitness: 32 - os: windows-latest - manylinux_image: manylinux_1 + manylinux_image: manylinux1 - os: windows-latest - manylinux_image: manylinux_2010 + manylinux_image: manylinux2010 - os: macos-latest - manylinux_image: manylinux_1 + manylinux_image: manylinux1 - os: macos-latest - manylinux_image: manylinux_2010 + manylinux_image: manylinux2010 steps: - name: Checkout scikit-learn From 578e62a2723dfc11f825dc523ee206eff5b5a000 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 22 Jan 2021 09:27:49 -0500 Subject: [PATCH 17/19] WIP Check build matrix [cd build] --- .github/workflows/wheels.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b95c93ea0946e..9e3fe8a60bbbb 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -71,14 +71,12 @@ jobs: exclude: - os: macos-latest bitness: 32 + # Remove one of the manylinux1 from the windows and osx build matrix + # since manylinux_image is not used for these platforms - os: windows-latest manylinux_image: manylinux1 - - os: windows-latest - manylinux_image: manylinux2010 - os: macos-latest manylinux_image: manylinux1 - - os: macos-latest - manylinux_image: manylinux2010 steps: - name: Checkout scikit-learn From 136908f37c7583c490d96174392f8fd34b5302d8 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 22 Jan 2021 09:30:57 -0500 Subject: [PATCH 18/19] ENH Check build matrix [cd build] --- .github/workflows/wheels.yml | 166 ++++++++++++++--------------- build_tools/github/check_wheels.py | 3 +- 2 files changed, 84 insertions(+), 85 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 9e3fe8a60bbbb..493825c49e458 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -85,86 +85,86 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 - # - name: Build and test wheels - # env: - # CONFTEST_PATH: ${{ github.workspace }}/conftest.py - # CONFTEST_NAME: conftest.py - # CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 - # OPENBLAS_NUM_THREADS=2 - # SKLEARN_SKIP_NETWORK_TESTS=1 - # SKLEARN_BUILD_PARALLEL=3 - # MACOSX_DEPLOYMENT_TARGET=10.13 - # CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} - # CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} - # CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} - # CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} - # CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} - # CIBW_TEST_REQUIRES: pytest pandas threadpoolctl - # CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh - # CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} - - # run: bash build_tools/github/build_wheels.sh - - # - name: Store artifacts - # uses: actions/upload-artifact@v2 - # with: - # path: wheelhouse/*.whl - - # # Build the source distribution under Linux - # build_sdist: - # name: Source distribution - # runs-on: ubuntu-latest - # needs: check_build_trigger - # if: needs.check_build_trigger.outputs.build - - # steps: - # - name: Checkout scikit-learn - # uses: actions/checkout@v1 - - # - name: Setup Python - # uses: actions/setup-python@v2 - - # - name: Build source distribution - # run: bash build_tools/github/build_source.sh - # env: - # SKLEARN_BUILD_PARALLEL: 3 - - # - name: Test source distribution - # run: bash build_tools/github/test_source.sh - # env: - # OMP_NUM_THREADS: 2 - # OPENBLAS_NUM_THREADS: 2 - # SKLEARN_SKIP_NETWORK_TESTS: 1 - - # - name: Store artifacts - # uses: actions/upload-artifact@v2 - # with: - # path: dist/*.tar.gz - - # # Upload the wheels and the source distribution - # upload_anaconda: - # name: Upload to Anaconda - # runs-on: ubuntu-latest - # needs: [build_wheels, build_sdist] - # # The artifacts cannot be uploaded on PRs - # if: github.event_name != 'pull_request' - - # steps: - # - name: Checkout scikit-learn - # uses: actions/checkout@v1 - - # - name: Download artifacts - # uses: actions/download-artifact@v2 - # with: - # path: dist - - # - name: Setup Python - # uses: actions/setup-python@v2 - - # - name: Upload artifacts - # env: - # # Secret variables need to be mapped to environment variables explicitly - # SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} - # SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} - # # Force a replacement if the remote file already exists - # run: bash build_tools/github/upload_anaconda.sh + - name: Build and test wheels + env: + CONFTEST_PATH: ${{ github.workspace }}/conftest.py + CONFTEST_NAME: conftest.py + CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 + OPENBLAS_NUM_THREADS=2 + SKLEARN_SKIP_NETWORK_TESTS=1 + SKLEARN_BUILD_PARALLEL=3 + MACOSX_DEPLOYMENT_TARGET=10.13 + CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} + CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} + CIBW_TEST_REQUIRES: pytest pandas threadpoolctl + CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh + CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} + + run: bash build_tools/github/build_wheels.sh + + - name: Store artifacts + uses: actions/upload-artifact@v2 + with: + path: wheelhouse/*.whl + + # Build the source distribution under Linux + build_sdist: + name: Source distribution + runs-on: ubuntu-latest + needs: check_build_trigger + if: needs.check_build_trigger.outputs.build + + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v1 + + - name: Setup Python + uses: actions/setup-python@v2 + + - name: Build source distribution + run: bash build_tools/github/build_source.sh + env: + SKLEARN_BUILD_PARALLEL: 3 + + - name: Test source distribution + run: bash build_tools/github/test_source.sh + env: + OMP_NUM_THREADS: 2 + OPENBLAS_NUM_THREADS: 2 + SKLEARN_SKIP_NETWORK_TESTS: 1 + + - name: Store artifacts + uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz + + # Upload the wheels and the source distribution + upload_anaconda: + name: Upload to Anaconda + runs-on: ubuntu-latest + needs: [build_wheels, build_sdist] + # The artifacts cannot be uploaded on PRs + if: github.event_name != 'pull_request' + + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v1 + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + path: dist + + - name: Setup Python + uses: actions/setup-python@v2 + + - name: Upload artifacts + env: + # Secret variables need to be mapped to environment variables explicitly + SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} + SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} + # Force a replacement if the remote file already exists + run: bash build_tools/github/upload_anaconda.sh diff --git a/build_tools/github/check_wheels.py b/build_tools/github/check_wheels.py index a0668270a43cf..c213991394a6b 100644 --- a/build_tools/github/check_wheels.py +++ b/build_tools/github/check_wheels.py @@ -8,8 +8,7 @@ with gh_wheel_path.open('r') as f: wheel_config = yaml.safe_load(f) -build_matrix = wheel_config['jobs'][ - 'build_wheels_windows_osx']['strategy']['matrix'] +build_matrix = wheel_config['jobs']['build_wheels']['strategy']['matrix'] n_python_versions = len(build_matrix['python']) # For each python version we have: 7 wheels From 414bbcfecc6af9ba917a3c6922e0a09aa9d83571 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Mon, 25 Jan 2021 09:47:27 +0100 Subject: [PATCH 19/19] Trigger [cd build]