From b0aa4e440857ff40ebca22e64fad92d0a22284cd Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Mon, 6 Sep 2021 18:47:47 +0200 Subject: [PATCH 01/16] Move pip install command into test command [cd build] To workaround travis arm64 buffering issues causing the job to be wrongly detected as stalled after 10min by travis. --- .travis.yml | 1 - build_tools/travis/test_wheels.sh | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 328940a165fee..bad821b169235 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,6 @@ env: - SKLEARN_SKIP_NETWORK_TESTS=1 # Custom environment variables for the ARM wheel builder - CIBW_BUILD_VERBOSITY=1 - - CIBW_TEST_REQUIRES="pytest pytest-xdist threadpoolctl" - CIBW_TEST_COMMAND="bash {project}/build_tools/travis/test_wheels.sh" - CIBW_ENVIRONMENT="CPU_COUNT=2 OMP_NUM_THREADS=2 diff --git a/build_tools/travis/test_wheels.sh b/build_tools/travis/test_wheels.sh index be2328e3d44d6..29b625719e9e1 100644 --- a/build_tools/travis/test_wheels.sh +++ b/build_tools/travis/test_wheels.sh @@ -1,9 +1,11 @@ #!/bin/bash set -e +pip install --upgrade pip +pip install pytest pytest-xdist # Faster run of the source code tests -pytest -n $CPU_COUNT --pyargs sklearn +python -m pytest -n $CPU_COUNT --pyargs sklearn # Test that there are no links to system libraries python -m threadpoolctl -i sklearn From d55bbd65a34d78a63c4e8abbbb6a5c110855eb64 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Mon, 6 Sep 2021 20:24:29 +0200 Subject: [PATCH 02/16] Try to parallelism in tests --- build_tools/travis/test_wheels.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build_tools/travis/test_wheels.sh b/build_tools/travis/test_wheels.sh index 29b625719e9e1..e6763c0104839 100644 --- a/build_tools/travis/test_wheels.sh +++ b/build_tools/travis/test_wheels.sh @@ -5,7 +5,9 @@ pip install --upgrade pip pip install pytest pytest-xdist # Faster run of the source code tests -python -m pytest -n $CPU_COUNT --pyargs sklearn +export USABLE_CPU_COUNT=`python -c "import joblib; print(joblib.cpu_count())"` +echo "Usable number of CPUs according to joblib: $USABLE_CPU_COUNT" +python -m pytest -n $USABLE_CPU_COUNT --pyargs sklearn # Test that there are no links to system libraries python -m threadpoolctl -i sklearn From c8b05ebc087c48073493942b970ce6b4a1388951 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Mon, 6 Sep 2021 20:40:29 +0200 Subject: [PATCH 03/16] forgot to [cd build], again... From 5713f9a77eca445861e05a854edf8cd1f1741253 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 09:18:55 +0200 Subject: [PATCH 04/16] Try not to use set -e when building / testing wheels on travis --- build_tools/travis/install.sh | 6 ++---- build_tools/travis/install_wheels.sh | 6 ++---- build_tools/travis/script.sh | 2 -- build_tools/travis/test_wheels.sh | 9 ++++----- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/build_tools/travis/install.sh b/build_tools/travis/install.sh index 1e8e2963711ef..178260c8dabcb 100644 --- a/build_tools/travis/install.sh +++ b/build_tools/travis/install.sh @@ -4,10 +4,8 @@ # defined in the ".travis.yml" file. In particular, it is # important that we call to the right installation script. -set -e - if [[ $BUILD_WHEEL == true ]]; then - source build_tools/travis/install_wheels.sh + source build_tools/travis/install_wheels.sh || travis_terminate 1 else - source build_tools/travis/install_main.sh + source build_tools/travis/install_main.sh || travis_terminate 1 fi diff --git a/build_tools/travis/install_wheels.sh b/build_tools/travis/install_wheels.sh index 4bb52f51f27f7..0f6cdf256e71b 100644 --- a/build_tools/travis/install_wheels.sh +++ b/build_tools/travis/install_wheels.sh @@ -1,6 +1,4 @@ #!/bin/bash -set -e - -python -m pip install cibuildwheel -python -m cibuildwheel --output-dir wheelhouse +python -m pip install cibuildwheel || travis_terminate $? +python -m cibuildwheel --output-dir wheelhouse || travis_terminate $? diff --git a/build_tools/travis/script.sh b/build_tools/travis/script.sh index 2b7aecb295d82..6e8b7e3deaee1 100644 --- a/build_tools/travis/script.sh +++ b/build_tools/travis/script.sh @@ -5,8 +5,6 @@ # continuous deployment jobs, we have to execute the scripts for # testing the continuous integration jobs. -set -e - if [[ $BUILD_WHEEL != true ]]; then # This trick will make Travis terminate the continuation of the pipeline bash build_tools/travis/test_script.sh || travis_terminate 1 diff --git a/build_tools/travis/test_wheels.sh b/build_tools/travis/test_wheels.sh index e6763c0104839..68d6a193642de 100644 --- a/build_tools/travis/test_wheels.sh +++ b/build_tools/travis/test_wheels.sh @@ -1,13 +1,12 @@ #!/bin/bash -set -e -pip install --upgrade pip -pip install pytest pytest-xdist +pip install --upgrade pip || travis_terminate $? +pip install pytest pytest-xdist || travis_terminate $? # Faster run of the source code tests export USABLE_CPU_COUNT=`python -c "import joblib; print(joblib.cpu_count())"` echo "Usable number of CPUs according to joblib: $USABLE_CPU_COUNT" -python -m pytest -n $USABLE_CPU_COUNT --pyargs sklearn +python -m pytest -n $USABLE_CPU_COUNT --pyargs sklearn || travis_terminate $? # Test that there are no links to system libraries -python -m threadpoolctl -i sklearn +python -m threadpoolctl -i sklearn || travis_terminate $? From 42cf411b474f8a318abf45d59e1301e1e2f8f6fa Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 09:19:03 +0200 Subject: [PATCH 05/16] [cd build] From 1067a4d817b8ff0d825aabb1e5f5f8f9b6365813 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 11:03:50 +0200 Subject: [PATCH 06/16] DEBUG: remove other CIs to speed-up debug iterations --- .circleci/config.yml | 210 ----------------------------- .github/workflows/wheels.yml | 171 ------------------------ azure-pipelines.yml | 248 ----------------------------------- 3 files changed, 629 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .github/workflows/wheels.yml delete mode 100644 azure-pipelines.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index b730ae0ff595a..0000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,210 +0,0 @@ -version: 2.1 - -jobs: - doc-min-dependencies: - docker: - - image: circleci/python:3.7.7-buster - environment: - - OMP_NUM_THREADS: 2 - - MKL_NUM_THREADS: 2 - - CONDA_ENV_NAME: testenv - - PYTHON_VERSION: 3.7 - - NUMPY_VERSION: 'min' - - SCIPY_VERSION: 'min' - - MATPLOTLIB_VERSION: 'min' - - CYTHON_VERSION: 'min' - - SCIKIT_IMAGE_VERSION: 'min' - - SPHINX_VERSION: 'min' - - PANDAS_VERSION: 'min' - - SPHINX_GALLERY_VERSION: 'min' - - NUMPYDOC_VERSION: 'min' - - SPHINX_PROMPT_VERSION: 'min' - - SPHINXEXT_OPENGRAPH_VERSION: 'min' - steps: - - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - - restore_cache: - key: v1-datasets-{{ .Branch }} - - restore_cache: - keys: - - doc-min-deps-ccache-{{ .Branch }} - - doc-min-deps-ccache - - run: ./build_tools/circle/build_doc.sh - - save_cache: - key: doc-min-deps-ccache-{{ .Branch }}-{{ .BuildNum }} - paths: - - ~/.ccache - - ~/.cache/pip - - save_cache: - key: v1-datasets-{{ .Branch }} - paths: - - ~/scikit_learn_data - - store_artifacts: - path: doc/_build/html/stable - destination: doc - - store_artifacts: - path: ~/log.txt - destination: log.txt - - doc: - docker: - - image: circleci/python:3.7.7-buster - environment: - - OMP_NUM_THREADS: 2 - - MKL_NUM_THREADS: 2 - - CONDA_ENV_NAME: testenv - - PYTHON_VERSION: 3 - - NUMPY_VERSION: 'latest' - - SCIPY_VERSION: 'latest' - - MATPLOTLIB_VERSION: 'latest' - - CYTHON_VERSION: 'latest' - - SCIKIT_IMAGE_VERSION: 'latest' - - SPHINX_VERSION: 'min' - - PANDAS_VERSION: 'latest' - - SPHINX_GALLERY_VERSION: 'latest' - - NUMPYDOC_VERSION: 'latest' - - SPHINX_PROMPT_VERSION: 'latest' - - SPHINXEXT_OPENGRAPH_VERSION: 'latest' - steps: - - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - - restore_cache: - key: v1-datasets-{{ .Branch }} - - restore_cache: - keys: - - doc-ccache-{{ .Branch }} - - doc-ccache - - run: ./build_tools/circle/build_doc.sh - - save_cache: - key: doc-ccache-{{ .Branch }}-{{ .BuildNum }} - paths: - - ~/.ccache - - ~/.cache/pip - - save_cache: - key: v1-datasets-{{ .Branch }} - paths: - - ~/scikit_learn_data - - store_artifacts: - path: doc/_build/html/stable - destination: doc - - store_artifacts: - path: ~/log.txt - destination: log.txt - # Persists generated documentation so that it can be attached and deployed - # in the 'deploy' step. - - persist_to_workspace: - root: doc/_build/html - paths: . - - lint: - docker: - - image: circleci/python:3.7 - steps: - - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - - run: - name: dependencies - command: sudo pip install flake8 - - run: - name: linting - command: ./build_tools/circle/linting.sh - - pypy3: - docker: - - image: condaforge/miniforge3 - environment: - # Avoid the interactive dialog when installing tzdata - - DEBIAN_FRONTEND: noninteractive - steps: - - restore_cache: - keys: - - pypy3-ccache-{{ .Branch }} - - pypy3-ccache - - run: apt-get -yq update && apt-get -yq install git ssh - - checkout - - run: conda init bash && source ~/.bashrc - - run: ./build_tools/circle/build_test_pypy.sh - - save_cache: - key: pypy3-ccache-{{ .Branch }}-{{ .BuildNum }} - paths: - - ~/.ccache - - ~/.cache/pip - - linux-arm64: - machine: - image: ubuntu-2004:202101-01 - resource_class: arm.medium - environment: - - OMP_NUM_THREADS: 2 - - OPENBLAS_NUM_THREADS: 2 - - CYTHON_VERSION: 'latest' - - JOBLIB_VERSION: 'latest' - - THREADPOOLCTL_VERSION: 'latest' - - PYTEST_VERSION: 'latest' - - PYTEST_XDIST_VERSION: 'latest' - - TEST_DOCSTRINGS: 'true' - steps: - - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - - restore_cache: - key: v1-datasets-{{ .Branch }} - - run: ./build_tools/circle/build_test_arm.sh - - save_cache: - key: doc-ccache-{{ .Branch }}-{{ .BuildNum }} - paths: - - ~/.ccache - - ~/.cache/pip - - save_cache: - key: v1-datasets-{{ .Branch }} - paths: - - ~/scikit_learn_data - - deploy: - docker: - - image: circleci/python:3.7 - steps: - - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - # Attach documentation generated in the 'doc' step so that it can be - # deployed. - - attach_workspace: - at: doc/_build/html - - run: ls -ltrh doc/_build/html/stable - - deploy: - command: | - if [[ "${CIRCLE_BRANCH}" =~ ^main$|^[0-9]+\.[0-9]+\.X$ ]]; then - bash build_tools/circle/push_doc.sh doc/_build/html/stable - fi - -workflows: - version: 2 - build-doc-and-deploy: - jobs: - - lint - - doc: - requires: - - lint - - doc-min-dependencies: - requires: - - lint - - pypy3: - filters: - branches: - only: - - 0.20.X - - deploy: - requires: - - doc - pypy: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - main - jobs: - - pypy3 - linux-arm64: - jobs: - - linux-arm64 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml deleted file mode 100644 index a280c29c31683..0000000000000 --- a/.github/workflows/wheels.yml +++ /dev/null @@ -1,171 +0,0 @@ -# Workflow to build and test wheels -name: Wheel builder - -on: - schedule: - # Nightly build at 3:42 A.M. - - cron: "42 3 */1 * *" - push: - branches: - - main - # Release branches - - "[0-9]+.[0-9]+.X" - pull_request: - branches: - - main - - "[0-9]+.[0-9]+.X" - # Manual run - workflow_dispatch: - -jobs: - # Check whether to build the wheels and the source tarball - check_build_trigger: - name: Check build trigger - runs-on: ubuntu-latest - if: github.repository == 'scikit-learn/scikit-learn' - outputs: - build: ${{ steps.check_build_trigger.outputs.build }} - - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - id: check_build_trigger - name: Check build trigger - run: bash build_tools/github/check_build_trigger.sh - - # Build the wheels for Linux, Windows and macOS for Python 3.7 and newer - build_wheels: - name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} - runs-on: ${{ matrix.os }} - needs: check_build_trigger - if: needs.check_build_trigger.outputs.build - - strategy: - # Ensure that a wheel builder finishes even if another fails - fail-fast: false - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - python: [37, 38, 39] - bitness: [32, 64] - manylinux_image: [manylinux1, manylinux2010] - include: - # Run 32 and 64 bit version in parallel for Linux and Windows - - os: windows-latest - bitness: 64 - platform_id: win_amd64 - - os: windows-latest - bitness: 32 - platform_id: win32 - - os: ubuntu-latest - bitness: 64 - platform_id: manylinux_x86_64 - - os: ubuntu-latest - bitness: 32 - platform_id: manylinux_i686 - - os: macos-latest - bitness: 64 - platform_id: macosx_x86_64 - exclude: - - os: macos-latest - bitness: 32 - # Remove manylinux1 from the windows and osx build matrix since - # manylinux_image is not used for these platforms - - os: windows-latest - manylinux_image: manylinux1 - - os: macos-latest - manylinux_image: manylinux1 - - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v1 - - - name: Setup Python - uses: actions/setup-python@v2 - - - name: Build and test wheels - env: - CONFTEST_PATH: ${{ github.workspace }}/conftest.py - CONFTEST_NAME: conftest.py - CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 - OPENBLAS_NUM_THREADS=2 - SKLEARN_SKIP_NETWORK_TESTS=1 - SKLEARN_BUILD_PARALLEL=3 - MACOSX_DEPLOYMENT_TARGET=10.13 - CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} - CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} - CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} - CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} - CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} - CIBW_TEST_REQUIRES: pytest pandas threadpoolctl - CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh - CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} - CIBW_BUILD_VERBOSITY: 1 - - run: bash build_tools/github/build_wheels.sh - - - name: Store artifacts - uses: actions/upload-artifact@v2 - with: - path: wheelhouse/*.whl - - # Build the source distribution under Linux - build_sdist: - name: Source distribution - runs-on: ubuntu-latest - needs: check_build_trigger - if: needs.check_build_trigger.outputs.build - - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v1 - - - name: Setup Python - uses: actions/setup-python@v2 - - - name: Build source distribution - run: bash build_tools/github/build_source.sh - env: - SKLEARN_BUILD_PARALLEL: 3 - - - name: Test source distribution - run: bash build_tools/github/test_source.sh - env: - OMP_NUM_THREADS: 2 - OPENBLAS_NUM_THREADS: 2 - SKLEARN_SKIP_NETWORK_TESTS: 1 - - - name: Store artifacts - uses: actions/upload-artifact@v2 - with: - path: dist/*.tar.gz - - # Upload the wheels and the source distribution - upload_anaconda: - name: Upload to Anaconda - runs-on: ubuntu-latest - needs: [build_wheels, build_sdist] - # The artifacts cannot be uploaded on PRs - if: github.event_name != 'pull_request' - - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v1 - - - name: Download artifacts - uses: actions/download-artifact@v2 - with: - path: dist - - - name: Setup Python - uses: actions/setup-python@v2 - - - name: Upload artifacts - env: - # Secret variables need to be mapped to environment variables explicitly - SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} - SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} - # Force a replacement if the remote file already exists - run: bash build_tools/github/upload_anaconda.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 2c19db7c9aaae..0000000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,248 +0,0 @@ -# Adapted from https://github.com/pandas-dev/pandas/blob/master/azure-pipelines.yml -schedules: -- cron: "30 2 * * *" - displayName: Run nightly build - branches: - include: - - main - always: true - -jobs: -- job: git_commit - displayName: Get Git Commit - pool: - vmImage: ubuntu-20.04 - steps: - - bash: | - set -ex - if [[ $BUILD_REASON == "PullRequest" ]]; then - # By default pull requests use refs/pull/PULL_ID/merge as the source branch - # which has a "Merge ID into ID" as a commit message. The latest commit - # message is the second to last commit - COMMIT_ID=$(echo $BUILD_SOURCEVERSIONMESSAGE | awk '{print $2}') - message=$(git log $COMMIT_ID -1 --pretty=%B) - else - message=$BUILD_SOURCEVERSIONMESSAGE - fi - echo "##vso[task.setvariable variable=message;isOutput=true]$message" - name: commit - displayName: Get source version message - -- job: linting - dependsOn: [git_commit] - condition: | - and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[lint skip]')), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')) - ) - displayName: Linting - pool: - vmImage: ubuntu-20.04 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.9' - - bash: | - # Include pytest compatibility with mypy - pip install pytest flake8 mypy==0.782 black==21.6b0 - displayName: Install linters - - bash: | - black --check . - displayName: Run black - - bash: | - ./build_tools/circle/linting.sh - displayName: Run linting - - bash: | - mypy sklearn/ - displayName: Run mypy - -- template: build_tools/azure/posix.yml - parameters: - name: Linux_Nightly - vmImage: ubuntu-20.04 - dependsOn: [git_commit, linting] - condition: | - and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), - or(eq(variables['Build.Reason'], 'Schedule'), - contains(dependencies['git_commit']['outputs']['commit.message'], '[scipy-dev]' - ) - ) - ) - matrix: - pylatest_pip_scipy_dev: - DISTRIB: 'conda-pip-scipy-dev' - PYTHON_VERSION: '*' - CHECK_WARNINGS: 'true' - CHECK_PYTEST_SOFT_DEPENDENCY: 'true' - TEST_DOCSTRINGS: 'true' - # Tests that require large downloads over the networks are skipped in CI. - # Here we make sure, that they are still run on a regular basis. - SKLEARN_SKIP_NETWORK_TESTS: '0' - -# Check compilation with intel C++ compiler (ICC) -- template: build_tools/azure/posix.yml - parameters: - name: Linux_Nightly_ICC - vmImage: ubuntu-20.04 - dependsOn: [git_commit, linting] - condition: | - and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), - or(eq(variables['Build.Reason'], 'Schedule'), - contains(dependencies['git_commit']['outputs']['commit.message'], '[icc-build]') - ) - ) - matrix: - pylatest_conda_mkl: - DISTRIB: 'conda' - PYTHON_VERSION: '*' - BLAS: 'mkl' - COVERAGE: 'false' - BUILD_WITH_ICC: 'true' - -# Will run all the time regardless of linting outcome. -- template: build_tools/azure/posix.yml - parameters: - name: Linux_Runs - vmImage: ubuntu-20.04 - dependsOn: [git_commit] - condition: | - and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')) - ) - matrix: - pylatest_conda_mkl: - DISTRIB: 'conda' - PYTHON_VERSION: '*' - BLAS: 'mkl' - COVERAGE: 'true' - -# Check compilation with Ubuntu bionic 18.04 LTS and scipy from conda-forge -- template: build_tools/azure/posix.yml - parameters: - name: Ubuntu_Bionic - vmImage: ubuntu-18.04 - dependsOn: [git_commit, linting] - condition: | - and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), - ne(variables['Build.Reason'], 'Schedule') - ) - matrix: - py37_conda: - DISTRIB: 'conda' - PYTHON_VERSION: '3.7' - BLAS: 'openblas' - COVERAGE: 'false' - BUILD_WITH_ICC: 'false' - -- template: build_tools/azure/posix.yml - parameters: - name: Linux - vmImage: ubuntu-20.04 - dependsOn: [linting, git_commit] - condition: | - and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), - ne(variables['Build.Reason'], 'Schedule') - ) - matrix: - # Linux environment to test that scikit-learn can be built against - # versions of numpy, scipy with ATLAS that comes with Ubuntu Focal 20.04 - # i.e. numpy 1.17.4 and scipy 1.3.3 - ubuntu_atlas: - DISTRIB: 'ubuntu' - JOBLIB_VERSION: 'min' - PANDAS_VERSION: 'none' - THREADPOOLCTL_VERSION: 'min' - COVERAGE: 'false' - # Linux + Python 3.7 build with OpenBLAS and without SITE_JOBLIB - py37_conda_openblas: - DISTRIB: 'conda' - PYTHON_VERSION: '3.7' - BLAS: 'openblas' - NUMPY_VERSION: 'min' - SCIPY_VERSION: 'min' - MATPLOTLIB_VERSION: 'min' - THREADPOOLCTL_VERSION: '2.0.0' - # Linux environment to test the latest available dependencies and MKL. - # It runs tests requiring lightgbm, pandas and PyAMG. - pylatest_pip_openblas_pandas: - DISTRIB: 'conda-pip-latest' - PYTHON_VERSION: '3.9' - PANDAS_VERSION: 'none' - CHECK_PYTEST_SOFT_DEPENDENCY: 'true' - TEST_DOCSTRINGS: 'true' - CHECK_WARNINGS: 'true' - -- template: build_tools/azure/posix-32.yml - parameters: - name: Linux32 - vmImage: ubuntu-20.04 - dependsOn: [linting, git_commit] - condition: | - and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), - ne(variables['Build.Reason'], 'Schedule') - ) - matrix: - debian_atlas_32bit: - DISTRIB: 'debian-32' - JOBLIB_VERSION: 'min' - # disable pytest xdist due to unknown bug with 32-bit container - PYTEST_XDIST_VERSION: 'none' - PYTEST_VERSION: 'min' - THREADPOOLCTL_VERSION: 'min' - -- template: build_tools/azure/posix.yml - parameters: - name: macOS - vmImage: macOS-10.14 - dependsOn: [linting, git_commit] - condition: | - and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), - ne(variables['Build.Reason'], 'Schedule') - ) - matrix: - pylatest_conda_forge_mkl: - DISTRIB: 'conda' - BLAS: 'mkl' - CONDA_CHANNEL: 'conda-forge' - pylatest_conda_mkl_no_openmp: - DISTRIB: 'conda' - BLAS: 'mkl' - SKLEARN_TEST_NO_OPENMP: 'true' - SKLEARN_SKIP_OPENMP_TEST: 'true' - -- template: build_tools/azure/windows.yml - parameters: - name: Windows - vmImage: vs2017-win2016 - dependsOn: [linting, git_commit] - condition: | - and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), - ne(variables['Build.Reason'], 'Schedule') - ) - matrix: - py37_conda_mkl: - PYTHON_VERSION: '3.7' - CHECK_WARNINGS: 'true' - PYTHON_ARCH: '64' - PYTEST_VERSION: '*' - COVERAGE: 'true' - DISTRIB: 'conda' - py37_pip_openblas_32bit: - PYTHON_VERSION: '3.7' - PYTHON_ARCH: '32' From 248f65f69fe32512d25a31d27c24cc26a73e61b5 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 11:05:09 +0200 Subject: [PATCH 07/16] restrict tests to sklearn.cluster [cd build] --- build_tools/travis/test_wheels.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build_tools/travis/test_wheels.sh b/build_tools/travis/test_wheels.sh index 68d6a193642de..e6e9abe42c8c6 100644 --- a/build_tools/travis/test_wheels.sh +++ b/build_tools/travis/test_wheels.sh @@ -6,7 +6,10 @@ pip install pytest pytest-xdist || travis_terminate $? # Faster run of the source code tests export USABLE_CPU_COUNT=`python -c "import joblib; print(joblib.cpu_count())"` echo "Usable number of CPUs according to joblib: $USABLE_CPU_COUNT" -python -m pytest -n $USABLE_CPU_COUNT --pyargs sklearn || travis_terminate $? + +# XXX: limit to sklearn.cluster instead of full sklearn to see if overall +# test duration is causing the problem. +python -m pytest -n $USABLE_CPU_COUNT --pyargs sklearn.cluster || travis_terminate $? # Test that there are no links to system libraries python -m threadpoolctl -i sklearn || travis_terminate $? From c80a7a14200cdc51d995e52dd852ba12f237f524 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 11:19:34 +0200 Subject: [PATCH 08/16] Try to disable python output buffering --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bad821b169235..dfb01be08ace7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ env: - OPENBLAS_NUM_THREADS=2 - SKLEARN_BUILD_PARALLEL=3 - SKLEARN_SKIP_NETWORK_TESTS=1 + - PYTHONUNBUFFERED=1 # Custom environment variables for the ARM wheel builder - CIBW_BUILD_VERBOSITY=1 - CIBW_TEST_COMMAND="bash {project}/build_tools/travis/test_wheels.sh" @@ -25,7 +26,8 @@ env: OMP_NUM_THREADS=2 OPENBLAS_NUM_THREADS=2 SKLEARN_BUILD_PARALLEL=3 - SKLEARN_SKIP_NETWORK_TESTS=1" + SKLEARN_SKIP_NETWORK_TESTS=1 + PYTHONUNBUFFERED=1" jobs: include: From 5882bda378adb842972f48ae111a48fe70245a98 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 11:20:57 +0200 Subject: [PATCH 09/16] Only use 2 cores in inner pytest command (instead o 80) --- build_tools/travis/test_wheels.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/build_tools/travis/test_wheels.sh b/build_tools/travis/test_wheels.sh index e6e9abe42c8c6..6933072d5fcce 100644 --- a/build_tools/travis/test_wheels.sh +++ b/build_tools/travis/test_wheels.sh @@ -3,13 +3,9 @@ pip install --upgrade pip || travis_terminate $? pip install pytest pytest-xdist || travis_terminate $? -# Faster run of the source code tests -export USABLE_CPU_COUNT=`python -c "import joblib; print(joblib.cpu_count())"` -echo "Usable number of CPUs according to joblib: $USABLE_CPU_COUNT" - # XXX: limit to sklearn.cluster instead of full sklearn to see if overall # test duration is causing the problem. -python -m pytest -n $USABLE_CPU_COUNT --pyargs sklearn.cluster || travis_terminate $? +python -m pytest -n $CPU_COUNT --pyargs sklearn.cluster || travis_terminate $? # Test that there are no links to system libraries python -m threadpoolctl -i sklearn || travis_terminate $? From 12eb65d24296fd4e89c4a4103066aead6a226717 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 11:21:12 +0200 Subject: [PATCH 10/16] [cd build] From 872ba66e3de486bf1c7bca3fa3040a3eee12dc3b Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 11:30:24 +0200 Subject: [PATCH 11/16] Try to slowly increase build and test parallelism in CIBW env [cd build] --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dfb01be08ace7..38e84ca27f0f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,10 +22,10 @@ env: # Custom environment variables for the ARM wheel builder - CIBW_BUILD_VERBOSITY=1 - CIBW_TEST_COMMAND="bash {project}/build_tools/travis/test_wheels.sh" - - CIBW_ENVIRONMENT="CPU_COUNT=2 + - CIBW_ENVIRONMENT="CPU_COUNT=6 OMP_NUM_THREADS=2 OPENBLAS_NUM_THREADS=2 - SKLEARN_BUILD_PARALLEL=3 + SKLEARN_BUILD_PARALLEL=6 SKLEARN_SKIP_NETWORK_TESTS=1 PYTHONUNBUFFERED=1" From a9d40e9446d70ecb5650f02a262abdb30040b64c Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 11:36:33 +0200 Subject: [PATCH 12/16] Run full sklearn test suite [cd build] --- build_tools/travis/test_wheels.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build_tools/travis/test_wheels.sh b/build_tools/travis/test_wheels.sh index 6933072d5fcce..aa3d0d8c0ef1b 100644 --- a/build_tools/travis/test_wheels.sh +++ b/build_tools/travis/test_wheels.sh @@ -3,9 +3,7 @@ pip install --upgrade pip || travis_terminate $? pip install pytest pytest-xdist || travis_terminate $? -# XXX: limit to sklearn.cluster instead of full sklearn to see if overall -# test duration is causing the problem. -python -m pytest -n $CPU_COUNT --pyargs sklearn.cluster || travis_terminate $? +python -m pytest -n $CPU_COUNT --pyargs sklearn || travis_terminate $? # Test that there are no links to system libraries python -m threadpoolctl -i sklearn || travis_terminate $? From 1743193ec7a84d23c9f087c8cb6e18fbf20a27f9 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 11:45:05 +0200 Subject: [PATCH 13/16] Increase parallelism to 10 cores [cd build] --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38e84ca27f0f6..2ebf119ba229b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,10 +22,10 @@ env: # Custom environment variables for the ARM wheel builder - CIBW_BUILD_VERBOSITY=1 - CIBW_TEST_COMMAND="bash {project}/build_tools/travis/test_wheels.sh" - - CIBW_ENVIRONMENT="CPU_COUNT=6 + - CIBW_ENVIRONMENT="CPU_COUNT=10 OMP_NUM_THREADS=2 OPENBLAS_NUM_THREADS=2 - SKLEARN_BUILD_PARALLEL=6 + SKLEARN_BUILD_PARALLEL=10 SKLEARN_SKIP_NETWORK_TESTS=1 PYTHONUNBUFFERED=1" From 6576a75a74fc6182d00c8d09dce0bb6ebb4fd665 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 11:53:36 +0200 Subject: [PATCH 14/16] Reduce pytest parallelism down to 6 to reduce likelihood of worker crash [cd build] --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2ebf119ba229b..456d94301d4c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ env: # Custom environment variables for the ARM wheel builder - CIBW_BUILD_VERBOSITY=1 - CIBW_TEST_COMMAND="bash {project}/build_tools/travis/test_wheels.sh" - - CIBW_ENVIRONMENT="CPU_COUNT=10 + - CIBW_ENVIRONMENT="CPU_COUNT=6 OMP_NUM_THREADS=2 OPENBLAS_NUM_THREADS=2 SKLEARN_BUILD_PARALLEL=10 From 15c7ce6fcc30a0cc68888e2a055f4e6c6a606c0d Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 12:01:41 +0200 Subject: [PATCH 15/16] Revert "DEBUG: remove other CIs to speed-up debug iterations" This reverts commit 1067a4d817b8ff0d825aabb1e5f5f8f9b6365813. --- .circleci/config.yml | 210 +++++++++++++++++++++++++++++ .github/workflows/wheels.yml | 171 ++++++++++++++++++++++++ azure-pipelines.yml | 248 +++++++++++++++++++++++++++++++++++ 3 files changed, 629 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 .github/workflows/wheels.yml create mode 100644 azure-pipelines.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000000..b730ae0ff595a --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,210 @@ +version: 2.1 + +jobs: + doc-min-dependencies: + docker: + - image: circleci/python:3.7.7-buster + environment: + - OMP_NUM_THREADS: 2 + - MKL_NUM_THREADS: 2 + - CONDA_ENV_NAME: testenv + - PYTHON_VERSION: 3.7 + - NUMPY_VERSION: 'min' + - SCIPY_VERSION: 'min' + - MATPLOTLIB_VERSION: 'min' + - CYTHON_VERSION: 'min' + - SCIKIT_IMAGE_VERSION: 'min' + - SPHINX_VERSION: 'min' + - PANDAS_VERSION: 'min' + - SPHINX_GALLERY_VERSION: 'min' + - NUMPYDOC_VERSION: 'min' + - SPHINX_PROMPT_VERSION: 'min' + - SPHINXEXT_OPENGRAPH_VERSION: 'min' + steps: + - checkout + - run: ./build_tools/circle/checkout_merge_commit.sh + - restore_cache: + key: v1-datasets-{{ .Branch }} + - restore_cache: + keys: + - doc-min-deps-ccache-{{ .Branch }} + - doc-min-deps-ccache + - run: ./build_tools/circle/build_doc.sh + - save_cache: + key: doc-min-deps-ccache-{{ .Branch }}-{{ .BuildNum }} + paths: + - ~/.ccache + - ~/.cache/pip + - save_cache: + key: v1-datasets-{{ .Branch }} + paths: + - ~/scikit_learn_data + - store_artifacts: + path: doc/_build/html/stable + destination: doc + - store_artifacts: + path: ~/log.txt + destination: log.txt + + doc: + docker: + - image: circleci/python:3.7.7-buster + environment: + - OMP_NUM_THREADS: 2 + - MKL_NUM_THREADS: 2 + - CONDA_ENV_NAME: testenv + - PYTHON_VERSION: 3 + - NUMPY_VERSION: 'latest' + - SCIPY_VERSION: 'latest' + - MATPLOTLIB_VERSION: 'latest' + - CYTHON_VERSION: 'latest' + - SCIKIT_IMAGE_VERSION: 'latest' + - SPHINX_VERSION: 'min' + - PANDAS_VERSION: 'latest' + - SPHINX_GALLERY_VERSION: 'latest' + - NUMPYDOC_VERSION: 'latest' + - SPHINX_PROMPT_VERSION: 'latest' + - SPHINXEXT_OPENGRAPH_VERSION: 'latest' + steps: + - checkout + - run: ./build_tools/circle/checkout_merge_commit.sh + - restore_cache: + key: v1-datasets-{{ .Branch }} + - restore_cache: + keys: + - doc-ccache-{{ .Branch }} + - doc-ccache + - run: ./build_tools/circle/build_doc.sh + - save_cache: + key: doc-ccache-{{ .Branch }}-{{ .BuildNum }} + paths: + - ~/.ccache + - ~/.cache/pip + - save_cache: + key: v1-datasets-{{ .Branch }} + paths: + - ~/scikit_learn_data + - store_artifacts: + path: doc/_build/html/stable + destination: doc + - store_artifacts: + path: ~/log.txt + destination: log.txt + # Persists generated documentation so that it can be attached and deployed + # in the 'deploy' step. + - persist_to_workspace: + root: doc/_build/html + paths: . + + lint: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - run: ./build_tools/circle/checkout_merge_commit.sh + - run: + name: dependencies + command: sudo pip install flake8 + - run: + name: linting + command: ./build_tools/circle/linting.sh + + pypy3: + docker: + - image: condaforge/miniforge3 + environment: + # Avoid the interactive dialog when installing tzdata + - DEBIAN_FRONTEND: noninteractive + steps: + - restore_cache: + keys: + - pypy3-ccache-{{ .Branch }} + - pypy3-ccache + - run: apt-get -yq update && apt-get -yq install git ssh + - checkout + - run: conda init bash && source ~/.bashrc + - run: ./build_tools/circle/build_test_pypy.sh + - save_cache: + key: pypy3-ccache-{{ .Branch }}-{{ .BuildNum }} + paths: + - ~/.ccache + - ~/.cache/pip + + linux-arm64: + machine: + image: ubuntu-2004:202101-01 + resource_class: arm.medium + environment: + - OMP_NUM_THREADS: 2 + - OPENBLAS_NUM_THREADS: 2 + - CYTHON_VERSION: 'latest' + - JOBLIB_VERSION: 'latest' + - THREADPOOLCTL_VERSION: 'latest' + - PYTEST_VERSION: 'latest' + - PYTEST_XDIST_VERSION: 'latest' + - TEST_DOCSTRINGS: 'true' + steps: + - checkout + - run: ./build_tools/circle/checkout_merge_commit.sh + - restore_cache: + key: v1-datasets-{{ .Branch }} + - run: ./build_tools/circle/build_test_arm.sh + - save_cache: + key: doc-ccache-{{ .Branch }}-{{ .BuildNum }} + paths: + - ~/.ccache + - ~/.cache/pip + - save_cache: + key: v1-datasets-{{ .Branch }} + paths: + - ~/scikit_learn_data + + deploy: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - run: ./build_tools/circle/checkout_merge_commit.sh + # Attach documentation generated in the 'doc' step so that it can be + # deployed. + - attach_workspace: + at: doc/_build/html + - run: ls -ltrh doc/_build/html/stable + - deploy: + command: | + if [[ "${CIRCLE_BRANCH}" =~ ^main$|^[0-9]+\.[0-9]+\.X$ ]]; then + bash build_tools/circle/push_doc.sh doc/_build/html/stable + fi + +workflows: + version: 2 + build-doc-and-deploy: + jobs: + - lint + - doc: + requires: + - lint + - doc-min-dependencies: + requires: + - lint + - pypy3: + filters: + branches: + only: + - 0.20.X + - deploy: + requires: + - doc + pypy: + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - main + jobs: + - pypy3 + linux-arm64: + jobs: + - linux-arm64 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000000000..a280c29c31683 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,171 @@ +# Workflow to build and test wheels +name: Wheel builder + +on: + schedule: + # Nightly build at 3:42 A.M. + - cron: "42 3 */1 * *" + push: + branches: + - main + # Release branches + - "[0-9]+.[0-9]+.X" + pull_request: + branches: + - main + - "[0-9]+.[0-9]+.X" + # Manual run + workflow_dispatch: + +jobs: + # Check whether to build the wheels and the source tarball + check_build_trigger: + name: Check build trigger + runs-on: ubuntu-latest + if: github.repository == 'scikit-learn/scikit-learn' + outputs: + build: ${{ steps.check_build_trigger.outputs.build }} + + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - id: check_build_trigger + name: Check build trigger + run: bash build_tools/github/check_build_trigger.sh + + # Build the wheels for Linux, Windows and macOS for Python 3.7 and newer + build_wheels: + name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} + runs-on: ${{ matrix.os }} + needs: check_build_trigger + if: needs.check_build_trigger.outputs.build + + strategy: + # Ensure that a wheel builder finishes even if another fails + fail-fast: false + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + python: [37, 38, 39] + bitness: [32, 64] + manylinux_image: [manylinux1, manylinux2010] + include: + # Run 32 and 64 bit version in parallel for Linux and Windows + - os: windows-latest + bitness: 64 + platform_id: win_amd64 + - os: windows-latest + bitness: 32 + platform_id: win32 + - os: ubuntu-latest + bitness: 64 + platform_id: manylinux_x86_64 + - os: ubuntu-latest + bitness: 32 + platform_id: manylinux_i686 + - os: macos-latest + bitness: 64 + platform_id: macosx_x86_64 + exclude: + - os: macos-latest + bitness: 32 + # Remove manylinux1 from the windows and osx build matrix since + # manylinux_image is not used for these platforms + - os: windows-latest + manylinux_image: manylinux1 + - os: macos-latest + manylinux_image: manylinux1 + + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v1 + + - name: Setup Python + uses: actions/setup-python@v2 + + - name: Build and test wheels + env: + CONFTEST_PATH: ${{ github.workspace }}/conftest.py + CONFTEST_NAME: conftest.py + CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 + OPENBLAS_NUM_THREADS=2 + SKLEARN_SKIP_NETWORK_TESTS=1 + SKLEARN_BUILD_PARALLEL=3 + MACOSX_DEPLOYMENT_TARGET=10.13 + CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} + CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} + CIBW_TEST_REQUIRES: pytest pandas threadpoolctl + CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh + CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} + CIBW_BUILD_VERBOSITY: 1 + + run: bash build_tools/github/build_wheels.sh + + - name: Store artifacts + uses: actions/upload-artifact@v2 + with: + path: wheelhouse/*.whl + + # Build the source distribution under Linux + build_sdist: + name: Source distribution + runs-on: ubuntu-latest + needs: check_build_trigger + if: needs.check_build_trigger.outputs.build + + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v1 + + - name: Setup Python + uses: actions/setup-python@v2 + + - name: Build source distribution + run: bash build_tools/github/build_source.sh + env: + SKLEARN_BUILD_PARALLEL: 3 + + - name: Test source distribution + run: bash build_tools/github/test_source.sh + env: + OMP_NUM_THREADS: 2 + OPENBLAS_NUM_THREADS: 2 + SKLEARN_SKIP_NETWORK_TESTS: 1 + + - name: Store artifacts + uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz + + # Upload the wheels and the source distribution + upload_anaconda: + name: Upload to Anaconda + runs-on: ubuntu-latest + needs: [build_wheels, build_sdist] + # The artifacts cannot be uploaded on PRs + if: github.event_name != 'pull_request' + + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v1 + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + path: dist + + - name: Setup Python + uses: actions/setup-python@v2 + + - name: Upload artifacts + env: + # Secret variables need to be mapped to environment variables explicitly + SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} + SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} + # Force a replacement if the remote file already exists + run: bash build_tools/github/upload_anaconda.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000000000..2c19db7c9aaae --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,248 @@ +# Adapted from https://github.com/pandas-dev/pandas/blob/master/azure-pipelines.yml +schedules: +- cron: "30 2 * * *" + displayName: Run nightly build + branches: + include: + - main + always: true + +jobs: +- job: git_commit + displayName: Get Git Commit + pool: + vmImage: ubuntu-20.04 + steps: + - bash: | + set -ex + if [[ $BUILD_REASON == "PullRequest" ]]; then + # By default pull requests use refs/pull/PULL_ID/merge as the source branch + # which has a "Merge ID into ID" as a commit message. The latest commit + # message is the second to last commit + COMMIT_ID=$(echo $BUILD_SOURCEVERSIONMESSAGE | awk '{print $2}') + message=$(git log $COMMIT_ID -1 --pretty=%B) + else + message=$BUILD_SOURCEVERSIONMESSAGE + fi + echo "##vso[task.setvariable variable=message;isOutput=true]$message" + name: commit + displayName: Get source version message + +- job: linting + dependsOn: [git_commit] + condition: | + and( + succeeded(), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[lint skip]')), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')) + ) + displayName: Linting + pool: + vmImage: ubuntu-20.04 + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9' + - bash: | + # Include pytest compatibility with mypy + pip install pytest flake8 mypy==0.782 black==21.6b0 + displayName: Install linters + - bash: | + black --check . + displayName: Run black + - bash: | + ./build_tools/circle/linting.sh + displayName: Run linting + - bash: | + mypy sklearn/ + displayName: Run mypy + +- template: build_tools/azure/posix.yml + parameters: + name: Linux_Nightly + vmImage: ubuntu-20.04 + dependsOn: [git_commit, linting] + condition: | + and( + succeeded(), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), + or(eq(variables['Build.Reason'], 'Schedule'), + contains(dependencies['git_commit']['outputs']['commit.message'], '[scipy-dev]' + ) + ) + ) + matrix: + pylatest_pip_scipy_dev: + DISTRIB: 'conda-pip-scipy-dev' + PYTHON_VERSION: '*' + CHECK_WARNINGS: 'true' + CHECK_PYTEST_SOFT_DEPENDENCY: 'true' + TEST_DOCSTRINGS: 'true' + # Tests that require large downloads over the networks are skipped in CI. + # Here we make sure, that they are still run on a regular basis. + SKLEARN_SKIP_NETWORK_TESTS: '0' + +# Check compilation with intel C++ compiler (ICC) +- template: build_tools/azure/posix.yml + parameters: + name: Linux_Nightly_ICC + vmImage: ubuntu-20.04 + dependsOn: [git_commit, linting] + condition: | + and( + succeeded(), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), + or(eq(variables['Build.Reason'], 'Schedule'), + contains(dependencies['git_commit']['outputs']['commit.message'], '[icc-build]') + ) + ) + matrix: + pylatest_conda_mkl: + DISTRIB: 'conda' + PYTHON_VERSION: '*' + BLAS: 'mkl' + COVERAGE: 'false' + BUILD_WITH_ICC: 'true' + +# Will run all the time regardless of linting outcome. +- template: build_tools/azure/posix.yml + parameters: + name: Linux_Runs + vmImage: ubuntu-20.04 + dependsOn: [git_commit] + condition: | + and( + succeeded(), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')) + ) + matrix: + pylatest_conda_mkl: + DISTRIB: 'conda' + PYTHON_VERSION: '*' + BLAS: 'mkl' + COVERAGE: 'true' + +# Check compilation with Ubuntu bionic 18.04 LTS and scipy from conda-forge +- template: build_tools/azure/posix.yml + parameters: + name: Ubuntu_Bionic + vmImage: ubuntu-18.04 + dependsOn: [git_commit, linting] + condition: | + and( + succeeded(), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), + ne(variables['Build.Reason'], 'Schedule') + ) + matrix: + py37_conda: + DISTRIB: 'conda' + PYTHON_VERSION: '3.7' + BLAS: 'openblas' + COVERAGE: 'false' + BUILD_WITH_ICC: 'false' + +- template: build_tools/azure/posix.yml + parameters: + name: Linux + vmImage: ubuntu-20.04 + dependsOn: [linting, git_commit] + condition: | + and( + succeeded(), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), + ne(variables['Build.Reason'], 'Schedule') + ) + matrix: + # Linux environment to test that scikit-learn can be built against + # versions of numpy, scipy with ATLAS that comes with Ubuntu Focal 20.04 + # i.e. numpy 1.17.4 and scipy 1.3.3 + ubuntu_atlas: + DISTRIB: 'ubuntu' + JOBLIB_VERSION: 'min' + PANDAS_VERSION: 'none' + THREADPOOLCTL_VERSION: 'min' + COVERAGE: 'false' + # Linux + Python 3.7 build with OpenBLAS and without SITE_JOBLIB + py37_conda_openblas: + DISTRIB: 'conda' + PYTHON_VERSION: '3.7' + BLAS: 'openblas' + NUMPY_VERSION: 'min' + SCIPY_VERSION: 'min' + MATPLOTLIB_VERSION: 'min' + THREADPOOLCTL_VERSION: '2.0.0' + # Linux environment to test the latest available dependencies and MKL. + # It runs tests requiring lightgbm, pandas and PyAMG. + pylatest_pip_openblas_pandas: + DISTRIB: 'conda-pip-latest' + PYTHON_VERSION: '3.9' + PANDAS_VERSION: 'none' + CHECK_PYTEST_SOFT_DEPENDENCY: 'true' + TEST_DOCSTRINGS: 'true' + CHECK_WARNINGS: 'true' + +- template: build_tools/azure/posix-32.yml + parameters: + name: Linux32 + vmImage: ubuntu-20.04 + dependsOn: [linting, git_commit] + condition: | + and( + succeeded(), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), + ne(variables['Build.Reason'], 'Schedule') + ) + matrix: + debian_atlas_32bit: + DISTRIB: 'debian-32' + JOBLIB_VERSION: 'min' + # disable pytest xdist due to unknown bug with 32-bit container + PYTEST_XDIST_VERSION: 'none' + PYTEST_VERSION: 'min' + THREADPOOLCTL_VERSION: 'min' + +- template: build_tools/azure/posix.yml + parameters: + name: macOS + vmImage: macOS-10.14 + dependsOn: [linting, git_commit] + condition: | + and( + succeeded(), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), + ne(variables['Build.Reason'], 'Schedule') + ) + matrix: + pylatest_conda_forge_mkl: + DISTRIB: 'conda' + BLAS: 'mkl' + CONDA_CHANNEL: 'conda-forge' + pylatest_conda_mkl_no_openmp: + DISTRIB: 'conda' + BLAS: 'mkl' + SKLEARN_TEST_NO_OPENMP: 'true' + SKLEARN_SKIP_OPENMP_TEST: 'true' + +- template: build_tools/azure/windows.yml + parameters: + name: Windows + vmImage: vs2017-win2016 + dependsOn: [linting, git_commit] + condition: | + and( + succeeded(), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), + ne(variables['Build.Reason'], 'Schedule') + ) + matrix: + py37_conda_mkl: + PYTHON_VERSION: '3.7' + CHECK_WARNINGS: 'true' + PYTHON_ARCH: '64' + PYTEST_VERSION: '*' + COVERAGE: 'true' + DISTRIB: 'conda' + py37_pip_openblas_32bit: + PYTHON_VERSION: '3.7' + PYTHON_ARCH: '32' From e2492da4d1839d1000a312c4f4a92616a89d9c7c Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 7 Sep 2021 15:02:13 +0200 Subject: [PATCH 16/16] [cd build] new run to assess stability