CI Fix pre-build checks to handle compilers from build_ext options (#16193)

jeremiedbb · web-flow · commit cac167244d9e · 2020-07-10T16:10:10.000+02:00
diff --git a/.travis.yml b/.travis.yml
@@ -22,9 +22,16 @@ matrix:
     # Linux environment to test scikit-learn against numpy and scipy master
     # installed from their CI wheels in a virtualenv with the Python
     # interpreter provided by travis.
-    -  python: 3.7
-       env: CHECK_WARNINGS="true"
-       if: type = cron OR commit_message =~ /\[scipy-dev\]/
+    - python: 3.7
+      env: CHECK_WARNINGS="true"
+      if: type = cron OR commit_message =~ /\[scipy-dev\]/
+    
+    # As above but build scikit-learn with Intel C compiler (ICC).
+    - python: 3.7
+      env:
+        - CHECK_WARNING="true"
+        - BUILD_WITH_ICC="true"
+      if: type = cron OR commit_message =~ /\[icc-build\]/
 
 install: source build_tools/travis/install.sh
 script:
diff --git a/build_tools/travis/install.sh b/build_tools/travis/install.sh
@@ -66,6 +66,25 @@ python --version
 python -c "import numpy; print('numpy %s' % numpy.__version__)"
 python -c "import scipy; print('scipy %s' % scipy.__version__)"
 
+if [[ "$BUILD_WITH_ICC" == "true" ]]; then
+    wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+    sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+    rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+    sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
+    sudo apt-get update
+    sudo apt-get install intel-oneapi-icc
+    source /opt/intel/inteloneapi/setvars.sh
+
+    # The build_clib command is implicitly used to build libsvm-skl. To compile
+    # with a different compiler we also need to specify the compiler for this
+    # command.
+    python setup.py build_ext --compiler=intelem -i -j 3 build_clib --compiler=intelem
+else
+    # Use setup.py instead of `pip install -e .` to be able to pass the -j flag
+    # to speed-up the building multicore CI machines.
+    python setup.py build_ext --inplace -j 3
+fi
+
 python setup.py develop
 
 ccache --show-stats
diff --git a/build_tools/travis/test_docs.sh b/build_tools/travis/test_docs.sh
@@ -3,4 +3,10 @@
 set -e
 set -x
 
+if [[ "$BUILD_WITH_ICC" == "true" ]]; then
+    # the tools in the oneAPI toolkits are configured via environment variables
+    # which are also required at runtime.
+    source /opt/intel/inteloneapi/setvars.sh
+fi
+
 make test-doc
diff --git a/build_tools/travis/test_script.sh b/build_tools/travis/test_script.sh
@@ -20,6 +20,12 @@ except ImportError:
 "
 python -c "import multiprocessing as mp; print('%d CPUs' % mp.cpu_count())"
 
+if [[ "$BUILD_WITH_ICC" == "true" ]]; then
+    # the tools in the oneAPI toolkits are configured via environment variables
+    # which are also required at runtime.
+    source /opt/intel/inteloneapi/setvars.sh
+fi
+
 run_tests() {
     TEST_CMD="pytest --showlocals --durations=20 --pyargs"
 
diff --git a/doc/developers/advanced_installation.rst b/doc/developers/advanced_installation.rst
@@ -388,3 +388,53 @@ the base system and these steps will not be necessary.
 .. _Homebrew: https://brew.sh
 .. _virtualenv: https://docs.python.org/3/tutorial/venv.html
 .. _conda environment: https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html
+
+Alternative compilers
+=====================
+
+The command::
+
+    pip install --verbose --editable .
+
+will build scikit-learn using your default C/C++ compiler. If you want to build
+scikit-learn with another compiler handled by ``distutils`` or by
+``numpy.distutils``, use the following command::
+
+    python setup.py build_ext --compiler=<compiler> -i build_clib --compiler=<compiler>
+
+To see the list of available compilers run::
+
+    python setup.py build_ext --help-compiler
+
+If your compiler is not listed here, you can specify it via the ``CC`` and
+``LDSHARED`` environment variables (does not work on windows)::
+
+    CC=<compiler> LDSHARED="<compiler> -shared" python setup.py build_ext -i
+
+Building with Intel C Compiler (ICC) using oneAPI on Linux
+----------------------------------------------------------
+
+Intel provides access to all of its oneAPI toolkits and packages through a
+public APT repository. First you need to get and install the public key of this
+repository::
+
+    wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+    sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+    rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+
+Then, add the oneAPI repository to your APT repositories::
+
+    sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
+    sudo apt-get update
+
+Install ICC, packaged under the name ``intel-oneapi-icc``::
+
+    sudo apt-get install intel-oneapi-icc
+
+Before using ICC, you need to set up environment variables::
+
+    source /opt/intel/inteloneapi/setvars.sh
+
+Finally, you can build scikit-learn. For example on Linux x86_64::
+
+    python setup.py build_ext --compiler=intelem -i build_clib --compiler=intelem
diff --git a/sklearn/_build_utils/openmp_helpers.py b/sklearn/_build_utils/openmp_helpers.py
@@ -1,7 +1,7 @@
 """Helpers for OpenMP support during the build."""
 
 # This code is adapted for a large part from the astropy openmp helpers, which
-# can be found at: https://github.com/astropy/astropy-helpers/blob/master/astropy_helpers/openmp_helpers.py  # noqa
+# can be found at: https://github.com/astropy/extension-helpers/blob/master/extension_helpers/_openmp_helpers.py  # noqa
 
 
 import os
@@ -25,8 +25,8 @@ def get_openmp_flag(compiler):
         return ['/Qopenmp']
     elif sys.platform == "win32":
         return ['/openmp']
-    elif sys.platform == "darwin" and ('icc' in compiler or 'icl' in compiler):
-        return ['-openmp']
+    elif sys.platform in ("darwin", "linux") and "icc" in compiler:
+        return ['-qopenmp']
     elif sys.platform == "darwin" and 'openmp' in os.getenv('CPPFLAGS', ''):
         # -fopenmp can't be passed as compile flag when using Apple-clang.
         # OpenMP support has to be enabled during preprocessing.
diff --git a/sklearn/_build_utils/pre_build_helpers.py b/sklearn/_build_utils/pre_build_helpers.py
@@ -7,14 +7,40 @@
 import textwrap
 import subprocess
 
+from distutils.dist import Distribution
 from distutils.sysconfig import customize_compiler
 from numpy.distutils.ccompiler import new_compiler
+from numpy.distutils.command.config_compiler import config_cc
+
+
+def _get_compiler():
+    """Get a compiler equivalent to the one that will be used to build sklearn
+
+    Handles compiler specified as follows:
+        - python setup.py build_ext --compiler=<compiler>
+        - CC=<compiler> python setup.py build_ext
+    """
+    dist = Distribution({'script_name': os.path.basename(sys.argv[0]),
+                         'script_args': sys.argv[1:],
+                         'cmdclass': {'config_cc': config_cc}})
+    dist.parse_config_files()
+    dist.parse_command_line()
+
+    cmd_opts = dist.command_options.get('build_ext')
+    if cmd_opts is not None and 'compiler' in cmd_opts:
+        compiler = cmd_opts['compiler'][1]
+    else:
+        compiler = None
+
+    ccompiler = new_compiler(compiler=compiler)
+    customize_compiler(ccompiler)
+
+    return ccompiler
 
 
 def compile_test_program(code, extra_preargs=[], extra_postargs=[]):
     """Check that some C code can be compiled and run"""
-    ccompiler = new_compiler()
-    customize_compiler(ccompiler)
+    ccompiler = _get_compiler()
 
     # extra_(pre/post)args can be a callable to make it possible to get its
     # value from the compiler
