diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2b9a8a5d05212..9fdead91dd309 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -48,7 +48,6 @@ jobs: DISTRIB: 'ubuntu' PYTHON_VERSION: '3.5' JOBLIB_VERSION: '0.11' - SKLEARN_NO_OPENMP: 'True' # Linux + Python 3.5 build with OpenBLAS and without SITE_JOBLIB py35_conda_openblas: DISTRIB: 'conda' @@ -86,7 +85,6 @@ jobs: DISTRIB: 'ubuntu-32' PYTHON_VERSION: '3.5' JOBLIB_VERSION: '0.11' - SKLEARN_NO_OPENMP: 'True' - template: build_tools/azure/posix.yml parameters: @@ -105,6 +103,19 @@ jobs: PYTEST_VERSION: '*' JOBLIB_VERSION: '*' COVERAGE: 'true' + pylatest_conda_mkl_no_openmp: + DISTRIB: 'conda' + PYTHON_VERSION: '*' + INSTALL_MKL: 'true' + NUMPY_VERSION: '*' + SCIPY_VERSION: '*' + CYTHON_VERSION: '*' + PILLOW_VERSION: '*' + PYTEST_VERSION: '*' + JOBLIB_VERSION: '*' + COVERAGE: 'true' + SKLEARN_TEST_NO_OPENMP: 'true' + SKLEARN_SKIP_OPENMP_TEST: 'true' - template: build_tools/azure/windows.yml parameters: diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index 1b4955acedbc8..61ee6bac7116f 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -47,8 +47,11 @@ if [[ "$DISTRIB" == "conda" ]]; then fi if [[ "$UNAMESTR" == "Darwin" ]]; then - # on macOS, install an OpenMP-enabled clang/llvm from conda-forge - TO_INSTALL="$TO_INSTALL conda-forge::compilers conda-forge::llvm-openmp" + if [[ "$SKLEARN_TEST_NO_OPENMP" != "true" ]]; then + # on macOS, install an OpenMP-enabled clang/llvm from conda-forge. + TO_INSTALL="$TO_INSTALL conda-forge::compilers \ + conda-forge::llvm-openmp" + fi fi # Old packages coming from the 'free' conda channel have been removed but diff --git a/build_tools/azure/posix-32.yml b/build_tools/azure/posix-32.yml index 4a87d180364c7..68e05e347f307 100644 --- a/build_tools/azure/posix-32.yml +++ b/build_tools/azure/posix-32.yml @@ -37,7 +37,6 @@ jobs: -e VIRTUALENV=testvenv -e JOBLIB_VERSION=$JOBLIB_VERSION -e PYTEST_VERSION=$PYTEST_VERSION - -e SKLEARN_NO_OPENMP=$SKLEARN_NO_OPENMP -e OMP_NUM_THREADS=$OMP_NUM_THREADS -e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS -e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS diff --git a/doc/developers/advanced_installation.rst b/doc/developers/advanced_installation.rst index 6f73f982fda63..f5b5f18521e34 100644 --- a/doc/developers/advanced_installation.rst +++ b/doc/developers/advanced_installation.rst @@ -114,10 +114,12 @@ Building Scikit-learn also requires: .. note:: - It is possible to build scikit-learn without OpenMP support by setting the - ``SKLEARN_NO_OPENMP`` environment variable (before cythonization). This is - not recommended since it will force some estimators to run in sequential - mode. + If OpenMP is not supported by the compiler, the build will be done with + OpenMP functionalities disabled. This is not recommended since it will force + some estimators to run in sequential mode instead of leveraging thread-based + parallelism. Setting the ``SKLEARN_FAIL_NO_OPENMP`` environment variable + (before cythonization) will force the build to fail if OpenMP is not + supported. Since version 0.21, scikit-learn automatically detects and use the linear algebrea library used by SciPy **at runtime**. Scikit-learn has therefore no diff --git a/doc/developers/performance.rst b/doc/developers/performance.rst index 85ab1e6776504..1be0dc9b575e1 100644 --- a/doc/developers/performance.rst +++ b/doc/developers/performance.rst @@ -332,16 +332,16 @@ memory alignment, direct blas calls... Using OpenMP ------------ -Since scikit-learn can be built without OpenMP support, it's necessary to +Since scikit-learn can be built without OpenMP, it's necessary to protect each direct call to OpenMP. This can be done using the following syntax:: # importing OpenMP - IF SKLEARN_OPENMP_SUPPORTED: + IF SKLEARN_OPENMP_PARALLELISM_ENABLED: cimport openmp # calling OpenMP - IF SKLEARN_OPENMP_SUPPORTED: + IF SKLEARN_OPENMP_PARALLELISM_ENABLED: max_threads = openmp.omp_get_max_threads() ELSE: max_threads = 1 diff --git a/setup.py b/setup.py index dc0791f609ca6..e444fcb07d5b1 100755 --- a/setup.py +++ b/setup.py @@ -125,7 +125,7 @@ class build_ext_subclass(build_ext): def build_extensions(self): from sklearn._build_utils.openmp_helpers import get_openmp_flag - if not os.getenv('SKLEARN_NO_OPENMP'): + if sklearn._OPENMP_SUPPORTED: openmp_flag = get_openmp_flag(self.compiler) for e in self.extensions: diff --git a/sklearn/_build_utils/__init__.py b/sklearn/_build_utils/__init__.py index aea76d8fd42a6..b3697c3dcf9e4 100644 --- a/sklearn/_build_utils/__init__.py +++ b/sklearn/_build_utils/__init__.py @@ -6,9 +6,12 @@ import os -from distutils.version import LooseVersion +import sklearn import contextlib +from distutils.version import LooseVersion + +from .pre_build_helpers import basic_check_build from .openmp_helpers import check_openmp_support @@ -42,7 +45,24 @@ def cythonize_extensions(top_path, config): _check_cython_version() from Cython.Build import cythonize - with_openmp = check_openmp_support() + # Fast fail before cythonization if compiler fails compiling basic test + # code even without OpenMP + basic_check_build() + + # check simple compilation with OpenMP. If it fails scikit-learn will be + # built without OpenMP and the test test_openmp_supported in the test suite + # will fail. + # `check_openmp_support` compiles a small test program to see if the + # compilers are properly configured to build with OpenMP. This is expensive + # and we only want to call this function once. + # The result of this check is cached as a private attribute on the sklearn + # module (only at build-time) to be used twice: + # - First to set the value of SKLEARN_OPENMP_PARALLELISM_ENABLED, the + # cython build-time variable passed to the cythonize() call. + # - Then in the build_ext subclass defined in the top-level setup.py file + # to actually build the compiled extensions with OpenMP flags if needed. + sklearn._OPENMP_SUPPORTED = check_openmp_support() + n_jobs = 1 with contextlib.suppress(ImportError): import joblib @@ -55,7 +75,8 @@ def cythonize_extensions(top_path, config): config.ext_modules = cythonize( config.ext_modules, nthreads=n_jobs, - compile_time_env={'SKLEARN_OPENMP_SUPPORTED': with_openmp}, + compile_time_env={ + 'SKLEARN_OPENMP_PARALLELISM_ENABLED': sklearn._OPENMP_SUPPORTED}, compiler_directives={'language_level': 3}) diff --git a/sklearn/_build_utils/openmp_helpers.py b/sklearn/_build_utils/openmp_helpers.py index e0b23f756b4aa..d4c377c67e05f 100644 --- a/sklearn/_build_utils/openmp_helpers.py +++ b/sklearn/_build_utils/openmp_helpers.py @@ -6,26 +6,13 @@ import os import sys -import glob -import tempfile import textwrap +import warnings import subprocess -from numpy.distutils.ccompiler import new_compiler -from distutils.sysconfig import customize_compiler from distutils.errors import CompileError, LinkError - -CCODE = textwrap.dedent( - """\ - #include - #include - int main(void) { - #pragma omp parallel - printf("nthreads=%d\\n", omp_get_num_threads()); - return 0; - } - """) +from .pre_build_helpers import compile_test_program def get_openmp_flag(compiler): @@ -59,85 +46,69 @@ def get_openmp_flag(compiler): def check_openmp_support(): """Check whether OpenMP test code can be compiled and run""" - ccompiler = new_compiler() - customize_compiler(ccompiler) - - if os.getenv('SKLEARN_NO_OPENMP'): - # Build explicitly without OpenMP support - return False - - start_dir = os.path.abspath('.') - - with tempfile.TemporaryDirectory() as tmp_dir: - try: - os.chdir(tmp_dir) - - # Write test program - with open('test_openmp.c', 'w') as f: - f.write(CCODE) - - os.mkdir('objects') + code = textwrap.dedent( + """\ + #include + #include + int main(void) { + #pragma omp parallel + printf("nthreads=%d\\n", omp_get_num_threads()); + return 0; + } + """) - # Compile, test program - openmp_flags = get_openmp_flag(ccompiler) - ccompiler.compile(['test_openmp.c'], output_dir='objects', - extra_postargs=openmp_flags) + extra_preargs = os.getenv('LDFLAGS', None) + if extra_preargs is not None: + extra_preargs = extra_preargs.strip().split(" ") + extra_preargs = [ + flag for flag in extra_preargs + if flag.startswith(('-L', '-Wl,-rpath', '-l'))] - # Link test program - extra_preargs = os.getenv('LDFLAGS', None) - if extra_preargs is not None: - extra_preargs = extra_preargs.strip().split(" ") - extra_preargs = [ - flag for flag in extra_preargs - if flag.startswith(('-L', '-Wl,-rpath', '-l'))] + extra_postargs = get_openmp_flag - objects = glob.glob( - os.path.join('objects', '*' + ccompiler.obj_extension)) - ccompiler.link_executable(objects, 'test_openmp', + try: + output = compile_test_program(code, extra_preargs=extra_preargs, - extra_postargs=openmp_flags) - - # Run test program - output = subprocess.check_output('./test_openmp') - output = output.decode(sys.stdout.encoding or 'utf-8').splitlines() - - # Check test program output - if 'nthreads=' in output[0]: - nthreads = int(output[0].strip().split('=')[1]) - openmp_supported = (len(output) == nthreads) - else: - openmp_supported = False + extra_postargs=extra_postargs) - except (CompileError, LinkError, subprocess.CalledProcessError): + if 'nthreads=' in output[0]: + nthreads = int(output[0].strip().split('=')[1]) + openmp_supported = len(output) == nthreads + else: openmp_supported = False - finally: - os.chdir(start_dir) + except (CompileError, LinkError, subprocess.CalledProcessError): + openmp_supported = False - err_message = textwrap.dedent( - """ - *** + if not openmp_supported: + if os.getenv("SKLEARN_FAIL_NO_OPENMP"): + raise CompileError("Failed to build with OpenMP") + else: + message = textwrap.dedent( + """ - It seems that scikit-learn cannot be built with OpenMP support. + *********** + * WARNING * + *********** - - Make sure you have followed the installation instructions: + It seems that scikit-learn cannot be built with OpenMP. - https://scikit-learn.org/dev/developers/advanced_installation.html + - Make sure you have followed the installation instructions: - - If your compiler supports OpenMP but the build still fails, please - submit a bug report at: + https://scikit-learn.org/dev/developers/advanced_installation.html - https://github.com/scikit-learn/scikit-learn/issues + - If your compiler supports OpenMP but you still see this + message, please submit a bug report at: - - If you want to build scikit-learn without OpenMP support, you can set - the environment variable SKLEARN_NO_OPENMP and rerun the build - command. Note however that some estimators will run in sequential - mode and their `n_jobs` parameter will have no effect anymore. + https://github.com/scikit-learn/scikit-learn/issues - *** - """) + - The build will continue with OpenMP-based parallelism + disabled. Note however that some estimators will run in + sequential mode instead of leveraging thread-based + parallelism. - if not openmp_supported: - raise CompileError(err_message) + *** + """) + warnings.warn(message) - return True + return openmp_supported diff --git a/sklearn/_build_utils/pre_build_helpers.py b/sklearn/_build_utils/pre_build_helpers.py new file mode 100644 index 0000000000000..bc3d83257dd7e --- /dev/null +++ b/sklearn/_build_utils/pre_build_helpers.py @@ -0,0 +1,70 @@ +"""Helpers to check build environment before actual build of scikit-learn""" + +import os +import sys +import glob +import tempfile +import textwrap +import subprocess + +from distutils.sysconfig import customize_compiler +from numpy.distutils.ccompiler import new_compiler + + +def compile_test_program(code, extra_preargs=[], extra_postargs=[]): + """Check that some C code can be compiled and run""" + ccompiler = new_compiler() + customize_compiler(ccompiler) + + # extra_(pre/post)args can be a callable to make it possible to get its + # value from the compiler + if callable(extra_preargs): + extra_preargs = extra_preargs(ccompiler) + if callable(extra_postargs): + extra_postargs = extra_postargs(ccompiler) + + start_dir = os.path.abspath('.') + + with tempfile.TemporaryDirectory() as tmp_dir: + try: + os.chdir(tmp_dir) + + # Write test program + with open('test_program.c', 'w') as f: + f.write(code) + + os.mkdir('objects') + + # Compile, test program + ccompiler.compile(['test_program.c'], output_dir='objects', + extra_postargs=extra_postargs) + + # Link test program + objects = glob.glob( + os.path.join('objects', '*' + ccompiler.obj_extension)) + ccompiler.link_executable(objects, 'test_program', + extra_preargs=extra_preargs, + extra_postargs=extra_postargs) + + # Run test program + # will raise a CalledProcessError if return code was non-zero + output = subprocess.check_output('./test_program') + output = output.decode(sys.stdout.encoding or 'utf-8').splitlines() + except Exception: + raise + finally: + os.chdir(start_dir) + + return output + + +def basic_check_build(): + """Check basic compilation and linking of C code""" + code = textwrap.dedent( + """\ + #include + int main(void) { + return 0; + } + """) + compile_test_program(code) diff --git a/sklearn/ensemble/_hist_gradient_boosting/splitting.pyx b/sklearn/ensemble/_hist_gradient_boosting/splitting.pyx index fda060e238514..0e74d6ba38c71 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/splitting.pyx +++ b/sklearn/ensemble/_hist_gradient_boosting/splitting.pyx @@ -15,7 +15,7 @@ cimport cython from cython.parallel import prange import numpy as np cimport numpy as np -IF SKLEARN_OPENMP_SUPPORTED: +IF SKLEARN_OPENMP_PARALLELISM_ENABLED: from openmp cimport omp_get_max_threads from libc.stdlib cimport malloc, free from libc.string cimport memcpy @@ -256,7 +256,7 @@ cdef class Splitter: unsigned int [::1] left_indices_buffer = self.left_indices_buffer unsigned int [::1] right_indices_buffer = self.right_indices_buffer - IF SKLEARN_OPENMP_SUPPORTED: + IF SKLEARN_OPENMP_PARALLELISM_ENABLED: int n_threads = omp_get_max_threads() ELSE: int n_threads = 1 diff --git a/sklearn/tests/test_build.py b/sklearn/tests/test_build.py new file mode 100644 index 0000000000000..36c4f7ee062dc --- /dev/null +++ b/sklearn/tests/test_build.py @@ -0,0 +1,32 @@ +import os +import pytest +import textwrap + +from sklearn import __version__ +from sklearn.utils._openmp_helpers import _openmp_parallelism_enabled + + +def test_openmp_parallelism_enabled(): + # Check that sklearn is built with OpenMP-based parallelism enabled. + # This test can be skipped by setting the environment variable + # ``SKLEARN_SKIP_OPENMP_TEST``. + if os.getenv("SKLEARN_SKIP_OPENMP_TEST"): + pytest.skip("test explicitly skipped (SKLEARN_SKIP_OPENMP_TEST)") + + base_url = "dev" if __version__.endswith(".dev0") else "stable" + err_msg = textwrap.dedent( + """ + This test fails because scikit-learn has been built without OpenMP. + This is not recommended since some estimators will run in sequential + mode instead of leveraging thread-based parallelism. + + You can find instructions to build scikit-learn with OpenMP at this + address: + + https://scikit-learn.org/{}/developers/advanced_installation.html + + You can skip this test by setting the environment variable + SKLEARN_SKIP_OPENMP_TEST to any value. + """).format(base_url) + + assert _openmp_parallelism_enabled(), err_msg diff --git a/sklearn/tests/test_common.py b/sklearn/tests/test_common.py index 33eee4ae554a5..221dd52834c90 100644 --- a/sklearn/tests/test_common.py +++ b/sklearn/tests/test_common.py @@ -139,16 +139,6 @@ def test_configure(): old_argv = sys.argv sys.argv = ['setup.py', 'config'] - # This test will run every setup.py and eventually call - # check_openmp_support(), which tries to compile a C file that uses - # OpenMP, unless SKLEARN_NO_OPENMP is set. Some users might want to run - # the tests without having build-support for OpenMP. In particular, mac - # users need to set some environment variables to build with openmp - # support, and these might not be set anymore at test time. We thus - # temporarily set SKLEARN_NO_OPENMP, so that this test runs smoothly. - old_env = os.getenv('SKLEARN_NO_OPENMP') - os.environ['SKLEARN_NO_OPENMP'] = "True" - with warnings.catch_warnings(): # The configuration spits out warnings when not finding # Blas/Atlas development headers @@ -157,10 +147,6 @@ def test_configure(): exec(f.read(), dict(__name__='__main__')) finally: sys.argv = old_argv - if old_env is not None: - os.environ['SKLEARN_NO_OPENMP'] = old_env - else: - del os.environ['SKLEARN_NO_OPENMP'] os.chdir(cwd) diff --git a/sklearn/utils/_openmp_helpers.pyx b/sklearn/utils/_openmp_helpers.pyx index 2986b4db74c7f..fb8920074a84e 100644 --- a/sklearn/utils/_openmp_helpers.pyx +++ b/sklearn/utils/_openmp_helpers.pyx @@ -1,9 +1,20 @@ -IF SKLEARN_OPENMP_SUPPORTED: +IF SKLEARN_OPENMP_PARALLELISM_ENABLED: import os cimport openmp from joblib import cpu_count +def _openmp_parallelism_enabled(): + """Determines whether scikit-learn has been built with OpenMP + + It allows to retrieve at runtime the information gathered at compile time. + """ + # SKLEARN_OPENMP_PARALLELISM_ENABLED is resolved at compile time during + # cythonization. It is defined via the `compile_time_env` kwarg of the + # `cythonize` call and behaves like the `-D` option of the C preprocessor. + return SKLEARN_OPENMP_PARALLELISM_ENABLED + + cpdef _openmp_effective_n_threads(n_threads=None): """Determine the effective number of threads to be used for OpenMP calls @@ -30,7 +41,7 @@ cpdef _openmp_effective_n_threads(n_threads=None): if n_threads == 0: raise ValueError("n_threads = 0 is invalid") - IF SKLEARN_OPENMP_SUPPORTED: + IF SKLEARN_OPENMP_PARALLELISM_ENABLED: if os.getenv("OMP_NUM_THREADS"): # Fall back to user provided number of threads making it possible # to exceed the number of cpus. diff --git a/sklearn/utils/_show_versions.py b/sklearn/utils/_show_versions.py index 75243caeab1a2..53bcf2f35269d 100644 --- a/sklearn/utils/_show_versions.py +++ b/sklearn/utils/_show_versions.py @@ -9,6 +9,8 @@ import sys import importlib +from ._openmp_helpers import _openmp_parallelism_enabled + def _get_sys_info(): """System information @@ -71,7 +73,7 @@ def get_version(module): def show_versions(): - "Print useful debugging information" + """Print useful debugging information""" sys_info = _get_sys_info() deps_info = _get_deps_info() @@ -80,6 +82,9 @@ def show_versions(): for k, stat in sys_info.items(): print("{k:>10}: {stat}".format(k=k, stat=stat)) - print('\nPython deps:') + print('\nPython dependencies:') for k, stat in deps_info.items(): print("{k:>10}: {stat}".format(k=k, stat=stat)) + + print("\n{k:>10}: {stat}".format(k="Built with OpenMP", + stat=_openmp_parallelism_enabled()))