From d8439fd7cf8b41bac60970952dbe9778c91ef461 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 26 Jun 2019 14:08:16 +0200 Subject: [PATCH 01/12] openmp helper equivalent of effective_n_jobs --- sklearn/utils/openmp_helpers.pyx | 26 ++++++++++++++++++++++++++ sklearn/utils/setup.py | 4 ++++ 2 files changed, 30 insertions(+) create mode 100644 sklearn/utils/openmp_helpers.pyx diff --git a/sklearn/utils/openmp_helpers.pyx b/sklearn/utils/openmp_helpers.pyx new file mode 100644 index 0000000000000..5c8c06cd160cf --- /dev/null +++ b/sklearn/utils/openmp_helpers.pyx @@ -0,0 +1,26 @@ +cimport openmp +from joblib import effective_n_jobs + + +cpdef _openmp_effective_n_threads(n_threads=None): + """Determine the effective number of threads used for parallel OpenMP calls + + - For ``n_threads = None``, returns the minimum between + openmp.omp_get_max_threads() and joblib.effective_n_jobs(-1). + - For ``n_threads > 0``, use this as the maximal number of threads for + parallel OpenMP calls. + - For ``n_threads < 0``, use the maximal number of threads minus + ``|n_threads + 1|``. + - Raise a ValueError for ``n_threads = 0``. + """ + if n_threads == 0: + raise ValueError("n_threads = 0 is invalid") + + max_threads = min(openmp.omp_get_max_threads(), effective_n_jobs(-1)) + + if n_threads is None: + return max_threads + elif n_threads < 0: + return max(1, max_threads + n_threads + 1) + + return n_threads diff --git a/sklearn/utils/setup.py b/sklearn/utils/setup.py index f3002ed3ffed9..593739915f3f8 100644 --- a/sklearn/utils/setup.py +++ b/sklearn/utils/setup.py @@ -46,6 +46,10 @@ def configuration(parent_package='', top_path=None): include_dirs=[numpy.get_include()], libraries=libraries) + config.add_extension('openmp_helpers', + sources=['openmp_helpers.pyx'], + libraries=libraries) + # generate files from a template pyx_templates = ['sklearn/utils/seq_dataset.pyx.tp', 'sklearn/utils/seq_dataset.pxd.tp'] From b8900ab23c7311f3508ca9c684b584c4876a0af6 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 26 Jun 2019 14:58:03 +0200 Subject: [PATCH 02/12] protect openmp calls --- sklearn/utils/openmp_helpers.pyx | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/sklearn/utils/openmp_helpers.pyx b/sklearn/utils/openmp_helpers.pyx index 5c8c06cd160cf..ff972de2d10e3 100644 --- a/sklearn/utils/openmp_helpers.pyx +++ b/sklearn/utils/openmp_helpers.pyx @@ -1,4 +1,6 @@ -cimport openmp +IF SKLEARN_OPENMP_SUPPORTED: + cimport openmp + from joblib import effective_n_jobs @@ -12,15 +14,23 @@ cpdef _openmp_effective_n_threads(n_threads=None): - For ``n_threads < 0``, use the maximal number of threads minus ``|n_threads + 1|``. - Raise a ValueError for ``n_threads = 0``. + + If scikit-learn is built without OpenMP support, always return 1. """ if n_threads == 0: raise ValueError("n_threads = 0 is invalid") - max_threads = min(openmp.omp_get_max_threads(), effective_n_jobs(-1)) + IF SKLEARN_OPENMP_SUPPORTED: + max_threads = min(openmp.omp_get_max_threads(), effective_n_jobs(-1)) + + if n_threads is None: + return max_threads + elif n_threads < 0: + return max(1, max_threads + n_threads + 1) - if n_threads is None: - return max_threads - elif n_threads < 0: - return max(1, max_threads + n_threads + 1) + return n_threads + ELSE: + # OpenMP not supported => sequential mode + return 1 - return n_threads + From e47bdb842765a6fdf1178f52887a0d87864e8274 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 26 Jun 2019 17:30:27 +0200 Subject: [PATCH 03/12] comment openmp max threads --- sklearn/utils/openmp_helpers.pyx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sklearn/utils/openmp_helpers.pyx b/sklearn/utils/openmp_helpers.pyx index ff972de2d10e3..d45ba73595028 100644 --- a/sklearn/utils/openmp_helpers.pyx +++ b/sklearn/utils/openmp_helpers.pyx @@ -21,12 +21,14 @@ cpdef _openmp_effective_n_threads(n_threads=None): raise ValueError("n_threads = 0 is invalid") IF SKLEARN_OPENMP_SUPPORTED: - max_threads = min(openmp.omp_get_max_threads(), effective_n_jobs(-1)) + # omp_get_max_threads can be influenced by environement variable + # OMP_NUM_THREADS or at runtime by omp_set_num_threads + max_n_threads = min(openmp.omp_get_max_threads(), effective_n_jobs(-1)) if n_threads is None: - return max_threads + return max_n_threads elif n_threads < 0: - return max(1, max_threads + n_threads + 1) + return max(1, max_n_threads + n_threads + 1) return n_threads ELSE: From 8050149a2a3a59b8a0117d51234841a64cffc4cd Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 26 Jun 2019 17:32:14 +0200 Subject: [PATCH 04/12] right place comment --- sklearn/utils/openmp_helpers.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/utils/openmp_helpers.pyx b/sklearn/utils/openmp_helpers.pyx index d45ba73595028..296473aa13499 100644 --- a/sklearn/utils/openmp_helpers.pyx +++ b/sklearn/utils/openmp_helpers.pyx @@ -9,6 +9,8 @@ cpdef _openmp_effective_n_threads(n_threads=None): - For ``n_threads = None``, returns the minimum between openmp.omp_get_max_threads() and joblib.effective_n_jobs(-1). + The result of ``omp_get_max_threads`` can be influenced by environement + variable ``OMP_NUM_THREADS`` or at runtime by ``omp_set_num_threads``. - For ``n_threads > 0``, use this as the maximal number of threads for parallel OpenMP calls. - For ``n_threads < 0``, use the maximal number of threads minus @@ -21,8 +23,6 @@ cpdef _openmp_effective_n_threads(n_threads=None): raise ValueError("n_threads = 0 is invalid") IF SKLEARN_OPENMP_SUPPORTED: - # omp_get_max_threads can be influenced by environement variable - # OMP_NUM_THREADS or at runtime by omp_set_num_threads max_n_threads = min(openmp.omp_get_max_threads(), effective_n_jobs(-1)) if n_threads is None: From 280f5516f895007bf26a5a6940b580709f1747e5 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 5 Jul 2019 15:18:58 +0200 Subject: [PATCH 05/12] don't import joblib if unecessary --- sklearn/utils/openmp_helpers.pyx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sklearn/utils/openmp_helpers.pyx b/sklearn/utils/openmp_helpers.pyx index 296473aa13499..9fb01b3d29e2f 100644 --- a/sklearn/utils/openmp_helpers.pyx +++ b/sklearn/utils/openmp_helpers.pyx @@ -1,7 +1,6 @@ IF SKLEARN_OPENMP_SUPPORTED: cimport openmp - -from joblib import effective_n_jobs + from joblib import effective_n_jobs cpdef _openmp_effective_n_threads(n_threads=None): From 2623403b393a357af3fff6963ebb1552bb462adf Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 11 Oct 2019 17:47:50 +0200 Subject: [PATCH 06/12] make module private --- sklearn/utils/{openmp_helpers.pyx => _openmp_helpers.pyx} | 0 sklearn/utils/setup.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename sklearn/utils/{openmp_helpers.pyx => _openmp_helpers.pyx} (100%) diff --git a/sklearn/utils/openmp_helpers.pyx b/sklearn/utils/_openmp_helpers.pyx similarity index 100% rename from sklearn/utils/openmp_helpers.pyx rename to sklearn/utils/_openmp_helpers.pyx diff --git a/sklearn/utils/setup.py b/sklearn/utils/setup.py index 5deadab71ec07..584d8b2ae5b7e 100644 --- a/sklearn/utils/setup.py +++ b/sklearn/utils/setup.py @@ -41,8 +41,8 @@ def configuration(parent_package='', top_path=None): include_dirs=[numpy.get_include()], libraries=libraries) - config.add_extension('openmp_helpers', - sources=['openmp_helpers.pyx'], + config.add_extension('_openmp_helpers', + sources=['_openmp_helpers.pyx'], libraries=libraries) # generate files from a template From 322f49904f5ab2365976a401b38d96d4959dc0da Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Mon, 14 Oct 2019 11:18:08 +0200 Subject: [PATCH 07/12] revert ht eunintended revert | cln --- sklearn/utils/_openmp_helpers.pyx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sklearn/utils/_openmp_helpers.pyx b/sklearn/utils/_openmp_helpers.pyx index 9fb01b3d29e2f..97bc9ac0ad667 100644 --- a/sklearn/utils/_openmp_helpers.pyx +++ b/sklearn/utils/_openmp_helpers.pyx @@ -1,19 +1,20 @@ IF SKLEARN_OPENMP_SUPPORTED: cimport openmp - from joblib import effective_n_jobs + from . import cpu_count cpdef _openmp_effective_n_threads(n_threads=None): """Determine the effective number of threads used for parallel OpenMP calls - - For ``n_threads = None``, returns the minimum between - openmp.omp_get_max_threads() and joblib.effective_n_jobs(-1). - The result of ``omp_get_max_threads`` can be influenced by environement + - For ``n_threads = None``, returns the minimum between the number of cpus + and ``openmp.omp_get_max_threads()``. + The result of ``omp_get_max_threads`` can be influenced by environment variable ``OMP_NUM_THREADS`` or at runtime by ``omp_set_num_threads``. - For ``n_threads > 0``, use this as the maximal number of threads for parallel OpenMP calls. - For ``n_threads < 0``, use the maximal number of threads minus - ``|n_threads + 1|``. + ``|n_threads + 1|``. In particular ``n_threads = -1`` will use as many + threads as there are available cores on the machine. - Raise a ValueError for ``n_threads = 0``. If scikit-learn is built without OpenMP support, always return 1. @@ -22,7 +23,7 @@ cpdef _openmp_effective_n_threads(n_threads=None): raise ValueError("n_threads = 0 is invalid") IF SKLEARN_OPENMP_SUPPORTED: - max_n_threads = min(openmp.omp_get_max_threads(), effective_n_jobs(-1)) + max_n_threads = min(openmp.omp_get_max_threads(), cpu_count()) if n_threads is None: return max_n_threads From 34d77350349ed9062936e09e123ec622720c652b Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 16 Oct 2019 15:31:25 +0200 Subject: [PATCH 08/12] allow more user control + more comments --- sklearn/utils/_openmp_helpers.pyx | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/sklearn/utils/_openmp_helpers.pyx b/sklearn/utils/_openmp_helpers.pyx index 97bc9ac0ad667..7e06ab85b765d 100644 --- a/sklearn/utils/_openmp_helpers.pyx +++ b/sklearn/utils/_openmp_helpers.pyx @@ -4,10 +4,12 @@ IF SKLEARN_OPENMP_SUPPORTED: cpdef _openmp_effective_n_threads(n_threads=None): - """Determine the effective number of threads used for parallel OpenMP calls + """Determine the effective number of threads to be used for OpenMP calls - - For ``n_threads = None``, returns the minimum between the number of cpus - and ``openmp.omp_get_max_threads()``. + - For ``n_threads = None``, returns the minimum between + ``openmp.omp_get_max_threads()`` and the number of cpus, taking cgroups + quotas into account. + Cgroups quotas can typically be set by tools such as Docker. The result of ``omp_get_max_threads`` can be influenced by environment variable ``OMP_NUM_THREADS`` or at runtime by ``omp_set_num_threads``. - For ``n_threads > 0``, use this as the maximal number of threads for @@ -23,7 +25,14 @@ cpdef _openmp_effective_n_threads(n_threads=None): raise ValueError("n_threads = 0 is invalid") IF SKLEARN_OPENMP_SUPPORTED: - max_n_threads = min(openmp.omp_get_max_threads(), cpu_count()) + if os.environ.get("OMP_NUM_THREADS"): + # Fall back to user provided number of threads making it possible + # to exceed the number of cpus. + # It's however inconsistent with `omp_set_num_threads` which can't + # be used this purpose. + max_n_threads = openmp.omp_get_max_threads() + else: + max_n_threads = min(openmp.omp_get_max_threads(), cpu_count()) if n_threads is None: return max_n_threads From 41691acfb9b45d93725d9787d70c88237dce910d Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 16 Oct 2019 15:54:31 +0200 Subject: [PATCH 09/12] fix indent --- sklearn/utils/_openmp_helpers.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sklearn/utils/_openmp_helpers.pyx b/sklearn/utils/_openmp_helpers.pyx index 7e06ab85b765d..51527d23cd8b4 100644 --- a/sklearn/utils/_openmp_helpers.pyx +++ b/sklearn/utils/_openmp_helpers.pyx @@ -25,14 +25,14 @@ cpdef _openmp_effective_n_threads(n_threads=None): raise ValueError("n_threads = 0 is invalid") IF SKLEARN_OPENMP_SUPPORTED: - if os.environ.get("OMP_NUM_THREADS"): + if os.getenv("OMP_NUM_THREADS"): # Fall back to user provided number of threads making it possible # to exceed the number of cpus. # It's however inconsistent with `omp_set_num_threads` which can't # be used this purpose. - max_n_threads = openmp.omp_get_max_threads() + max_n_threads = openmp.omp_get_max_threads() else: - max_n_threads = min(openmp.omp_get_max_threads(), cpu_count()) + max_n_threads = min(openmp.omp_get_max_threads(), cpu_count()) if n_threads is None: return max_n_threads From fe1f75f58b78ad40dfa12e47cc927ef1910e1723 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 16 Oct 2019 16:01:04 +0200 Subject: [PATCH 10/12] import os :) --- sklearn/utils/_openmp_helpers.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/sklearn/utils/_openmp_helpers.pyx b/sklearn/utils/_openmp_helpers.pyx index 51527d23cd8b4..dd4e50365b171 100644 --- a/sklearn/utils/_openmp_helpers.pyx +++ b/sklearn/utils/_openmp_helpers.pyx @@ -1,4 +1,5 @@ IF SKLEARN_OPENMP_SUPPORTED: + import os cimport openmp from . import cpu_count From 11be7123b0191024e3268c744f7648822a952b45 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 16 Oct 2019 17:39:45 +0200 Subject: [PATCH 11/12] address comments --- sklearn/utils/_openmp_helpers.pyx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sklearn/utils/_openmp_helpers.pyx b/sklearn/utils/_openmp_helpers.pyx index dd4e50365b171..0affcbedd5612 100644 --- a/sklearn/utils/_openmp_helpers.pyx +++ b/sklearn/utils/_openmp_helpers.pyx @@ -29,8 +29,6 @@ cpdef _openmp_effective_n_threads(n_threads=None): if os.getenv("OMP_NUM_THREADS"): # Fall back to user provided number of threads making it possible # to exceed the number of cpus. - # It's however inconsistent with `omp_set_num_threads` which can't - # be used this purpose. max_n_threads = openmp.omp_get_max_threads() else: max_n_threads = min(openmp.omp_get_max_threads(), cpu_count()) @@ -42,7 +40,7 @@ cpdef _openmp_effective_n_threads(n_threads=None): return n_threads ELSE: - # OpenMP not supported => sequential mode + # OpenMP disabled at build-time => sequential mode return 1 From eca64f75c9da2218b5ee0b23f0772baa14b8cde2 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 16 Oct 2019 20:52:18 +0200 Subject: [PATCH 12/12] better docstring --- sklearn/utils/_openmp_helpers.pyx | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/sklearn/utils/_openmp_helpers.pyx b/sklearn/utils/_openmp_helpers.pyx index 0affcbedd5612..2986b4db74c7f 100644 --- a/sklearn/utils/_openmp_helpers.pyx +++ b/sklearn/utils/_openmp_helpers.pyx @@ -1,23 +1,28 @@ IF SKLEARN_OPENMP_SUPPORTED: import os cimport openmp - from . import cpu_count + from joblib import cpu_count cpdef _openmp_effective_n_threads(n_threads=None): """Determine the effective number of threads to be used for OpenMP calls - - For ``n_threads = None``, returns the minimum between - ``openmp.omp_get_max_threads()`` and the number of cpus, taking cgroups - quotas into account. - Cgroups quotas can typically be set by tools such as Docker. + - For ``n_threads = None``, + - if the ``OMP_NUM_THREADS`` environment variable is set, return + ``openmp.omp_get_max_threads()`` + - otherwise, return the minimum between ``openmp.omp_get_max_threads()`` + and the number of cpus, taking cgroups quotas into account. Cgroups + quotas can typically be set by tools such as Docker. The result of ``omp_get_max_threads`` can be influenced by environment variable ``OMP_NUM_THREADS`` or at runtime by ``omp_set_num_threads``. - - For ``n_threads > 0``, use this as the maximal number of threads for + + - For ``n_threads > 0``, return this as the maximal number of threads for parallel OpenMP calls. - - For ``n_threads < 0``, use the maximal number of threads minus + + - For ``n_threads < 0``, return the maximal number of threads minus ``|n_threads + 1|``. In particular ``n_threads = -1`` will use as many threads as there are available cores on the machine. + - Raise a ValueError for ``n_threads = 0``. If scikit-learn is built without OpenMP support, always return 1.