8000 FIX propagate configuration to workers in parallel (#25363) · scikit-learn/scikit-learn@79da465 · GitHub
[go: up one dir, main page]

Skip to content

Commit 79da465

Browse files
glemaitreogriselthomasjpfan
authored andcommi 8000 tted
FIX propagate configuration to workers in parallel (#25363)
Co-authored-by: Olivier Grisel <olivier.grisel@ensta.org> Co-authored-by: Thomas J. Fan <thomasjpfan@gmail.com>
1 parent 1d6d023 commit 79da465

40 files changed

+314
-116
lines changed

benchmarks/bench_saga.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,7 @@
77
import time
88
import os
99

10-
from joblib import Parallel
11-
from sklearn.utils.fixes import delayed
10+
from sklearn.utils.parallel import delayed, Parallel
1211
import matplotlib.pyplot as plt
1312
import numpy as np
1413

build_tools/azure/linting.sh

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,15 @@ then
3434
exit 1
3535
fi
3636

37-
joblib_import="$(git grep -l -A 10 -E "joblib import.+delayed" -- "*.py" ":!sklearn/utils/_joblib.py" ":!sklearn/utils/fixes.py")"
38-
39-
if [ ! -z "$joblib_import" ]; then
40-
echo "Use from sklearn.utils.fixes import delayed instead of joblib delayed. The following files contains imports to joblib.delayed:"
41-
echo "$joblib_import"
37+
joblib_delayed_import="$(git grep -l -A 10 -E "joblib import.+delayed" -- "*.py" ":!sklearn/utils/_joblib.py" ":!sklearn/utils/parallel.py")"
38+
if [ ! -z "$joblib_delayed_import" ]; then
39+
echo "Use from sklearn.utils.parallel import delayed instead of joblib delayed. The following files contains imports to joblib.delayed:"
40+
echo "$joblib_delayed_import"
41+
exit 1
42+
fi
43+
joblib_Parallel_import="$(git grep -l -A 10 -E "joblib import.+Parallel" -- "*.py" ":!sklearn/utils/_joblib.py" ":!sklearn/utils/parallel.py")"
44+
if [ ! -z "$joblib_Parallel_import" ]; then
45+
echo "Use from sklearn.utils.parallel import Parallel instead of joblib Parallel. The following files contains imports to joblib.Parallel:"
46+
echo "$joblib_Parallel_import"
4247
exit 1
4348
fi

doc/modules/classes.rst

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1666,9 +1666,16 @@ Utilities from joblib:
16661666
:toctree: generated/
16671667
:template: function.rst
16681668

1669+
utils.parallel.delayed
16691670
utils.parallel_backend
16701671
utils.register_parallel_backend
16711672

1673+
.. autosummary::
1674+
:toctree: generated/
1675+
:template: class.rst
1676+
1677+
utils.parallel.Parallel
1678+
16721679

16731680
Recently deprecated
16741681
===================

doc/whats_new/v1.2.rst

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,16 @@ Version 1.2.1
99

1010
**In Development**
1111

12+
Changes impacting all modules
13+
-----------------------------
14+
15+
- |Fix| Fix a bug where the current configuration was ignored in estimators using
16+
`n_jobs > 1`. This bug was triggered for tasks dispatched by the auxillary
17+
thread of `joblib` as :func:`sklearn.get_config` used to access an empty thread
18+
local configuration instead of the configuration visible from the thread where
19+
`joblib.Parallel` was first called.
20+
:pr:`25363` by :user:`Guillaume Lemaitre <glemaitre>`.
21+
1222
Changed models
1323
--------------
1424

@@ -139,6 +149,13 @@ Changelog
139149
boolean. The type is maintained, instead of converting to `float64.`
140150
:pr:`25147` by :user:`Tim Head <betatim>`.
141151

152+
- |API| :func:`utils.fixes.delayed` is deprecated in 1.2.1 and will be removed
153+
in 1.5. Instead, import :func:`utils.parallel.delayed` and use it in
154+
conjunction with the newly introduced :func:`utils.parallel.Parallel`
155+
to ensure proper propagation of the scikit-learn configuration to
156+
the workers.
157+
:pr:`25363` by :user:`Guillaume Lemaitre <glemaitre>`.
158+
142159
.. _changes_1_2:
143160

144161
Version 1.2.0

sklearn/calibration.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414

1515
from math import log
1616
import numpy as np
17-
from joblib import Parallel
1817

1918
from scipy.special import expit
2019
from scipy.special import xlogy
@@ -36,7 +35,7 @@
3635
)
3736
3837
from .utils.multiclass import check_classification_targets
39-
from .utils.fixes import delayed
38+
from .utils.parallel import delayed, Parallel
4039
from .utils._param_validation import StrOptions, HasMethods, Hidden
4140
from .utils.validation import (
4241
_check_fit_params,

sklearn/cluster/_mean_shift.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,12 @@
1616

1717
import numpy as np
1818
import warnings
19-
from joblib import Parallel
2019
from numbers import Integral, Real
2120

2221
from collections import defaultdict
2322
from ..utils._param_validation import Interval
2423
from ..utils.validation import check_is_fitted
25-
from ..utils.fixes import delayed
24+
from ..utils.parallel import delayed, Parallel
2625
from ..utils import check_random_state, gen_batches, check_array
2726
from ..base import BaseEstimator, ClusterMixin
2827
from ..neighbors import NearestNeighbors

sklearn/compose/_column_transformer.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111

1212
import numpy as np
1313
from scipy import sparse
14-
from joblib import Parallel
1514

1615
from ..base import clone, TransformerMixin
1716
from ..utils._estimator_html_repr import _VisualBlock
@@ -24,7 +23,7 @@
2423
from ..utils import check_pandas_support
2524
from ..utils.metaestimators import _BaseComposition
2625
from ..utils.validation import check_array, check_is_fitted, _check_feature_names_in
27-
from ..utils.fixes import delayed
26+
from ..utils.parallel import delayed, Parallel
2827

2928

3029
__all__ = ["ColumnTransformer", "make_column_transformer", "make_column_selector"]

sklearn/covariance/_graph_lasso.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
from numbers import Integral, Real
1414
import numpy as np
1515
from scipy import linalg
16-
from joblib import Parallel
1716

1817
from . import empirical_covariance, EmpiricalCovariance, log_likelihood
1918

@@ -23,7 +22,7 @@
2322
check_random_state,
2423
check_scalar,
2524
)
26-
from ..utils.fixes import delayed
25+
from ..utils.parallel import delayed, Parallel
2726
from ..utils._param_validation import Interval, StrOptions
2827

2928
# mypy error: Module 'sklearn.linear_model' has no attribute '_cd_fast'

sklearn/decomposition/_dict_learning.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,15 @@
1313

1414
import numpy as np
1515
from scipy import linalg
16-
from joblib import Parallel, effective_n_jobs
16+
from joblib import effective_n_jobs
1717

1818
from ..base import BaseEstimator, TransformerMixin, ClassNamePrefixFeaturesOutMixin
1919
from ..utils import check_array, check_random_state, gen_even_slices, gen_batches
2020
from ..utils import deprecated
2121
from ..utils._param_validation import Hidden, Interval, StrOptions
2222
from ..utils.extmath import randomized_svd, row_norms, svd_flip
2323
from ..utils.validation import check_is_fitted
24-
from ..utils.fixes import delayed
24+
from ..utils.parallel import delayed, Parallel
2525
from ..linear_model import Lasso, orthogonal_mp_gram, LassoLars, Lars
2626

2727

sklearn/decomposition/_lda.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,13 @@
1515
import numpy as np
1616
import scipy.sparse as sp
1717
from scipy.special import gammaln, logsumexp
18-
from joblib import Parallel, effective_n_jobs
18+
from joblib import effective_n_jobs
1919

2020
from ..base import BaseEstimator, TransformerMixin, ClassNamePrefixFeaturesOutMixin
2121
from ..utils import check_random_state, gen_batches, gen_even_slices
2222
from ..utils.validation import check_non_negative
2323
from ..utils.validation import check_is_fitted
24-
from ..utils.fixes import delayed
24+
from ..utils.parallel import delayed, Parallel
2525
from ..utils._param_validation import Interval, StrOptions
2626

2727
from ._online_lda_fast import (

0 commit comments

Comments
 (0)
0