8000 API prepare change of default solver QuantileRegressor (#23637) · scikit-learn/scikit-learn@a7e27da · GitHub
[go: up one dir, main page]
Skip to content

Commit a7e27da

Browse files
glemaitreogrisel
glemaitre
and
ogrisel
authored
API prepare change of default solver QuantileRegressor (#23637)
Co-authored-by: Olivier Grisel <olivier.grisel@ensta.org>
1 parent 549fbb8 commit a7e27da

File tree

5 files changed

+116
-35
lines changed

5 files changed

+116
-35
lines changed

‎doc/whats_new/v1.2.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,11 @@ Changelog
146146
`solver="newton-cg"`, `fit_intercept=True`, and a single feature. :pr:`23608`
147147
by `Tom Dupre la Tour`_.
148148

149+
- |API| The default value for the `solver` parameter in
150+
:class:`linear_model.QuantileRegressor` will change from `"interior-point"`
151+
to `"highs"` in version 1.4.
152+
:pr:`23637` by :user:`Guillaume Lemaitre <glemaitre>`.
153+
149154
:mod:`sklearn.metrics`
150155
......................
151156

‎examples/linear_model/plot_quantile_regression.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,13 +111,20 @@
111111
#
112112
# We will use the quantiles at 5% and 95% to find the outliers in the training
113113
# sample beyond the central 90% interval.
114+
from sklearn.utils.fixes import sp_version, parse_version
115+
116+
# This is line is to avoid incompatibility if older SciPy version.
117+
# You should use `solver="highs"` with recent version of SciPy.
118+
solver = "highs" if sp_version >= parse_version("1.6.0") else "interior-point"
119+
120+
# %%
114121
from sklearn.linear_model import QuantileRegressor
115122

116123
quantiles = [0.05, 0.5, 0.95]
117124
predictions = {}
118125
out_bounds_predictions = np.zeros_like(y_true_mean, dtype=np.bool_)
119126
for quantile in quantiles:
120-
qr = QuantileRegressor(quantile=quantile, alpha=0)
127+
qr = QuantileRegressor(quantile=quantile, alpha=0, solver=solver)
121128
y_pred = qr.fit(X, y_normal).predict(X)
122129
predictions[quantile] = y_pred
123130

@@ -179,7 +186,7 @@
179186
predictions = {}
180187
out_bounds_predictions = np.zeros_like(y_true_mean, dtype=np.bool_)
181188
for quantile in quantiles:
182-
qr = QuantileRegressor(quantile=quantile, alpha=0)
189+
qr = QuantileRegressor(quantile=quantile, alpha=0, solver=solver)
183190
y_pred = qr.fit(X, y_pareto).predict(X)
184191
predictions[quantile] = y_pred
185192

@@ -250,7 +257,7 @@
250257
from sklearn.metrics import mean_squared_error
251258

252259
linear_regression = LinearRegression()
253-
quantile_regression = QuantileRegressor(quantile=0.5, alpha=0)
260+
quantile_regression = QuantileRegressor(quantile=0.5, alpha=0, solver=solver)
254261

255262
y_pred_lr = linear_regression.fit(X, y_pareto).predict(X)
256263
y_pred_qr = quantile_regression.fit(X, y_pareto).predict(X)

‎sklearn/linear_model/_quantile.py

Lines changed: 42 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,16 @@ class QuantileRegressor(LinearModel, RegressorMixin, BaseEstimator):
4444
solver : {'highs-ds', 'highs-ipm', 'highs', 'interior-point', \
4545
'revised simplex'}, default='interior-point'
4646
Method used by :func:`scipy.optimize.linprog` to solve the linear
47-
programming formulation. Note that the highs methods are recommended
48-
for usage with `scipy>=1.6.0` because they are the fastest ones.
49-
Solvers "highs-ds", "highs-ipm" and "highs" support
50-
sparse input data and, in fact, always convert to sparse csc.
47+
programming formulation.
48+
49+
From `scipy>=1.6.0`, it is recommended to use the highs methods because
50+
they are the fastest ones. Solvers "highs-ds", "highs-ipm" and "highs"
51+
support sparse input data and, in fact, always convert to sparse csc.
52+
53+
From `scipy>=1.11.0`, "interior-point" is not available anymore.
54+
55+
.. versionchanged:: 1.4
56+
The default of `solver` will change to `"highs"` in version 1.4.
5157
5258
solver_options : dict, default=None
5359
Additional parameters passed to :func:`scipy.optimize.linprog` as
@@ -91,7 +97,10 @@ class QuantileRegressor(LinearModel, RegressorMixin, BaseEstimator):
9197
>>> rng = np.random.RandomState(0)
9298
>>> y = rng.randn(n_samples)
9399
>>> X = rng.randn(n_samples, n_features)
94-
>>> reg = QuantileRegressor(quantile=0.8).fit(X, y)
100+
>>> # the two following lines are optional in practice
101+
>>> from sklearn.utils.fixes import sp_version, parse_version
102+
>>> solver = "highs" if sp_version >= parse_version("1.6.0") else "interior-point"
103+
>>> reg = QuantileRegressor(quantile=0.8, solver=solver).fit(X, y)
95104
>>> np.mean(y <= reg.predict(X))
96105
0.8
97106
"""
@@ -102,7 +111,7 @@ def __init__(
102111
quantile=0.5,
103112
alpha=1.0,
104113
fit_intercept=True,
105-
solver="interior-point",
114+
solver="warn",
106115
solver_options=None,
107116
):
108117
self.quantile = quantile
@@ -166,14 +175,14 @@ def fit(self, X, y, sample_weight=None):
166175
f"The argument fit_intercept must be bool, got {self.fit_intercept}"
167176
)
168177

169-
if self.solver not in (
170-
"highs-ds",
171-
"highs-ipm",
172-
"highs",
173-
"interior-point",
174-
"revised simplex",
175-
):
176-
raise ValueError(f"Invalid value for argument solver, got {self.solver}")
178+
if self.solver == "warn":
179+
warnings.warn(
180+
"The default solver will change from 'interior-point' to 'highs' in "
181+
"version 1.4. Set `solver='highs'` or to the desired solver to silence "
182+
"this warning.",
183+
FutureWarning,
184+
)
185+
solver = "interior-point"
177186
elif self.solver in (
178187
"highs-ds",
179188
"highs-ipm",
@@ -183,8 +192,23 @@ def fit(self, X, y, sample_weight=None):
183192
f"Solver {self.solver} is only available "
184193
f"with scipy>=1.6.0, got {sp_version}"
185194
)
195+
else:
196+
solver = self.solver
197+
198+
if solver not in (
199+
"highs-ds",
200+
"highs-ipm",
201+
"highs",
202+
"interior-point",
203+
"revised simplex",
204+
):
205+
raise ValueError(f"Invalid value for argument solver, got {solver}")
206+
elif solver == "interior-point" and sp_version >= parse_version("1.11.0"):
207+
raise ValueError(
208+
f"Solver {solver} is not anymore available in SciPy >= 1.11.0."
209+
)
186210

187-
if sparse.issparse(X) and self.solver not in ["highs", "highs-ds", "highs-ipm"]:
211+
if sparse.issparse(X) and solver not in ["highs", "highs-ds", "highs-ipm"]:
188212
raise ValueError(
189213
f"Solver {self.solver} does not support sparse X. "
190214
"Use solver 'highs' for example."
@@ -200,7 +224,7 @@ def fit(self, X, y, sample_weight=None):
200224
)
201225

202226
# make default solver more stable
203-
if self.solver_options is None and self.solver == "interior-point":
227+
if self.solver_options is None and solver == "interior-point":
204228
solver_options = {"lstsq": True}
205229
else:
206230
solver_options = self.solver_options
@@ -243,7 +267,7 @@ def fit(self, X, y, sample_weight=None):
243267
c[0] = 0
244268
c[n_params] = 0
245269

246-
if self.solver in ["highs", "highs-ds", "highs-ipm"]:
270+
if solver in ["highs", "highs-ds", "highs-ipm"]:
247271
# Note that highs methods always use a sparse CSC memory layout internally,
248272
# even for optimization problems parametrized using dense numpy arrays.
249273
# Therefore, we work with CSC matrices as early as possible to limit
@@ -268,7 +292,7 @@ def fit(self, X, y, sample_weight=None):
268292
c=c,
269293
A_eq=A_eq,
270294
b_eq=b_eq,
271-
method=self.solver,
295+
method=solver,
272296
options=solver_options,
273297
)
274298
solution = result.x

‎sklearn/linear_model/tests/test_quantile.py

Lines changed: 53 additions & 14 deletions
< 10000 td data-grid-cell-id="diff-ec32c281bdc17dd6726cb2691a58ae0ac2ebba9e4be4e26df6cdc2e6737fe17b-109-122-2" data-line-anchor="diff-ec32c281bdc17dd6726cb2691a58ae0ac2ebba9e4be4e26df6cdc2e6737fe17bR122" data-selected="false" role="gridcell" style="background-color:var(--bgColor-default);padding-right:24px" tabindex="-1" valign="top" class="focusable-grid-cell diff-text-cell right-side-diff-cell left-side">
assert_allclose(huber.coef_, quant.coef_, atol=1e-1)
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@ def X_y_data():
2323
return X, y
2424

2525

26+
@pytest.fixture
27+
def default_solver():
28+
return "highs" if sp_version >= parse_version("1.6.0") else "interior-point"
29+
30+
2631
@pytest.mark.parametrize(
2732
"params, err_msg",
2833
[
@@ -40,6 +45,10 @@ def X_y_data():
4045
),
4146
],
4247
)
48+
@pytest.mark.filterwarnings(
49+
# FIXME (1.4): remove once we changed the default solver to "highs"
50+
"ignore:The default solver will change from 'interior-point'"
51+
)
4352
def test_init_parameters_validation(X_y_data, params, err_msg):
4453
"""Test that invalid init parameters raise errors."""
4554
X, y = X_y_data
@@ -85,11 +94,13 @@ def test_too_new_solver_methods_raise_error(X_y_data, solver):
8594
[0.5, 100, 2, 0],
8695
],
8796
)
88-
def test_quantile_toy_example(quantile, alpha, intercept, coef):
97+
def test_quantile_toy_example(quantile, alpha, intercept, coef, default_solver):
8998
# test how different parameters affect a small intuitive example
9099
X = [[0], [1], [1]]
91100
y = [1, 2, 11]
92-
model = QuantileRegressor(quantile=quantile, alpha=alpha).fit(X, y)
101+
model = QuantileRegressor(
102+
quantile=quantile, alpha=alpha, solver=default_solver
103+
).fit(X, y)
93104
assert_allclose(model.intercept_, intercept, atol=1e-2)
94105
if coef is not None:
95106
assert_allclose(model.coef_[0], coef, atol=1e-2)
@@ -99,13 +110,15 @@ def test_quantile_toy_example(quantile, alpha, intercept, coef):
99110

100111

101112
@pytest.mark.parametrize("fit_intercept", [True, False])
102-
def test_quantile_equals_huber_for_low_epsilon(fit_intercept):
113+
def test_quantile_equals_huber_for_low_epsilon(fit_intercept, default_solver):
103114
X, y = make_regression(n_samples=100, n_features=20, random_state=0, noise=1.0)
104115
alpha = 1e-4
105116
huber = HuberRegressor(
106117
epsilon=1 + 1e-4, alpha=alpha, fit_intercept=fit_intercept
107118
).fit(X, y)
108-
quant = QuantileRegressor(alpha=alpha, fit_intercept=fit_intercept).fit(X, y)
119+
quant = QuantileRegressor(
120+
alpha=alpha, fit_intercept=fit_intercept, solver=default_solver
121+
).fit(X, y)
109122
110123
if fit_intercept:
111124
assert huber.intercept_ == approx(quant.intercept_, abs=1e-1)
@@ -114,26 +127,26 @@ def test_quantile_equals_huber_for_low_epsilon(fit_intercept):
114127

115128

116129
@pytest.mark.parametrize("q", [0.5, 0.9, 0.05])
117-
def test_quantile_estimates_calibration(q):
130+
def test_quantile_estimates_calibration(q, default_solver):
118131
# Test that model estimates percentage of points below the prediction
119132
X, y = make_regression(n_samples=1000, n_features=20, random_state=0, noise=1.0)
120133
quant = QuantileRegressor(
121134
quantile=q,
122135
alpha=0,
123-
solver_options={"lstsq": False},
136+
solver=default_solver,
124137
).fit(X, y)
125138
assert np.mean(y < quant.predict(X)) == approx(q, abs=1e-2)
126139

127140

128-
def test_quantile_sample_weight():
141+
def test_quantile_sample_weight(default_solver):
129142
# test that with unequal sample weights we still estimate weighted fraction
130143
n = 1000
131144
X, y = make_regression(n_samples=n, n_features=5, random_state=0, noise=10.0)
132145
weight = np.ones(n)
133146
# when we increase weight of upper observations,
134147
# estimate of quantile should go up
135148
weight[y > y.mean()] = 100
136-
quant = QuantileRegressor(quantile=0.5, alpha=1e-8, solver_options={"lstsq": False})
149+
quant = QuantileRegressor(quantile=0.5, alpha=1e-8, solver=default_solver)
137150
quant.fit(X, y, sample_weight=weight)
138151
fraction_below = np.mean(y < quant.predict(X))
139152
assert fraction_below > 0.5
@@ -146,7 +159,7 @@ def test_quantile_sample_weight():
146159
reason="The `highs` solver is available from the 1.6.0 scipy version",
147160
)
148161
@pytest.mark.parametrize("quantile", [0.2, 0.5, 0.8])
149-
def test_asymmetric_error(quantile):
162+
def test_asymmetric_error(quantile, default_solver):
150163
"""Test quantile regression for asymmetric distributed targets."""
151164
n_samples = 1000
152165
rng = np.random.RandomState(42)
@@ -171,7 +184,7 @@ def test_asymmetric_error(quantile):
171184
model = QuantileRegressor(
172185
quantile=quantile,
173186
alpha=0,
174-
solver="highs",
187+
solver=default_solver,
175188
).fit(X, y)
176189
# This test can be made to pass with any solver but in the interest
177190
# of sparing continuous integration resources, the test is performed
@@ -206,7 +219,7 @@ def func(coef):
206219

207220

208221
@pytest.mark.parametrize("quantile", [0.2, 0.5, 0.8])
209-
def test_equivariance(quantile):
222+
def test_equivariance(quantile, default_solver):
210223
"""Test equivariace of quantile regression.
211224
212225
See Koenker (2005) Quantile Regression, Chapter 2.2.3.
@@ -223,7 +236,7 @@ def test_equivariance(quantile):
223236
)
224237
# make y asymmetric
225238
y += rng.exponential(scale=100, size=y.shape)
226-
params = dict(alpha=0, solver_options={"lstsq": True, "tol": 1e-10})
239+
params = dict(alpha=0, solver=default_solver)
227240
model1 = QuantileRegressor(quantile=quantile, **params).fit(X, y)
228241

229242
# coef(q; a*y, X) = a * coef(q; y, X)
@@ -252,6 +265,7 @@ def test_equivariance(quantile):
252265
assert_allclose(model2.coef_, np.linalg.solve(A, model1.coef_), rtol=1e-5)
253266

254267

268+
@pytest.mark.filterwarnings("ignore:`method='interior-point'` is deprecated")
255269
def test_linprog_failure():
256270
"""Test that linprog fails."""
257271
X = np.linspace(0, 10, num=10).reshape(-1, 1)
@@ -275,12 +289,14 @@ def test_linprog_failure():
275289
)
276290
@pytest.mark.parametrize("solver", ["highs", "highs-ds", "highs-ipm"])
277291
@pytest.mark.parametrize("fit_intercept", [True, False])
278-
def test_sparse_input(sparse_format, solver, fit_intercept):
292+
def test_sparse_input(sparse_format, solver, fit_intercept, default_solver):
279293
"""Test that sparse and dense X give same results."""
280294
X, y = make_regression(n_samples=100, n_features=20, random_state=1, noise=1.0)
281295
X_sparse = sparse_format(X)
282296
alpha = 1e-4
283-
quant_dense = QuantileRegressor(alpha=< 10000 span class=pl-s1>alpha, fit_intercept=fit_intercept).fit(X, y)
297+
quant_dense = QuantileRegressor(
298+
alpha=alpha, fit_intercept=fit_intercept, solver=default_solver
299+
).fit(X, y)
284300
quant_sparse = QuantileRegressor(
285301
alpha=alpha, fit_intercept=fit_intercept, solver=solver
286302
).fit(X_sparse, y)
@@ -289,3 +305,26 @@ def test_sparse_input(sparse_format, solver, fit_intercept):
289305
assert quant_sparse.intercept_ == approx(quant_dense.intercept_)
290306
# check that we still predict fraction
291307
assert 0.45 <= np.mean(y < quant_sparse.predict(X_sparse)) <= 0.55
308+
309+
310+
# TODO (1.4): remove this test in 1.4
311+
def test_warning_new_default(X_y_data):
312+
"""Check that we warn about the new default solver."""
313+
X, y = X_y_data
314+
model = QuantileRegressor()
315+
with pytest.warns(FutureWarning, match="The default solver will change"):
316+
model.fit(X, y)
317+
318+
319+
def test_error_interior_point_future(X_y_data, monkeypatch):
320+
"""Check that we will raise a proper error when requesting
321+
`solver='interior-point'` in SciPy >= 1.11.
322+
"""
323+
X, y = X_y_data
324+
import sklearn.linear_model._quantile
325+
326+
with monkeypatch.context() as m:
327+
m.setattr(sklearn.linear_model._quantile, "sp_version", parse_version("1.11.0"))
328+
err_msg = "Solver interior-point is not anymore available in SciPy >= 1.11.0."
329+
with pytest.raises(ValueError, match=err_msg):
330+
QuantileRegressor(solver="interior-point").fit(X, y)

‎sklearn/tests/test_docstring_parameters.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from sklearn.utils.estimator_checks import _enforce_estimator_tags_y
2121
from sklearn.utils.estimator_checks import _enforce_estimator_tags_x
2222
from sklearn.utils.estimator_checks import _construct_instance
23+
from sklearn.utils.fixes import sp_version, parse_version
2324
from sklearn.utils.deprecation import _is_deprecated
2425
from sklearn.datasets import make_classification
2526
from sklearn.linear_model import LogisticRegression
@@ -266,6 +267,11 @@ def test_fit_docstring_attributes(name, Estimator):
266267
if Estimator.__name__ in ("KMeans", "MiniBatchKMeans"):
267268
est.set_params(n_init="auto")
268269

270+
# TODO(1.4): TO BE REMOVED for 1.4 (avoid FutureWarning)
271+
if Estimator.__name__ == "QuantileRegressor":
272+
solver = "highs" if sp_version >= parse_version("1.6.0") else "interior-point"
273+
est.set_params(solver=solver)
274+
269275
# In case we want to deprecate some attributes in the future
270276
skipped_attributes = {}
271277

0 commit comments

Comments
 (0)
0