8000 Added fit and score times for learning_curve (#13938) · scikit-learn/scikit-learn@b28aadf · GitHub
[go: up one dir, main page]
Skip to content

Commit b28aadf

Browse files
H4dr1enNicolasHug
H4dr1en
authored and
NicolasHug
committed
Added fit and score times for learning_curve (#13938)
1 parent 36b688e commit b28aadf

File tree

4 files changed

+127
-47
lines changed

4 files changed

+127
-47
lines changed

‎doc/whats_new/v0.22.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,14 @@ Changelog
6363
of the maximization procedure in :term:`fit`.
6464
:pr:`13618` by :user:`Yoshihiro Uchida <c56pony>`.
6565

66+
:mod:`sklearn.model_selection`
67+
..................
68+
69+
- |Enhancement| :class:`model_selection.learning_curve` now accepts parameter
70+
``return_times`` which can be used to retrieve computation times in order to
71+
plot model scalability (see learning_curve example).
72+
:pr:`13938` by :user:`Hadrien Reboul <H4dr1en>`.
73+
6674
:mod:`sklearn.pipeline`
6775
.......................
6876

‎examples/model_selection/plot_learning_curve.py

Lines changed: 71 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,18 @@
22
========================
33
Plotting Learning Curves
44
========================
5-
6-
On the left side the learning curve of a naive Bayes classifier is shown for
7-
the digits dataset. Note that the training score and the cross-validation score
8-
are both not very good at the end. However, the shape of the curve can be found
9-
in more complex datasets very often: the training score is very high at the
10-
beginning and decreases and the cross-validation score is very low at the
11-
beginning and increases. On the right side we see the learning curve of an SVM
12-
with RBF kernel. We can see clearly that the training score is still around
13-
the maximum and the validation score could be increased with more training
14-
samples.
5+
In the first column, first row the learning curve of a naive Bayes classifier
6+
is shown for the digits dataset. Note that the training score and the
7+
cross-validation score are both not very good at the end. However, the shape
8+
of the curve can be found in more complex datasets very often: the training
9+
score is very high at the beginning and decreases and the cross-validation
10+
score is very low at the beginning and increases. In the second column, first
11+
row we see the learning curve of an SVM with RBF kernel. We can see clearly
12+
that the training score is still around the maximum and the validation score
13+
could be increased with more training samples. The plots in the second row
14+
show the times required by the models to train with various sizes of training
15+
dataset. The plots in the third row show how much time was required to train
16+
the models for each training sizes.
1517
"""
1618
print(__doc__)
1719

@@ -24,10 +26,11 @@
2426
from sklearn.model_selection import ShuffleSplit
2527

2628

27-
def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
29+
def plot_learning_curve(estimator, title, X, y, axes=None, ylim=None, cv=None,
2830
n_jobs=None, train_sizes=np.linspace(.1, 1.0, 5)):
2931
"""
30-
Generate a simple plot of the test and training learning curve.
32+
Generate 3 plots: the test and training learning curve, the training
33+
samples vs fit times curve, the fit times vs score curve.
3134
3235
Parameters
3336
----------
@@ -45,6 +48,9 @@ def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
4548
Target relative to X for classification or regression;
4649
None for unsupervised learning.
4750
51+
axes : array of 3 axes, optional (default=None)
52+
Axes to use for plotting the curves.
53+
4854
ylim : tuple, shape (ymin, ymax), optional
4955
Defines minimum and maximum yvalues plotted.
5056
@@ -79,34 +85,63 @@ def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
7985
be big enough to contain at least one sample from each class.
8086
(default: np.linspace(0.1, 1.0, 5))
8187
"""
82-
plt.figure()
83-
plt.title(title)
88+
if axes is None:
89+
_, axes = plt.subplots(1, 3, figsize=(20, 5))
90+
91+
axes[0].set_title(title)
8492
if ylim is not None:
85-
plt.ylim(*ylim)
86-
plt.xlabel("Training examples")
87-
plt.ylabel("Score")
88-
train_sizes, train_scores, test_scores = learning_curve(
89-
estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
93+
axes[0].set_ylim(*ylim)
94+
axes[0].set_xlabel("Training examples")
95+
axes[0].set_ylabel("Score")
96+
97+
train_sizes, train_scores, test_scores, fit_times, _ = \
98+
learning_curve(estimator, X, y, cv=cv, n_jobs=n_jobs,
99+
train_sizes=train_sizes,
100+
return_times=True)
90101
train_scores_mean = np.mean(train_scores, axis=1)
91102
train_scores_std = np.std(train_scores, axis=1)
92103
test_scores_mean = np.mean(test_scores, axis=1)
93104
test_scores_std = np.std(test_scores, axis=1)
94-
plt.grid()
95-
96-
plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
97-
train_scores_mean + train_scores_std, alpha=0.1,
98-
color="r")
99-
plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
100-
test_scores_mean + test_scores_std, alpha=0.1, color="g")
101-
plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
102-
label="Training score")
103-
plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
104-
label="Cross-validation score")
105-
106-
plt.legend(loc="best")
105+
fit_times_mean = np.mean(fit_times, axis=1)
106+
fit_times_std = np.std(fit_times, axis=1)
107+
108+
# Plot learning curve
109+
axes[0].grid()
110+
axes[0].fill_between(train_sizes, train_scores_mean - train_scores_std,
111+
train_scores_mean + train_scores_std, alpha=0.1,
112+
color="r")
113+
axes[0].fill_between(train_sizes, test_scores_mean - test_scores_std,
114+
test_scores_mean + test_scores_std, alpha=0.1,
115+
color="g")
116+
axes[0].plot(train_sizes, train_scores_mean, 'o-', color="r",
117+
label="Training score")
118+
axes[0].plot(train_sizes, test_scores_mean, 'o-', color="g",
119+
label="Cross-validation score")
120+
axes[0].legend(loc="best")
121+
122+
# Plot n_samples vs fit_times
123+
axes[1].grid()
124+
axes[1].plot(train_sizes, fit_times_mean, 'o-')
125+
axes[1].fill_between(train_sizes, fit_times_mean - fit_times_std,
126+
fit_times_mean + fit_times_std, alpha=0.1)
127+
axes[1].set_xlabel("Training examples")
128+
axes[1].set_ylabel("fit_times")
129+
axes[1].set_title("Scalability of the model")
130+
131+
# Plot fit_time vs score
132+
axes[2].grid()
133+
axes[2].plot(fit_times_mean, test_scores_mean, 'o-')
134+
axes[2].fill_between(fit_times_mean, test_scores_mean - test_scores_std,
135+
test_scores_mean + test_scores_std, alpha=0.1)
136+
axes[2].set_xlabel("fit_times")
137+
axes[2].set_ylabel("Score")
138+
axes[2].set_title("Performance of the model")
139+
107140
return plt
108141

109142

143+
fig, axes = plt.subplots(3, 2, figsize=(10, 15))
144+
110145
digits = load_digits()
111146
X, y = digits.data, digits.target
112147

@@ -117,12 +152,14 @@ def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
117152
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=0)
118153

119154
estimator = GaussianNB()
120-
plot_learning_curve(estimator, title, X, y, ylim=(0.7, 1.01), cv=cv, n_jobs=4)
155+
plot_learning_curve(estimator, title, X, y, axes=axes[:, 0], ylim=(0.7, 1.01),
156+
cv=cv, n_jobs=4)
121157

122158
title = r"Learning Curves (SVM, RBF kernel, $\gamma=0.001$)"
123159
# SVC is more expensive so we do a lower number of CV iterations:
124160
cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0)
125161
estimator = SVC(gamma=0.001)
126-
plot_learning_curve(estimator, title, X, y, (0.7, 1.01), cv=cv, n_jobs=4)
162+
plot_learning_curve(estimator, title, X, y, axes=axes[:, 1], ylim=(0.7, 1.01),
163+
cv=cv, n_jobs=4)
127164

128165
plt.show()

‎sklearn/model_selection/_validation.py

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1094,7 +1094,7 @@ def learning_curve(estimator, X, y, groups=None,
10941094
train_sizes=np.linspace(0.1, 1.0, 5), cv=None,
10951095
scoring=None, exploit_incremental_learning=False,
10961096
n_jobs=None, pre_dispatch="all", verbose=0, shuffle=False,
1097-
random_state=None, error_score=np.nan):
1097+
random_state=None, error_score=np.nan, return_times=False):
10981098
"""Learning curve.
10991099
11001100
Determines cross-validated training and test scores for different training
@@ -1193,6 +1193,9 @@ def learning_curve(estimator, X, y, groups=None,
11931193
If a numeric value is given, FitFailedWarning is raised. This parameter
11941194
does not affect the refit step, which will always raise the error.
11951195
1196+
return_times : boolean, optional (default: False)
1197+
Whether to return the fit and score times.
1198+
11961199
Returns
11971200
-------
11981201
train_sizes_abs : array, shape (n_unique_ticks,), dtype int
@@ -1206,6 +1209,12 @@ def learning_curve(estimator, X, y, groups=None,
12061209
test_scores : array, shape (n_ticks, n_cv_folds)
12071210
Scores on test set.
12081211
1212+
fit_times : array, shape (n_ticks, n_cv_folds)
1213+
Times spent for fitting in seconds.
1214+
1215+
score_times : array, shape (n_ticks, n_cv_folds)
1216+
Times spent for scoring in seconds.
1217+
12091218
Notes
12101219
-----
12111220
See :ref:`examples/model_selection/plot_learning_curve.py
@@ -1243,7 +1252,7 @@ def learning_curve(estimator, X, y, groups=None,
12431252
classes = np.unique(y) if is_classifier(estimator) else None
12441253
out = parallel(delayed(_incremental_fit_estimator)(
12451254
clone(estimator), X, y, classes, train, test, train_sizes_abs,
1246-
scorer, verbose) for train, test in cv_iter)
1255+
scorer, verbose, return_times) for train, test in cv_iter)
12471256
else:
12481257
train_test_proportions = []
12491258
for train, test in cv_iter:
@@ -1253,15 +1262,21 @@ def learning_curve(estimator, X, y, groups=None,
12531262
out = parallel(delayed(_fit_and_score)(
12541263
clone(estimator), X, y, scorer, train, test, verbose,
12551264
parameters=None, fit_params=None, return_train_score=True,
1256-
error_score=error_score)
1265+
error_score=error_score, return_times=return_times)
12571266
for train, test in train_test_proportions)
12581267
out = np.array(out)
12591268
n_cv_folds = out.shape[0] // n_unique_ticks
1260-
out = out.reshape(n_cv_folds, n_unique_ticks, 2)
1269+
dim = 4 if return_times else 2
1270+
out = out.reshape(n_cv_folds, n_unique_ticks, dim)
12611271

12621272
out = np.asarray(out).transpose((2, 1, 0))
12631273

1264-
return train_sizes_abs, out[0], out[1]
1274+
ret = train_sizes_abs, out[0], out[1]
1275+
1276+
if return_times:
1277+
ret = ret + (out[2], out[3])
1278+
1279+
return ret
12651280

12661281

12671282
def _translate_train_sizes(train_sizes, n_max_training_samples):
@@ -1324,24 +1339,37 @@ def _translate_train_sizes(train_sizes, n_max_training_samples):
13241339

13251340

13261341
def _incremental_fit_estimator(estimator, X, y, classes, train, test,
1327-
train_sizes, scorer, verbose):
1342+
train_sizes, scorer, verbose, return_times):
13281343
"""Train estimator on training subsets incrementally and compute scores."""
1329-
train_scores, test_scores = [], []
1344+
train_scores, test_scores, fit_times, score_times = [], [], [], []
13301345
partitions = zip(train_sizes, np.split(train, train_sizes)[:-1])
13311346
for n_train_samples, partial_train in partitions:
13321347
train_subset = train[:n_train_samples]
13331348
X_train, y_train = _safe_split(estimator, X, y, train_subset)
13341349
X_partial_train, y_partial_train = _safe_split(estimator, X, y,
13351350
partial_train)
13361351
X_test, y_test = _safe_split(estimator, X, y, test, train_subset)
1352+
start_fit = time.time()
13371353
if y_partial_train is None:
13381354
estimator.partial_fit(X_partial_train, classes=classes)
13391355
else:
13401356
estimator.partial_fit(X_partial_train, y_partial_train,
13411357
classes=classes)
1342-
train_scores.append(_score(estimator, X_train, y_train, scorer))
1358+
fit_time = time.time() - start_fit
1359+
fit_times.append(fit_time)
1360+
1361+
start_score = time.time()
1362+
13431363
test_scores.append(_score(estimator, X_test, y_test, scorer))
1344-
return np.array((train_scores, test_scores)).T
1364+
train_scores.append(_score(estimator, X_train, y_train, scorer))
1365+
1366+
score_time = time.time() - start_score
1367+
score_times.append(score_time)
1368+
1369+
ret = ((train_scores, test_scores, fit_times, score_times)
1370+
if return_times else (train_scores, test_scores))
1371+
1372+
return np.array(ret).T
13451373

13461374

13471375
def validation_curve(estimator, X, y, param_name, param_range, groups=None,

‎sklearn/model_selection/tests/test_validation.py

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -991,20 +991,27 @@ def test_learning_curve():
991991
estimator = MockImprovingEstimator(n_samples * ((n_splits - 1) / n_splits))
992992
for shuffle_train in [False, True]:
993993
with warnings.catch_warnings(record=True) as w:
994-
train_sizes, train_scores, test_scores = learning_curve(
995-
estimator, X, y, cv=KFold(n_splits=n_splits),
996-
train_sizes=np.linspace(0.1, 1.0, 10),
997-
shuffle=shuffle_train)
994+
train_sizes, train_scores, test_scores, fit_times, score_times = \
995+
learning_curve(estimator, X, y, cv=KFold(n_splits=n_splits),
996+
train_sizes=np.linspace(0.1, 1.0, 10),
997+
shuffle=shuffle_train, return_times=True)
998998
if len(w) > 0:
999999
raise RuntimeError("Unexpected warning: %r" % w[0].message)
10001000
assert_equal(train_scores.shape, (10, 3))
10011001
assert_equal(test_scores.shape, (10, 3))
1002+
assert_equal(fit_times.shape, (10, 3))
1003+
assert_equal(score_times.shape, (10, 3))
10021004
assert_array_equal(train_sizes, np.linspace(2, 20, 10))
10031005
assert_array_almost_equal(train_scores.mean(axis=1),
10041006
np.linspace(1.9, 1.0, 10))
10051007
assert_array_almost_equal(test_scores.mean(axis=1),
10061008
np.linspace(0.1, 1.0, 10))
10071009

1010+
# Cannot use assert_array_almost_equal for fit and score times because
1011+
# the values are hardware-dependant
1012+
assert_equal(fit_times.dtype, "float64")
1013+
assert_equal(score_times.dtype, "float64")
1014+
10081015
# Test a custom cv splitter that can iterate only once
10091016
with warnings.catch_warnings(record=True) as w:
10101017
train_sizes2, train_scores2, test_scores2 = learning_curve(

0 commit comments

Comments
 (0)
0