8000 [MRG+2] Timing and training score in GridSearchCV (#7325) · scikit-learn/scikit-learn@b444cc9 · GitHub
[go: up one dir, main page]
Skip to content

Commit b444cc9

Browse files
raghavrvjnothman
raghavrv
authored and
jnothman
committed
[MRG+2] Timing and training score in GridSearchCV (#7325)
* Resolved issue #6894 and #6895: Now *SearchCV.results_ includes both timing and training scores. wrote new test (sklearn/model_selection/test_search.py) and new doctest (sklearn/model_selection/_search.py) added a few more lines in the docstring of GridSearchCV and RandomizedSearchCV. Revised code according to suggestions. Add a few more lines to test_grid_search_results(): 1. check test_rank_score always >= 1 2. check all regular scores (test/train_mean/std_score) and timing >= 0 3. check all regular scores <= 1 Note that timing can be greater than 1 in general, and std of regular scores always <= 1 because the scores are bounded between 0 and 1. * ENH/FIX timing and training score. * ENH separate fit / score times * Make score_time=0 if errored; Ignore warnings in test * Cleanup docstrings * ENH Use helper to store the results * Move fit time computation to else of try...except...else * DOC readable sample scores * COSMIT Add a commnent on why time test is >= 0 instead of > 0 (Windows time.time precision is not accurate enought to be non-zero for trivial fits) * Convey that times are in seconds
1 parent 3a106fc commit b444cc9

File tree

4 files changed

+299
-128
lines changed

4 files changed

+299
-128
lines changed

‎doc/whats_new.rst

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,20 @@ Model Selection Enhancements and API Changes
116116
The parameter ``n_labels`` in the newly renamed
117117
:class:`model_selection.LeavePGroupsOut` is changed to ``n_groups``.
118118

119+
- Training scores and Timing information
120+
121+
``cv_results_`` also includes the training scores for each
122+
cross-validation split (with keys such as ``'split0_train_score'``), as
123+
well as their mean (``'mean_train_score'``) and standard deviation
124+
(``'std_train_score'``). To avoid the cost of evaluating training score,
125+
set ``return_train_score=False``.
126+
127+
Additionally the mean and standard deviation of the times taken to split,
128+
train and score the model across all the cross-validation splits is
129+
available at the key ``'mean_time'`` and ``'std_time'`` respectively.
130+
131+
Changelog
132+
---------
119133

120134
New features
121135
............
@@ -362,6 +376,12 @@ Enhancements
362376
now accept arbitrary kernel functions in addition to strings ``knn`` and ``rbf``.
363377
(`#5762 <https://github.com/scikit-learn/scikit-learn/pull/5762>`_) By `Utkarsh Upadhyay`_.
364378

379+
- The training scores and time taken for training followed by scoring for
380+
each search candidate are now available at the ``cv_results_`` dict.
381+
See :ref:`model_selection_changes` for more information.
382+
(`#7324 <https://github.com/scikit-learn/scikit-learn/pull/7325>`)
383+
By `Eugene Chen`_ and `Raghav RV`_.
384+
365385

366386
Bug fixes
367387
.........
@@ -4731,3 +4751,5 @@ David Huard, Dave Morrill, Ed Schofield, Travis Oliphant, Pearu Peterson.
47314751
.. _Russell Smith: https://github.com/rsmith54
47324752

47334753
.. _Utkarsh Upadhyay: https://github.com/musically-ut
4754+
4755+
.. _Eugene Chen: https://github.com/eyc88

‎sklearn/model_selection/_search.py

Lines changed: 106 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,9 @@ def fit_grid_point(X, y, estimator, parameters, train, test, scorer,
319319
"""
320320
score, n_samples_test, _ = _fit_and_score(estimator, X, y, scorer, train,
321321
test, verbose, parameters,
322-
fit_params, error_score)
322+
fit_params=fit_params,
323+
return_n_test_samples=True,
324+
error_score=error_score)
323325
return score, parameters, n_samples_test
324326

325327

@@ -374,7 +376,7 @@ class BaseSearchCV(six.with_metaclass(ABCMeta, BaseEstimator,
374376
def __init__(self, estimator, scoring=None,
375377
fit_params=None, n_jobs=1, iid=True,
376378
refit=True, cv=None, verbose=0, pre_dispatch='2*n_jobs',
377-
error_score='raise'):
379+
error_score='raise', return_train_score=True):
378380

379381
self.scoring = scoring
380382
self.estimator = estimator
@@ -386,6 +388,7 @@ def __init__(self, estimator, scoring=None,
386388
self.verbose = verbose
387389
self.pre_dispatch = pre_dispatch
388390
self.error_score = error_score
391+
self.return_train_score = return_train_score
389392

390393
@property
391394
def _estimator_type(self):
@@ -551,41 +554,61 @@ def _fit(self, X, y, groups, parameter_iterable):
551554
pre_dispatch=pre_dispatch
552555
)(delayed(_fit_and_score)(clone(base_estimator), X, y, self.scorer_,
553556
train, test, self.verbose, parameters,
554-
self.fit_params, return_parameters=True,
557+
fit_params=self.fit_params,
558+
return_train_score=self.return_train_score,
559+
return_n_test_samples=True,
560+
return_times=True, return_parameters=True,
555561
error_score=self.error_score)
556562
for parameters in parameter_iterable
557563
for train, test in cv.split(X, y, groups))
558564

559-
test_scores, test_sample_counts, _, parameters = zip(*out)
565+
# if one choose to see train score, "out" will contain train score info
566+
if self.return_train_score:
567+
(train_scores, test_scores, test_sample_counts,
568+
fit_time, score_time, parameters) = zip(*out)
569+
else:
570+
(test_scores, test_sample_counts,
571+
fit_time, score_time, parameters) = zip(*out)
560572

561573
candidate_params = parameters[::n_splits]
562574
n_candidates = len(candidate_params)
563575

564-
test_scores = np.array(test_scores,
565-
dtype=np.float64).reshape(n_candidates,
566-
n_splits)
576+
results = dict()
577+
578+
def _store(key_name, array, weights=None, splits=False, rank=False):
579+
"""A small helper to store the scores/times to the cv_results_"""
580+
array = np.array(array, dtype=np.float64).reshape(n_candidates,
581+
n_splits)
582+
if splits:
583+
for split_i in range(n_splits):
584+
results["split%d_%s"
585+
% (split_i, key_name)] = array[:, split_i]
586+
587+
array_means = np.average(array, axis=1, weights=weights)
588+
results['mean_%s' % key_name] = array_means
589+
# Weighted std is not directly available in numpy
590+
array_stds = np.sqrt(np.average((array -
591+
array_means[:, np.newaxis]) ** 2,
592+
axis=1, weights=weights))
593+
results['std_%s' % key_name] = array_stds
594+
595+
if rank:
596+
results["rank_%s" % key_name] = np.asarray(
597+
rankdata(-array_means, method='min'), dtype=np.int32)
598+
599+
# Computed the (weighted) mean and std for test scores alone
567600
# NOTE test_sample counts (weights) remain the same for all candidates
568601
test_sample_counts = np.array(test_sample_counts[:n_splits],
569602
dtype=np.int)
570603

571-
# Computed the (weighted) mean and std for all the candidates
572-
weights = test_sample_counts if self.iid else None
573-
means = np.average(test_scores, axis=1, weights=weights)
574-
stds = np.sqrt(np.average((test_scores - means[:, np.newaxis]) ** 2,
575-
axis=1, weights=weights))
576-
577-
cv_results = dict()
578-
for split_i in range(n_splits):
579-
cv_results["split%d_test_score" % split_i] = test_scores[:,
580-
split_i]
581-
cv_results["mean_test_score"] = means
582-
cv_results["std_test_score"] = stds
583-
584-
ranks = np.asarray(rankdata(-means, method='min'), dtype=np.int32)
604+
_store('test_score', test_scores, splits=True, rank=True,
605+
weights=test_sample_counts if self.iid else None)
606+
_store('train_score', train_scores, splits=True)
607+
_store('fit_time', fit_time)
608+
_store('score_time', score_time)
585609

586-
best_index = np.flatno 10000 nzero(ranks == 1)[0]
610+
best_index = np.flatnonzero(results["rank_test_score"] == 1)[0]
587611
best_parameters = candidate_params[best_index]
588-
cv_results["rank_test_score"] = ranks
589612

590613
# Use one np.MaskedArray and mask all the places where the param is not
591614
# applicable for that candidate. Use defaultdict as each candidate may
@@ -599,12 +622,12 @@ def _fit(self, X, y, groups, parameter_iterable):
599622
# Setting the value at an index also unmasks that index
600623
param_results["param_%s" % name][cand_i] = value
601624

602-
cv_results.update(param_results)
625+
results.update(param_results)
603626

604627
# Store a list of param dicts at the key 'params'
605-
cv_results['params'] = candidate_params
628+
results['params'] = candidate_params
606629

607-
self.cv_results_ = cv_results
630+
self.cv_results_ = results
608631
self.best_index_ = best_index
609632
self.n_splits_ = n_splits
610633

@@ -746,6 +769,10 @@ class GridSearchCV(BaseSearchCV):
746769
FitFailedWarning is raised. This parameter does not affect the refit
747770
step, which will always raise the error.
748771
772+
return_train_score : boolean, default=True
773+
If ``'False'``, the ``cv_results_`` attribute will not include training
774+
scores.
775+
749776
750777
Examples
751778
--------
@@ -764,13 +791,16 @@ class GridSearchCV(BaseSearchCV):
764791
random_state=None, shrinking=True, tol=...,
765792
verbose=False),
766793
fit_params={}, iid=..., n_jobs=1,
767-
param_grid=..., pre_dispatch=..., refit=...,
794+
param_grid=..., pre_dispatch=..., refit=..., return_train_score=...,
768795
scoring=..., verbose=...)
769796
>>> sorted(clf.cv_results_.keys())
770797
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
771-
['mean_test_score', 'param_C', 'param_kernel', 'params',...
772-
'rank_test_score', 'split0_test_score', 'split1_test_score',...
773-
'split2_test_score', 'std_test_score']
798+
['mean_fit_time', 'mean_score_time', 'mean_test_score',...
799+
'mean_train_score', 'param_C', 'param_kernel', 'params',...
800+
'rank_test_score', 'split0_test_score',...
801+
'split0_train_score', 'split1_test_score', 'split1_train_score',...
802+
'split2_test_score', 'split2_train_score',...
803+
'std_fit_time', 'std_score_time', 'std_test_score', 'std_train_score'...]
774804
775805
Attributes
776806
----------
@@ -801,17 +831,28 @@ class GridSearchCV(BaseSearchCV):
801831
mask = [ True True False False]...),
802832
'param_degree': masked_array(data = [2.0 3.0 -- --],
803833
mask = [False False True True]...),
804-
'split0_test_score' : [0.8, 0.7, 0.8, 0.9],
805-
'split1_test_score' : [0.82, 0.5, 0.7, 0.78],
806-
'mean_test_score' : [0.81, 0.60, 0.75, 0.82],
807-
'std_test_score' : [0.02, 0.01, 0.03, 0.03],
808-
'rank_test_score' : [2, 4, 3, 1],
809-
'params' : [{'kernel': 'poly', 'degree': 2}, ...],
834+
'split0_test_score' : [0.8, 0.7, 0.8, 0.9],
835+
'split1_test_score' : [0.82, 0.5, 0.7, 0.78],
836+
'mean_test_score' : [0.81, 0.60, 0.75, 0.82],
837+
'std_test_score' : [0.02, 0.01, 0.03, 0.03],
838+
'rank_test_score' : [2, 4, 3, 1],
839+
'split0_train_score' : [0.8, 0.9, 0.7],
840+
'split1_train_score' : [0.82, 0.5, 0.7],
841+
'mean_train_score' : [0.81, 0.7, 0.7],
842+
'std_train_score' : [0.03, 0.03, 0.04],
843+
'mean_fit_time' : [0.73, 0.63, 0.43, 0.49],
844+
'std_fit_time' : [0.01, 0.02, 0.01, 0.01],
845+
'mean_score_time' : [0.007, 0.06, 0.04, 0.04],
846+
'std_score_time' : [0.001, 0.002, 0.003, 0.005],
847+
'params' : [{'kernel': 'poly', 'degree': 2}, ...],
810848
}
811849
812850
NOTE that the key ``'params'`` is used to store a list of parameter
813851
settings dict for all the parameter candidates.
814852
853+
The ``mean_fit_time``, ``std_fit_time``, ``mean_score_time`` and
854+
``std_score_time`` are all in seconds.
855+
815856
best_estimator_ : estimator
816857
Estimator that was chosen by the search, i.e. estimator
817858
which gave highest score (or smallest loss if specified)
@@ -868,11 +909,13 @@ class GridSearchCV(BaseSearchCV):
868909

869910
def __init__(self, estimator, param_grid, scoring=None, fit_params=None,
870911
n_jobs=1, iid=True, refit=True, cv=None, verbose=0,
871-
pre_dispatch='2*n_jobs', error_score='raise'):
912+
pre_dispatch='2*n_jobs', error_score='raise',
913+
return_train_score=True):
872914
super(GridSearchCV, self).__init__(
873915
estimator=estimator, scoring=scoring, fit_params=fit_params,
874916
n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose,
875-
pre_dispatch=pre_dispatch, error_score=error_score)
917+
pre_dispatch=pre_dispatch, error_score=error_score,
918+
return_train_score=return_train_score)
876919
self.param_grid = param_grid
877920
_check_param_grid(param_grid)
878921

@@ -1006,6 +1049,10 @@ class RandomizedSearchCV(BaseSearchCV):
10061049
FitFailedWarning is raised. This parameter does not affect the refit
10071050
step, which will always raise the error.
10081051
1052+
return_train_score : boolean, default=True
1053+
If ``'False'``, the ``cv_results_`` attribute will not include training
1054+
scores.
1055+
10091056
Attributes
10101057
----------
10111058
cv_results_ : dict of numpy (masked) ndarrays
@@ -1030,17 +1077,28 @@ class RandomizedSearchCV(BaseSearchCV):
10301077
'param_kernel' : masked_array(data = ['rbf', rbf', 'rbf'],
10311078
mask = False),
10321079
'param_gamma' : masked_array(data = [0.1 0.2 0.3], mask = False),
1033-
'split0_test_score' : [0.8, 0.9, 0.7],
1034-
'split1_test_score' : [0.82, 0.5, 0.7],
1035-
'mean_test_score' : [0.81, 0.7, 0.7],
1036-
'std_test_score' : [0.02, 0.2, 0.],
1037-
'rank_test_score' : [3, 1, 1],
1080+
'split0_test_score' : [0.8, 0.9, 0.7],
1081+
'split1_test_score' : [0.82, 0.5, 0.7],
1082+
'mean_test_score' : [0.81, 0.7, 0.7],
1083+
'std_test_score' : [0.02, 0.2, 0.],
1084+
'rank_test_score' : [3, 1, 1],
1085+
'split0_train_score' : [0.8, 0.9, 0.7],
1086+
'split1_train_score' : [0.82, 0.5, 0.7],
1087+
'mean_train_score' : [0.81, 0.7, 0.7],
1088+
'std_train_score' : [0.03, 0.03, 0.04],
1089+
'mean_fit_time' : [0.73, 0.63, 0.43, 0.49],
1090+
'std_fit_time' : [0.01, 0.02, 0.01, 0.01],
1091+
'mean_score_time' : [0.007, 0.06, 0.04, 0.04],
1092+
'std_score_time' : [0.001, 0.002, 0.003, 0.005],
10381093
'params' : [{'kernel' : 'rbf', 'gamma' : 0.1}, ...],
10391094
}
10401095
10411096
NOTE that the key ``'params'`` is used to store a list of parameter
10421097
settings dict for all the parameter candidates.
10431098
1099+
The ``mean_fit_time``, ``std_fit_time``, ``mean_score_time`` and
1100+
``std_score_time`` are all in seconds.
1101+
10441102
best_estimator_ : estimator
10451103
Estimator that was chosen by the search, i.e. estimator
10461104
which gave highest score (or smallest loss if specified)
@@ -1094,15 +1152,15 @@ class RandomizedSearchCV(BaseSearchCV):
10941152
def __init__(self, estimator, param_distributions, n_iter=10, scoring=None,
10951153
fit_params=None, n_jobs=1, iid=True, refit=True, cv=None,
10961154
verbose=0, pre_dispatch='2*n_jobs', random_state=None,
1097-
error_score='raise'):
1098-
1155+
error_score='raise', return_train_score=True):
10991156
self.param_distributions = param_distributions
11001157
self.n_iter = n_iter
11011158
self.random_state = random_state
11021159
super(RandomizedSearchCV, self).__init__(
1103-
estimator=estimator, scoring=scoring, fit_params=fit_params,
1104-
n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose,
1105-
pre_dispatch=pre_dispatch, error_score=error_score)
1160+
estimator=estimator, scoring=scoring, fit_params=fit_params,
1161+
n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose,
1162+
pre_dispatch=pre_dispatch, error_score=error_score,
1163+
return_train_score=return_train_score)
11061164

11071165
def fit(self, X, y=None, groups=None):
11081166
"""Run fit on the estimator with randomly drawn parameters.

0 commit comments

Comments
 (0)
0