8000 Resolved issue #6894 and #6895: · raghavrv/scikit-learn@479d23c · GitHub
[go: up one dir, main page]
Skip to content

Commit 479d23c

Browse files
Eugene Chenraghavrv
Eugene Chen
authored and
raghavrv
committed
Now *SearchCV.results_ includes both timing and training scores. wrote new test (sklearn/model_selection/test_search.py) and new doctest (sklearn/model_selection/_search.py) added a few more lines in the docstring of GridSearchCV and RandomizedSearchCV. Revised code according to suggestions. Add a few more lines to test_grid_search_results(): 1. check test_rank_score always >= 1 2. check all regular scores (test/train_mean/std_score) and timing >= 0 3. check all regular scores <= 1 Note that timing can be greater than 1 in general, and std of regular scores always <= 1 because the scores are bounded between 0 and 1.
1 parent 9a12555 commit 479d23c

File tree

2 files changed

+118
-37
lines changed

2 files changed

+118
-37
lines changed

‎sklearn/model_selection/_search.py

Lines changed: 94 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ class BaseSearchCV(six.with_metaclass(ABCMeta, BaseEstimator,
374374
def __init__(self, estimator, scoring=None,
375375
fit_params=None, n_jobs=1, iid=True,
376376
refit=True, cv=None, verbose=0, pre_dispatch='2*n_jobs',
377-
error_score='raise'):
377+
error_score='raise', return_train_score=True):
378378

379379
self.scoring = scoring
380380
self.estimator = estimator
@@ -386,6 +386,7 @@ def __init__(self, estimator, scoring=None,
386386
self.verbose = verbose
387387
self.pre_dispatch = pre_dispatch
388388
self.error_score = error_score
389+
self.return_train_score = return_train_score
389390

390391
@property
391392
def _estimator_type(self):
@@ -551,28 +552,52 @@ def _fit(self, X, y, groups, parameter_iterable):
551552
pre_dispatch=pre_dispatch
552553
)(delayed(_fit_and_score)(clone(base_estimator), X, y, self.scorer_,
553554
train, test, self.verbose, parameters,
554-
self.fit_params, return_parameters=True,
555+
self.fit_params,
556+
return_train_score=self.return_train_score,
557+
return_parameters=True,
555558
D7AF error_score=self.error_score)
556559
for parameters in parameter_iterable
557560
for train, test in cv.split(X, y, groups))
558561

559-
test_scores, test_sample_counts, _, parameters = zip(*out)
562+
# if one choose to see train score, "out" will contain train score info
563+
if self.return_train_score:
564+
train_scores, test_scores, test_sample_counts, time, parameters =\
565+
zip(*out)
566+
else:
567+
test_scores, test_sample_counts, time, parameters = zip(*out)
560568

561569
candidate_params = parameters[::n_splits]
562570
n_candidates = len(candidate_params)
563571

572+
# if one choose to return train score, reshape the train_scores array
573+
if self.return_train_score:
574+
train_scores = np.array(train_scores,
575+
dtype=np.float64).reshape(n_candidates,
576+
n_splits)
564577
test_scores = np.array(test_scores,
565578
dtype=np.float64).reshape(n_candidates,
566579
n_splits)
567580
# NOTE test_sample counts (weights) remain the same for all candidates
568581
test_sample_counts = np.array(test_sample_counts[:n_splits],
569582
dtype=np.int)
570583

571-
# Computed the (weighted) mean and std for all the candidates
584+
# Computed the (weighted) mean and std for test scores
572585
weights = test_sample_counts if self.iid else None
573-
means = np.average(test_scores, axis=1, weights=weights)
574-
stds = np.sqrt(np.average((test_scores - means[:, np.newaxis]) ** 2,
575-
axis=1, weights=weights))
586+
test_means = np.average(test_scores, axis=1, weights=weights)
587+
test_stds = np.sqrt(
588+
np.average((test_scores - test_means[:, np.newaxis]) ** 2, axis=1,
589+
weights=weights))
590+
591+
time = np.array(time, dtype=np.float64).reshape(n_candidates, n_splits)
592+
time_means = np.average(time, axis=1)
593+
time_stds = np.sqrt(
594+
np.average((time - time_means[:, np.newaxis]) ** 2,
595+
axis=1))
596+
if self.return_train_score:
597+
train_means = np.average(train_scores, axis=1)
598+
train_stds = np.sqrt(
599+
np.average((train_scores - train_means[:, np.newaxis]) ** 2,
600+
axis=1))
576601

577602
cv_results = dict()
578603
for split_i in range(n_splits):
@@ -581,7 +606,19 @@ def _fit(self, X, y, groups, parameter_iterable):
581606
cv_results["mean_test_score"] = means
582607
cv_results["std_test_score"] = stds
583608

584-
ranks = np.asarray(rankdata(-means, method='min'), dtype=np.int32)
609+
if self.return_train_score:
610+
for split_i in range(n_splits):
611+
results["train_split%d_score" % split_i] = (
612+
train_scores[:, split_i])
613+
results["mean_train_score"] = train_means
614+
results["std_train_scores"] = train_stds
615+
results["rank_train_scores"] = np.asarray(rankdata(-train_means,
616+
method='min'),
617+
dtype=np.int32)
618+
619+
results["mean_test_time"] = time_means
620+
results["std_test_time"] = time_stds
621+
ranks = np.asarray(rankdata(-test_means, method='min'), dtype=np.int32)
585622

586623
best_index = np.flatnonzero(ranks == 1)[0]
587624
best_parameters = candidate_params[best_index]
@@ -746,6 +783,10 @@ class GridSearchCV(BaseSearchCV):
746783
FitFailedWarning is raised. This parameter does not affect the refit
747784
step, which will always raise the error.
748785
786+
return_train_score: boolean, default=True
787+
If ``'False'``, the results_ attribute will not include training
788+
scores.
789+
749790
750791
Examples
751792
--------
@@ -764,13 +805,14 @@ class GridSearchCV(BaseSearchCV):
764805
random_state=None, shrinking=True, tol=...,
765806
verbose=False),
766807
fit_params={}, iid=..., n_jobs=1,
767-
param_grid=..., pre_dispatch=..., refit=...,
808+
param_grid=..., pre_dispatch=..., refit=..., return_train_score=...,
768809
scoring=..., verbose=...)
769810
>>> sorted(clf.cv_results_.keys())
770811
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
771-
['mean_test_score', 'param_C', 'param_kernel', 'params',...
772-
'rank_test_score', 'split0_test_score', 'split1_test_score',...
773-
'split2_test_score', 'std_test_score']
812+
['mean_test_score', 'mean_test_time', 'mean_train_score',...
813+
'param_C', 'param_kernel', 'params', 'rank_test_score',...
814+
'split0_test_score', 'split1_test_score',...
815+
'split2_test_score', 'std_test_score', 'std_test_time'...]
774816
775817
Attributes
776818
----------
@@ -806,11 +848,20 @@ class GridSearchCV(BaseSearchCV):
806848
'mean_test_score' : [0.81, 0.60, 0.75, 0.82],
807849
'std_test_score' : [0.02, 0.01, 0.03, 0.03],
808850
'rank_test_score' : [2, 4, 3, 1],
851+
'split0_train_score': [0.9, 0.8, 0.85, 1.]
852+
'split1_train_score': [0.95, 0.7, 0.8, 0.8]
853+
'mean_train_score' : [0.93, 0.75, 0.83, 0.9]
854+
'std_train_score' : [0.02, 0.01, 0.03, 0.03],
855+
'rank_train_score' : [2, 4, 3, 1],
856+
'mean_test_time' : [0.00073, 0.00063, 0.00043, 0.00049]
857+
'std_test_time' : [1.62e-4, 3.37e-5, 1.42e-5, 1.1e-5]
809858
'params' : [{'kernel': 'poly', 'degree': 2}, ...],
810859
}
811860
812861
NOTE that the key ``'params'`` is used to store a list of parameter
813-
settings dict for all the parameter candidates.
862+
settings dict for all the parameter candidates. Besides,
863+
``'train_mean_score'``, ``'train_split*_score'``, ... will be present
864+
when ``return_train_score=True``.
814865
815866
best_estimator_ : estimator
816867
Estimator that was chosen by the search, i.e. estimator
@@ -868,11 +919,13 @@ class GridSearchCV(BaseSearchCV):
868919

869920
def __init__(self, estimator, param_grid, scoring=None, fit_params=None,
870921
n_jobs=1, iid=True, refit=True, cv=None, verbose=0,
871-
pre_dispatch='2*n_jobs', error_score='raise'):
922+
pre_dispatch='2*n_jobs', error_score='raise',
923+
return_train_score=False):
872924
super(GridSearchCV, self).__init__(
873925
estimator=estimator, scoring=scoring, fit_params=fit_params,
874926
n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose,
875-
pre_dispatch=pre_dispatch, error_score=error_score)
927+
pre_dispatch=pre_dispatch, error_score=error_score,
928+
return_train_score=return_train_score)
876929
self.param_grid = param_grid
877930
_check_param_grid(param_grid)
878931

@@ -1006,6 +1059,10 @@ class RandomizedSearchCV(BaseSearchCV):
10061059
FitFailedWarning is raised. This parameter does not affect the refit
10071060
step, which will always raise the error.
10081061
1062+
return_train_score: boolean, default=True
1063+
If ``'False'``, the results_ attribute will not include training
1064+
scores.
1065+
10091066
Attributes
10101067
----------
10111068
cv_results_ : dict of numpy (masked) ndarrays
@@ -1030,16 +1087,27 @@ class RandomizedSearchCV(BaseSearchCV):
10301087
'param_kernel' : masked_array(data = ['rbf', rbf', 'rbf'],
10311088
mask = False),
10321089
'param_gamma' : masked_array(data = [0.1 0.2 0.3], mask = False),
1033-
'split0_test_score' : [0.8, 0.9, 0.7],
1034-
'split1_test_score' : [0.82, 0.5, 0.7],
1035-
'mean_test_score' : [0.81, 0.7, 0.7],
1036-
'std_test_score' : [0.02, 0.2, 0.],
1037-
'rank_test_score' : [3, 1, 1],
1090+
'split0_test_score' : [0.8, 0.9, 0.7],
1091+
'split1_test_score' : [0.82, 0.5, 0.7],
1092+
'mean_test_score' : [0.81, 0.7, 0.7],
1093+
'std_test_score' : [0.02, 0.2, 0.],
1094+
'rank_test_score' : [3, 1, 1],
1095+
'split0_train_score' : [0.8, 0.9, 0.7],
1096+
'split1_train_score' : [0.82, 0.5, 0.7],
1097+
'mean_train_score' : [0.81, 0.7, 0.7],
1098+
'std_train_score' : [0.00073, 0.00063, 0.00043]
1099+
'rank_train_score' : [1.62e-4, 3.37e-5, 1.1e-5]
1100+
'test_mean_time' : [0.00073, 0.00063, 0.00043]
1101+
'test_std_time' : [1.62e-4, 3.37e-5, 1.1e-5]
1102+
'test_std_score' : [0.02, 0.2, 0.],
1103+
'test_rank_score' : [3, 1, 1],
10381104
'params' : [{'kernel' : 'rbf', 'gamma' : 0.1}, ...],
10391105
}
10401106
10411107
NOTE that the key ``'params'`` is used to store a list of parameter
1042-
settings dict for all the parameter candidates.
1108+
settings dict for all the parameter candidates. Besides,
1109+
'train_mean_score', 'train_split*_score', ... will be present when
1110+
return_train_score is set to True.
10431111
10441112
best_estimator_ : estimator
10451113
Estimator that was chosen by the search, i.e. estimator
@@ -1094,15 +1162,15 @@ class RandomizedSearchCV(BaseSearchCV):
10941162
def __init__(self, estimator, param_distributions, n_iter=10, scoring=None,
10951163
fit_params=None, n_jobs=1, iid=True, refit=True, cv=None,
10961164
verbose=0, pre_dispatch='2*n_jobs', random_state=None,
1097-
error_score='raise'):
1098-
1165+
error_score='raise', return_train_score=False):
10991166
self.param_distributions = param_distributions
11001167
self.n_iter = n_iter
11011168
self.random_state = random_state
11021169
super(RandomizedSearchCV, self).__init__(
1103-
estimator=estimator, scoring=scoring, fit_params=fit_params,
1104-
n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose,
1105-
pre_dispatch=pre_dispatch, error_score=error_score)
1170+
estimator=estimator, scoring=scoring, fit_params=fit_params,
1171+
n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose,
1172+
pre_dispatch=pre_dispatch, error_score=error_score,
1173+
return_train_score=return_train_score)
11061174

11071175
def fit(self, X, y=None, groups=None):
11081176
"""Run fit on the estimator with randomly drawn parameters.

‎sklearn/model_selection/tests/test_search.py

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -642,21 +642,30 @@ def test_grid_search_results():
642642
params = [dict(kernel=['rbf', ], C=[1, 10], gamma=[0.1, 1]),
643643
dict(kernel=['poly', ], degree=[1, 2])]
644644
grid_search = GridSearchCV(SVC(), cv=n_splits, iid=False,
645-
param_grid=params)
645+
param_grid=params, return_train_score=True)
646646
grid_search.fit(X, y)
647647
grid_search_iid = GridSearchCV(SVC(), cv=n_splits, iid=True,
648-
param_grid=params)
648+
param_grid=params, return_train_score=True)
649649
grid_search_iid.fit(X, y)
650650

651651
param_keys = ('param_C', 'param_degree', 'param_gamma', 'param_kernel')
652-
score_keys = ('mean_test_score', 'rank_test_score',
653-
'split0_test_score', 'split1_test_score',
654-
'split2_test_score', 'std_test_score')
652+
score_keys = ('mean_test_score', 'mean_train_score', 'mean_test_time',
653+
'rank_test_score', 'split0_test_score', 'split1_test_score',
654+
'split2_test_score', 'split0_train_score',
655+
'split1_train_score', 'split2_train_score',
656+
'std_test_score', 'std_train_score', 'std_test_time')
655657
n_candidates = n_grid_points
656658

657659
for search, iid in zip((grid_search, grid_search_iid), (False, True)):
658660
assert_equal(iid, search.iid)
659661
results = search.cv_results_
662+
# Check if score and timing are reasonable
663+
assert_true(all(results['test_rank_test_score'] >= 1))
664+
assert_true(all(results[k] >= 0) for k in score_keys
665+
if k is not 'rank_test_score')
666+
assert_true(all(results[k] <= 1) for k in score_keys
667+
if not k.endswith('time') and
668+
k is not 'rank_test_score')
660669
# Check results structure
661670
check_cv_results_array_types(results, param_keys, score_keys)
662671
check_cv_results_keys(results, param_keys, score_keys, n_candidates)
@@ -689,18 +698,22 @@ def test_random_search_results():
689698
n_search_iter = 30
690699
params = dict(C=expon(scale=10), gamma=expon(scale=0.1))
691700
random_search = RandomizedSearchCV(SVC(), n_iter=n_search_iter,
692-
cv=n_splits,
693-
iid=False, param_distributions=params)
701+
cv=n_splits, iid=False,
702+
param_distributions=params,
703+
return_train_score=True)
694704
random_search.fit(X, y)
695705
random_search_iid = RandomizedSearchCV(SVC(), n_iter=n_search_iter,
696706
cv=n_splits, iid=True,
697-
param_distributions=params)
707+
param_distributions=params,
708+
return_train_score=True)
698709
random_search_iid.fit(X, y)
699710

700711
param_keys = ('param_C', 'param_gamma')
701-
score_keys = ('mean_test_score', 'rank_test_score',
702-
'split0_test_score', 'split1_test_score',
703-
'split2_test_score', 'std_test_score')
712+
score_keys = ('test_mean_score', 'train_mean_score', 'test_mean_time',
713+
'test_rank_score', 'test_split0_score', 'test_split1_score',
714+
'test_split2_score', 'train_split0_score',
715+
'train_split1_score', 'train_split2_score',
716+
'test_std_score', 'train_std_score', 'test_std_time')
704717
n_cand = n_search_iter
705718

706719
for search, iid in zip((random_search, random_search_iid), (False, True)):

0 commit comments

Comments
 (0)
0