8000 Improve multi-metric scoring computation. · scikit-learn/scikit-learn@9459bf9 · GitHub
[go: up one dir, main page]
Skip to content

Commit 9459bf9

Browse files
gamazepsgamazeps
committed
Improve multi-metric scoring computation.
Previously multi metric scoring called the `predict` method of an estimator once for each scorer, this could lead to drastic increases in costs. This change avoids calling the scorers directly and instead allows to call the scorer with the predicted results. This is only done for `_PredictScorer` and `_ProbaScorer` generated with `make_scorer`, this means that `_ThresholdScorer` and scorers not generated with `make_scorer` do not benefit from this change. Works on improving #10802
1 parent 20fcae2 commit 9459bf9

File tree

2 files changed

+105
-18
lines changed

2 files changed

+105
-18
lines changed

‎sklearn/metrics/scorer.py

Lines changed: 64 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@
2626
from . import (r2_score, median_absolute_error, mean_absolute_error,
2727
mean_squared_error, mean_squared_log_error, accuracy_score,
2828
f1_score, roc_auc_score, average_precision_score,
29-
precision_score, recall_score, log_loss, balanced_accuracy_score,
30-
explained_variance_score, brier_score_loss)
29+
precision_score, recall_score, log_loss,
30+
balanced_accuracy_score, explained_variance_score,
31+
brier_score_loss)
3132

3233
from .cluster import adjusted_rand_score
3334
from .cluster import homogeneity_score
@@ -96,9 +97,32 @@ def __call__(self, estimator, X, y_true, sample_weight=None):
9697
score : float
9798
Score function applied to prediction of estimator on X.
9899
"""
99-
super(_PredictScorer, self).__call__(estimator, X, y_true,
100-
sample_weight=sample_weight)
101100
y_pred = estimator.predict(X)
101+
return self.score_predict(y_pred, y_true, sample_weight)
102+
103+
def score_predict(self, y_pred, y_true, sample_weight=None):
104+
"""Evaluate predicted target values y_pred relative to y_true.
105+
106+
Parameters
107+
----------
108+
y_pred : array-like
109+
Prodicted values for y.
110+
111+
y_true : array-like
112+
Gold standard target values for y.
113+
114+
sample_weight : array-like, optional (default=None)
115+
Sample weights.
116+
117+
Returns
118+
-------
119+
score : float
120+
Score function applied to prediction of estimator on X.
121+
"""
122+
# We call __call__ with no arguments as it only serves to show
123+
# deprecation warnings.
124+
super(_PredictScorer, self).__call__(None, None, None,
125+
sample_weight=sample_weight)
102126
if sample_weight is not None:
103127
return self._sign * self._score_func(y_true, y_pred,
104128
sample_weight=sample_weight,
@@ -133,21 +157,49 @@ def __call__(self, clf, X, y, sample_weight=None):
133157
score : float
134158
Score function applied to prediction of estimator on X.
135159
"""
136-
super(_ProbaScorer, self).__call__(clf, X, y,
137-
sample_weight=sample_weight)
138-
y_type = type_of_target(y)
139160
y_pred = clf.predict_proba(X)
161+
162+
return self.score_predict(y_pred, y, sample_weight)
163+
164+
def _factory_args(self):
165+
return ", needs_proba=True"
166+
167+
def score_predict(self, y_pred, y_true, sample_weight=None):
168+
"""Evaluate predicted y_pred relative to y_true.
169+
170+
Parameters
171+
----------
172+
y_pred : array-like
173+
Predicted values for y by a classifier. These must be class labels,
174+
not probabilities.
175+
176+
y_true : array-like
177+
Gold standard target values for y. These must be class labels,
178+
not probabilities.
179+
180+
sample_weight : array-like, optional (default=None)
181+
Sample weights.
182+
183+
Returns
184+
-------
185+
score : float
186+
Score function applied to prediction of estimator on X.
187+
"""
188+
# We call __call__ with no arguments as it only serves to show
189+
# deprecation warnings.
190+
super(_ProbaScorer, self).__call__(None, None, None,
191+
sample_weight=sample_weight)
192+
y_type = type_of_target(y_true)
140193
if y_type == "binary":
141194
y_pred = y_pred[:, 1]
142195
if sample_weight is not None:
143-
return self._sign * self._score_func(y, y_pred,
196+
return self._sign * self._score_func(y_true, y_pred,
144197
sample_weight=sample_weight,
145198
**self._kwargs)
146199
else:
147-
return self._sign * self._score_func(y, y_pred, **self._kwargs)
148-
149-
def _factory_args(self):
150-
return ", needs_proba=True"
200+
return self._sign * self._score_func(y_true,
201+
y_pred,
202+
**self._kwargs)
151203

152204

153205
class _ThresholdScorer(_BaseScorer):

‎sklearn/model_selection/_validation.py

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@
2727
from ..utils.metaestimators import _safe_split
2828
from ..externals.joblib import Parallel, delayed, logger
2929
from ..externals.six.moves import zip
30-
from ..metrics.scorer import check_scoring, _check_multimetric_scoring
30+
from ..metrics.scorer import (check_scoring, _check_multimetric_scoring,
31+
_PredictScorer, _ProbaScorer)
3132
from ..exceptions import FitFailedWarning
3233
from ._split import check_cv
3334
from ..preprocessing import LabelEncoder
@@ -577,14 +578,48 @@ def _score(estimator, X_test, y_test, scorer, is_multimetric=False):
577578

578579
def _multimetric_score(estimator, X_test, y_test, scorers):
579580
"""Return a dict of score for multimetric scoring"""
580-
scores = {}
581-
582-
for name, scorer in scorers.items():
581+
def _is_proba(x):
582+
return isinstance(x, _ProbaScorer)
583+
584+
def _is_predict(x):
585+
return isinstance(x, _PredictScorer)
586+
587+
# We want to keep the memmap and score types in a single
588+
# loop.
589+
tmp_scores = {}
590+
591+
# The following two are special cases where we want to compute
592+
# the `predict` and `predict_proba` only once.
593+
# This is ugly but gives a good performance boost, see #10802
594+
# for more details.
595+
predict_scorers = [
596+
(name, sc) for name, sc in scorers.items()
597+
if _is_predict(sc)]
598+
if predict_scorers:
599+
y_pred = estimator.predict(X_test)
600+
for (name, scorer) in predict_scorers:
601+
tmp_scores[name] = scorer.score_predict(y_pred, y_test)
602+
603+
proba_scorers = [
604+
(name, sc) for name, sc in scorers.items()
605+
if _is_proba(sc)]
606+
if proba_scorers:
607+
y_pred = estimator.predict_proba(X_test)
608+
for (name, scorer) in proba_scorers:
609+
tmp_scores[name] = scorer.score_predict(y_pred, y_test)
610+
611+
other_scorers = [
612+
(name, sc) for name, sc in scorers.items()
613+
if not (_is_proba(sc) or _is_predict(sc))]
614+
for name, scorer in other_scorers:
583615
if y_test is None:
584-
score = scorer(estimator, X_test)
616+
tmp_scores[name] = scorer(estimator, X_test)
585617
else:
586-
score = scorer(estimator, X_test, y_test)
618+
tmp_scores[name] = scorer(estimator, X_test, y_test)
587619

620+
scores = {}
621+
for name in scorers:
622+
score = tmp_scores[name]
588623
if hasattr(score, 'item'):
589624
try:
590625
# e.g. unwrap memmapped scalars

0 commit comments

Comments
 (0)
0