8000 Improve multi-metric scoring computation. · scikit-learn/scikit-learn@4469067 · GitHub
[go: up one dir, main page]
Skip to content

Commit 4469067

Browse files
gamazepsgamazeps
committed
Improve multi-metric scoring computation.
Previously multi metric scoring called the `predict` method of an estimator once for each scorer, this could lead to drastic increases in costs. This change avoids calling the scorers directly and instead allows to call the scorer with the predicted results. This is only done for `_PredictScorer` and `_ProbaScorer` generated with `make_scorer`, this means that `_ThresholdScorer` and scorers not generated with `make_scorer` do not benefit from this change. Works on improving #10802
1 parent 20fcae2 commit 4469067

File tree

2 files changed

+105
-20
lines changed

2 files changed

+105
-20
lines changed

‎sklearn/metrics/scorer.py

Lines changed: 66 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@
2626
from . import (r2_score, median_absolute_error, mean_absolute_error,
2727
mean_squared_error, mean_squared_log_error, accuracy_score,
2828
f1_score, roc_auc_score, average_precision_score,
29-
precision_score, recall_score, log_loss, balanced_accuracy_score,
30-
explained_variance_score, brier_score_loss)
29+
precision_score, recall_score, log_loss,
30+
balanced_accuracy_score, explained_variance_score,
31+
brier_score_loss)
3132

3233
from .cluster import adjusted_rand_score
3334
from .cluster import homogeneity_score
@@ -96,9 +97,32 @@ def __call__(self, estimator, X, y_true, sample_weight=None):
9697
score : float
9798
Score function applied to prediction of estimator on X.
9899
"""
99-
super(_PredictScorer, self).__call__(estimator, X, y_true,
100-
sample_weight=sample_weight)
101100
y_pred = estimator.predict(X)
101+
return self.score_predictions(y_true, y_pred, sample_weight)
102+
103+
def score_predictions(self, y_true, y_pred, sample_weight=None):
104+
"""Evaluate predicted target values y_pred relative to y_true.
105+
106+
Parameters
107+
----------
108+
y_pred : array-like
109+
Prodicted values for y.
110+
111+
y_true : array-like
112+
Gold standard target values for y.
113+
114+
sample_weight : array-like, optional (default=None)
115+
Sample weights.
116+
117+
Returns
118+
-------
119+
score : float
120+
Score function applied to prediction of estimator on X.
121+
"""
122+
# We call __call__ with no arguments as it only serves to show
123+
# deprecation warnings.
124+
super(_PredictScorer, self).__call__(None, None, None,
125+
sample_weight=sample_weight)
102126
if sample_weight is not None:
103127
return self._sign * self._score_func(y_true, y_pred,
104128
sample_weight=sample_weight,
@@ -109,7 +133,7 @@ def __call__(self, estimator, X, y_true, sample_weight=None):
109133

110134

111135
class _ProbaScorer(_BaseScorer):
112-
def __call__(self, clf, X, y, sample_weight=None):
136+
def __call__(self, clf, X, y_true, sample_weight=None):
113137
"""Evaluate predicted probabilities for X relative to y_true.
114138
115139
Parameters
@@ -121,7 +145,7 @@ def __call__(self, clf, X, y, sample_weight=None):
121145
X : array-like or sparse matrix
122146
Test data that will be fed to clf.predict_proba.
123147
124-
y : array-like
148+
y_true : array-like
125149
Gold standard target values for X. These must be class labels,
126150
not probabilities.
127151
@@ -133,21 +157,49 @@ def __call__(self, clf, X, y, sample_weight=None):
133157
score : float
134158
Score function applied to prediction of estimator on X.
135159
"""
136-
super(_ProbaScorer, self).__call__(clf, X, y,
137-
sample_weight=sample_weight)
138-
y_type = type_of_target(y)
139160
y_pred = clf.predict_proba(X)
161+
162+
return self.score_predictions(y_true, y_pred, sample_weight)
163+
164+
def _factory_args(self):
165+
return ", needs_proba=True"
166+
167+
def score_predictions(self, y_true, y_pred, sample_weight=None):
168+
"""Evaluate predicted y_pred relative to y_true.
169+
170+
Parameters
171+
----------
172+
y_pred : array-like
173+
Predicted values for y by a classifier. These must be class labels,
174+
not probabilities.
175+
176+
y_true : array-like
177+
Gold standard target values for y. These must be class labels,
178+
not probabilities.
179+
180+
sample_weight : array-like, optional (default=None)
181+
Sample weights.
182+
183+
Returns
184+
-------
185+
score : float
186+
Score function applied to prediction of estimator on X.
187+
"""
188+
# We call __call__ with no arguments as it only serves to show
189+
# deprecation warnings.
190+
super(_ProbaScorer, self).__call__(None, None, None,
191+
sample_weight=sample_weight)
192+
y_type = type_of_target(y_true)
140193
if y_type == "binary":
141194
y_pred = y_pred[:, 1]
142195
if sample_weight is not None:
143-
return self._sign * self._score_func(y, y_pred,
196+
return self._sign * self._score_func(y_true, y_pred,
144197
sample_weight=sample_weight,
145198
**self._kwargs)
146199
else:
147-
return self._sign * self._score_func(y, y_pred, **self._kwargs)
148-
149-
def _factory_args(self):
150-
return ", needs_proba=True"
200+
return self._sign * self._score_func(y_true,
201+
y_pred,
202+
**self._kwargs)
151203

152204

153205
class _ThresholdScorer(_BaseScorer):

‎sklearn/model_selection/_validation.py

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@
2727
from ..utils.metaestimators import _safe_split
2828
from ..externals.joblib import Parallel, delayed, logger
2929
from ..externals.six.moves import zip
30-
from ..metrics.scorer import check_scoring, _check_multimetric_scoring
30+
from ..metrics.scorer import (check_scoring, _check_multimetric_scoring,
31+
_PredictScorer, _ProbaScorer)
3132
from ..exceptions import FitFailedWarning
3233
from ._split import check_cv
3334
from ..preprocessing import LabelEncoder
@@ -577,14 +578,46 @@ def _score(estimator, X_test, y_test, scorer, is_multimetric=False):
577578

578579
def _multimetric_score(estimator, X_test, y_test, scorers):
579580
"""Return a dict of score for multimetric scoring"""
580-
scores = {}
581-
582-
for name, scorer in scorers.items():
581+
def _is_proba(x):
582+
return isinstance(x, _ProbaScorer)
583+
584+
def _is_predict(x):
585+
return isinstance(x, _PredictScorer)
586+
587+
tmp_scores = {}
588+
589+
# The following two are special cases where we want to compute
590+
# the `predict` and `predict_proba` only once.
591+
# This is ugly but gives a good performance boost, see #10802
592+
# for more details.
593+
predict_scorers = [
594+
(name, sc) for name, sc in scorers.items()
595+
if _is_predict(sc)]
596+
if predict_scorers:
597+
y_pred = estimator.predict(X_test)
598+
for (name, scorer) in predict_scorers:
599+
tmp_scores[name] = scorer.score_predictions(y_test, y_pred)
600+
601+
proba_scorers = [
602+
(name, sc) for name, sc in scorers.items()
603+
if _is_proba(sc)]
604+
if proba_scorers:
605+
y_pred = estimator.predict_proba(X_test)
606+
for (name, scorer) in proba_scorers:
607+
tmp_scores[name] = scorer.score_predictions(y_test, y_pred)
608+
609+
other_scorers = [
610+
(name, sc) for name, sc in scorers.items()
611+
if not (_is_proba(sc) or _is_predict(sc))]
612+
for name, scorer in other_scorers:
583613
if y_test is None:
584-
score = scorer(estimator, X_test)
614+
tmp_scores[name] = scorer(estimator, X_test)
585615
else:
586-
score = scorer(estimator, X_test, y_test)
616+
tmp_scores[name] = scorer(estimator, X_test, y_test)
587617

618+
scores = {}
619+
for name in scorers:
620+
score = tmp_scores[name]
588621
if hasattr(score, 'item'):
589622
try:
590623
# e.g. unwrap memmapped scalars

0 commit comments

Comments
 (0)
0