8000 Revert "[MRG+2] Fix LDA predict_proba() (#11796)" · xhluca/scikit-learn@1f6bed6 · GitHub
Skip to content

Commit 1f6bed6

Browse files
author
Xing
authored
Revert "[MRG+2] Fix LDA predict_proba() (scikit-learn#11796)"
This reverts commit deea1e8.
1 parent 5dc497e commit 1f6bed6

File tree

3 files changed

+10
-96
lines changed

3 files changed

+10
-96
lines changed

‎doc/whats_new/v0.21.rst

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,6 @@ parameters, may produce different models from the previous version. This often
1717
occurs due to changes in the modelling logic (bug fixes or enhancements), or in
1818
random sampling procedures.
1919

20-
- :class:`discriminant_analysis.LinearDiscriminantAnalysis` for multiclass
21-
classification. |Fix|
22-
- :class:`discriminant_analysis.LinearDiscriminantAnalysis` with 'eigen'
23-
solver. |Fix|
2420
- :class:`linear_model.BayesianRidge` |Fix|
2521
- Decision trees and derived ensembles when both `max_depth` and
2622
`max_leaf_nodes` are set. |Fix|
@@ -111,16 +107,6 @@ Support for Python 3.4 and below has been officially dropped.
111107
Previously the change was made, but silently. :issue:`11526` by
112108
:user:`William de Vazelhes<wdevazelhes>`.
113109

114-
- |Fix| Fixed a bug in :class:`discriminant_analysis.LinearDiscriminantAnalysis`
115-
where the predicted probabilities would be incorrectly computed in the
116-
multiclass case. :issue:`6848`, by :user:`Agamemnon Krasoulis
117-
<agamemnonc>` and `Guillaume Lemaitre <glemaitre>`.
118-
119-
- |Fix| Fixed a bug in :class:`discriminant_analysis.LinearDiscriminantAnalysis`
120-
where the predicted probabilities would be incorrectly computed with ``eigen``
121-
solver. :issue:`11727`, by :user:`Agamemnon Krasoulis
122-
<agamemnonc>`.
123-
124110
:mod:`sklearn.dummy`
125111
....................
126112

‎sklearn/discriminant_analysis.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
from .utils import check_array, check_X_y
2323
from .utils.validation import check_is_fitted
2424
from .utils.multiclass import check_classification_targets
25-
from .utils.extmath import softmax
2625
from .preprocessing import StandardScaler
2726

2827

@@ -339,6 +338,7 @@ class scatter). This solver supports both classification and
339338
self.explained_variance_ratio_ = np.sort(evals / np.sum(evals)
340339
)[::-1][:self._max_components]
341340
evecs = evecs[:, np.argsort(evals)[::-1]] # sort eigenvectors
341+
evecs /= np.linalg.norm(evecs, axis=0)
342342

343343
self.scalings_ = evecs
344344
self.coef_ = np.dot(self.means_, evecs).dot(evecs.T)
@@ -531,14 +531,14 @@ def predict_proba(self, X):
531531
C : array, shape (n_samples, n_classes)
532532
Estimated probabilities.
533533
"""
534-
check_is_fitted(self, 'classes_')
535-
536-
decision = self.decision_function(X)
537-
if self.classes_.size == 2:
538-
proba = expit(decision)
539-
return np.vstack([1-proba, proba]).T
534+
prob = self.decision_function(X)
535+
expit(prob, out=prob)
536+
if len(self.classes_) == 2: # binary case
537+
return np.column_stack([1 - prob, prob])
540538
else:
541-
return softmax(decision)
539+
# OvR normalization, like LibLinear's predict_probability
540+
prob /= prob.sum(axis=1).reshape((prob.shape[0], -1))
541+
return prob
542542

543543
def predict_log_proba(self, X):
544544
"""Estimate log probability.

‎sklearn/tests/test_discriminant_analysis.py

Lines changed: 2 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22

33
import pytest
44

5-
from numpy.testing import assert_allclose
6-
from scipy import linalg
7-
85
from sklearn.exceptions import ChangedBehaviorWarning
96
from sklearn.utils import check_random_state
107
from sklearn.utils.testing import (assert_array_equal, assert_no_warnings,
@@ -98,75 +95,6 @@ def test_lda_predict():
9895
assert_raises(ValueError, clf.fit, X, y)
9996

10097

101-
@pytest.mark.parametrize("n_classes", [2, 3])
102-
@pytest.mark.parametrize("solver", ["svd", "lsqr", "eigen"])
103-
def test_lda_predict_proba(solver, n_classes):
104-
def generate_dataset(n_samples, centers, covariances, random_state=None):
105-
"""Generate a multivariate normal data given some centers and
106-
covariances"""
107-
rng = check_random_state(random_state)
108-
X = np.vstack([rng.multivariate_normal(mean, cov,
109-
size=n_samples // len(centers))
110-
for mean, cov in zip(centers, covariances)])
111-
y = np.hstack([[clazz] * (n_samples // len(centers))
112-
for clazz in range(len(centers))])
113-
return X, y
114-
115-
blob_centers = np.array([[0, 0], [-10, 40], [-30, 30]])[:n_classes]
116-
blob_stds = np.array([[[10, 10], [10, 100]]] * len(blob_centers))
117-
X, y = generate_dataset(
118-
n_samples=90000, centers=blob_centers, covariances=blob_stds,
119-
random_state=42
120-
)
121-
lda = LinearDiscriminantAnalysis(solver=solver, store_covariance=True,
122-
shrinkage=None).fit(X, y)
123-
# check that the empirical means and covariances are close enough to the
124-
# one used to generate the data
125-
assert_allclose(lda.means_, blob_centers, atol=1e-1)
126-
assert_allclose(lda.covariance_, blob_stds[0], atol=1)
127-
128-
# implement the method to compute the probability given in The Elements
129-
# of Statistical Learning (cf. p.127, Sect. 4.4.5 "Logistic Regression
130-
# or LDA?")
131-
precision = linalg.inv(blob_stds[0])
132-
alpha_k = []
133-
alpha_k_0 = []
134-
for clazz in range(len(blob_centers) - 1):
135-
alpha_k.append(
136-
np.dot(precision,
137-
(blob_centers[clazz] - blob_centers[-1])[:, np.newaxis]))
138-
alpha_k_0.append(
139-
np.dot(- 0.5 * (blob_centers[clazz] +
140-
blob_centers[-1])[np.newaxis, :], alpha_k[-1]))
141-
142-
sample = np.array([[-22, 22]])
143-
144-
def discriminant_func(sample, coef, intercept, clazz):
145-
return np.exp(intercept[clazz] + np.dot(sample, coef[clazz]))
146-
147-
prob = np.array([float(
148-
discriminant_func(sample, alpha_k, alpha_k_0, clazz) /
149-
(1 + sum([discriminant_func(sample, alpha_k, alpha_k_0, clazz)
150-
for clazz in range(n_classes - 1)]))) for clazz in range(
151-
n_classes - 1)])
152-
153-
prob_ref = 1 - np.sum(prob)
154-
155-
# check the consistency of the computed probability
156-
# all probabilities should sum to one
157-
prob_ref_2 = float(
158-
1 / (1 + sum([discriminant_func(sample, alpha_k, alpha_k_0, clazz)
159-
for clazz in range(n_classes - 1)]))
160-
)
161-
162-
assert prob_ref == pytest.approx(prob_ref_2)
163-
# check that the probability of LDA are close to the theoretical
164-
# probabilties
165-
assert_allclose(lda.predict_proba(sample),
166-
np.hstack([prob, prob_ref])[np.newaxis],
167-
atol=1e-2)
168-
169-
17098
def test_lda_priors():
17199
# Test priors (negative priors)
172100
priors = np.array([0.5, -0.5])
@@ -301,7 +229,7 @@ def test_lda_scaling():
301229

302230

303231
def test_lda_store_covariance():
304-
# Test for solver 'lsqr' and 'eigen'
232+
# Test for slover 'lsqr' and 'eigen'
305233
# 'store_covariance' has no effect on 'lsqr' and 'eigen' solvers
306234
for solver in ('lsqr', 'eigen'):
307235
clf = LinearDiscriminantAnalysis(solver=solver).fit(X6, y6)
@@ -317,7 +245,7 @@ def test_lda_store_covariance():
< 61CE code>317245
np.array([[0.422222, 0.088889], [0.088889, 0.533333]])
318246
)
319247

320-
# Test for SVD solver, the default is to not set the covariances_ attribute
248+
# Test for SVD slover, the default is to not set the covariances_ attribute
321249
clf = LinearDiscriminantAnalysis(solver='svd').fit(X6, y6)
322250
assert not hasattr(clf, 'covariance_')
323251

0 commit comments

Comments
 (0)
0