8000 FIX make sure the decision function of weak learner is symmetric (#26… · scikit-learn/scikit-learn@9f03c03 · GitHub
[go: up one dir, main page]
Skip to content

Commit 9f03c03

Browse files
glemaitrejeremiedbbogrisel
authored
FIX make sure the decision function of weak learner is symmetric (#26521)
Co-authored-by: Jérémie du Boisberranger <34657725+jeremiedbb@users.noreply.github.com> Co-authored-by: Olivier Grisel <olivier.grisel@ensta.org>
1 parent 4f17b5a commit 9f03c03

File tree

3 files changed

+61
-3
lines changed

3 files changed

+61
-3
lines changed

‎doc/whats_new/v1.3.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,14 @@ Changelog
2828
- |Fix| :class:`cluster.BisectingKMeans` now works with data that has a single feature.
2929
:pr:`27243` by `Jérémie du Boisberranger <jeremiedbb>`.
3030

31+
:mod:`sklearn.ensemble`
32+
.......................
33+
34+
- |Fix| Fix a bug in :class:`ensemble.AdaBoostClassifier` with `algorithm="SAMME"`
35+
where the decision function of each weak learner should be symmetric (i.e.
36+
the sum of the scores should sum to zero for a sample).
37+
:pr:`26521` by :user:`Guillaume Lemaitre <glemaitre>`.
38+
3139
:mod:`sklearn.impute`
3240
.....................
3341

‎sklearn/ensemble/_weight_boosting.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -780,7 +780,11 @@ class in ``classes_``, respectively.
780780
)
781781
else: # self.algorithm == "SAMME"
782782
pred = sum(
783-
(estimator.predict(X) == classes).T * w
783+
np.where(
784+
(estimator.predict(X) == classes).T,
785+
w,
786+
-1 / (n_classes - 1) * w,
787+
)
784788
for estimator, w in zip(self.estimators_, self.estimator_weights_)
785789
)
786790

@@ -827,8 +831,11 @@ class in ``classes_``, respectively.
827831
# The weights are all 1. for SAMME.R
828832
current_pred = _samme_proba(estimator, n_classes, X)
829833
else: # elif self.algorithm == "SAMME":
830-
current_pred = estimator.predict(X)
831-
current_pred = (current_pred == classes).T * weight
834+
current_pred = np.where(
835+
(estimator.predict(X) == classes).T,
836+
weight,
837+
-1 / (n_classes - 1) * weight,
838+
)
832839

833840
if pred is None:
834841
pred = current_pred

‎sklearn/ensemble/tests/test_weight_boosting.py

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from sklearn.utils import shuffle
1818
from sklearn.utils._mocking import NoSampleWeightWrapper
1919
from sklearn.utils._testing import (
20+
assert_allclose,
2021
assert_array_almost_equal,
2122
assert_array_equal,
2223
assert_array_less,
@@ -693,3 +694,45 @@ def test_deprecated_base_estimator_parameters_can_be_set():
693694

694695
with pytest.warns(FutureWarning, match="Parameter 'base_estimator' of"):
695696
clf.set_params(base_estimator__max_depth=2)
697+
698+
699+
@pytest.mark.parametrize("algorithm", ["SAMME", "SAMME.R"])
700+
def test_adaboost_decision_function(algorithm, global_random_seed):
701+
"""Check that the decision function respects the symmetric constraint for weak
702+
learners.
703+
704+
Non-regression test for:
705+
https://github.com/scikit-learn/scikit-learn/issues/26520
706+
"""
707+
n_classes = 3
708+
X, y = datasets.make_classification(
709+
n_classes=n_classes, n_clusters_per_class=1, random_state=global_random_seed
710+
)
711+
clf = AdaBoostClassifier(
712+
n_estimators=1, random_state=global_random_seed, algorithm=algorithm
713+
).fit(X, y)
714+
715+
y_score = clf.decision_function(X)
716+
assert_allclose(y_score.sum(axis=1), 0, atol=1e-8)
717+
718+
if algorithm == "SAMME":
719+
# With a single learner, we expect to have a decision function in
720+
# {1, - 1 / (n_classes - 1)}.
721+
assert set(np.unique(y_score)) == {1, -1 / (n_classes - 1)}
722+
723+
# We can assert the same for staged_decision_function since we have a single learner
724+
for y_score in clf.staged_decision_function(X):
725+
assert_allclose(y_score.sum(axis=1), 0, atol=1e-8)
726+
727+
if algorithm == "SAMME":
728+
# With a single learner, we expect to have a decision function in
729+
# {1, - 1 / (n_classes - 1)}.
730+
assert set(np.unique(y_score)) == {1, -1 / (n_classes - 1)}
731+
732+
clf.set_params(n_estimators=5).fit(X, y)
733+
734+
y_score = clf.decision_function(X)
735+
assert_allclose(y_score.sum(axis=1), 0, atol=1e-8)
736+
737+
for y_score in clf.staged_decision_function(X):
738+
assert_allclose(y_score.sum(axis=1), 0, atol=1e-8)

0 commit comments

Comments
 (0)
0