FIX Corrects negative gradient of AdaBoost loss in GBDT (scikit-learn#22050)

glemaitre · lorentzenchr · venkyyuvy · commit f45b09fba19a · 2022-01-01T10:17:10.000+05:30
Co-authored-by: Christian Lorentzen &lt;lorentzen.ch@gmail.com&gt;
diff --git a/doc/whats_new/v1.1.rst b/doc/whats_new/v1.1.rst
@@ -211,6 +211,11 @@ Changelog
   :class:`ensemble.ExtraTreesClassifier`.
   :pr:`20803` by :user:`Brian Sun <bsun94>`.
 
+- |Fix| Solve a bug in :class:`ensemble.GradientBoostingClassifier` where the
+  exponential loss was computing the positive gradient instead of the
+  negative one.
+  :pr:`22050` by :user:`Guillaume Lemaitre <glemaitre>`.
+
 :mod:`sklearn.feature_extraction.text`
 ......................................
 
diff --git a/sklearn/ensemble/_gb_losses.py b/sklearn/ensemble/_gb_losses.py
@@ -935,8 +935,8 @@ def negative_gradient(self, y, raw_predictions, **kargs):
             The raw predictions (i.e. values from the tree leaves) of the
             tree ensemble at iteration ``i - 1``.
         """
-        y_ = -(2.0 * y - 1.0)
-        return y_ * np.exp(y_ * raw_predictions.ravel())
+        y_ = 2.0 * y - 1.0
+        return y_ * np.exp(-y_ * raw_predictions.ravel())
 
     def _update_terminal_region(
         self,
diff --git a/sklearn/ensemble/tests/test_gradient_boosting_loss_functions.py b/sklearn/ensemble/tests/test_gradient_boosting_loss_functions.py
@@ -320,3 +320,18 @@ def test_lad_equals_quantiles(seed, alpha):
         y_true, raw_predictions, sample_weight=weights, alpha=alpha
     )
     assert pbl_weighted_loss == approx(ql_weighted_loss)
+
+
+def test_exponential_loss():
+    """Check that we compute the negative gradient of the exponential loss.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/9666
+    """
+    loss = ExponentialLoss(n_classes=2)
+    y_true = np.array([0])
+    y_pred = np.array([0])
+    # we expect to have loss = exp(0) = 1
+    assert loss(y_true, y_pred) == pytest.approx(1)
+    # we expect to have negative gradient = -1 * (1 * exp(0)) = -1
+    assert_allclose(loss.negative_gradient(y_true, y_pred), -1)
