FIX Fixed gradient computation in gradient boosting for multiclass classification (scikit-learn#12715)

NicolasHug · adrinjalali · commit e73acef80de4 · 2018-12-19T15:14:38.000+01:00
* Fixed gradient update

* Added whatsnew entry
diff --git a/doc/whats_new/v0.21.rst b/doc/whats_new/v0.21.rst
@@ -24,6 +24,8 @@ random sampling procedures.
   `max_leaf_nodes` are set. |Fix|
 - :class:`linear_model.LogisticRegression` and
   :class:`linear_model.LogisticRegressionCV` with 'saga' solver. |Fix|
+- :class:`ensemble.GradientBoostingClassifier` for multiclass
+  classification. |Fix|
 
 
 Details are listed in the changelog below.
@@ -71,6 +73,10 @@ Support for Python 3.4 and below has been officially dropped.
   communication overhead. :issue:`12543` by :user:`Isaac Storch <istorch>`
   and `Olivier Grisel`_.
 
+- |Fix| Fixed a bug in :class:`ensemble.GradientBoostingClassifier` where
+  the gradients would be incorrectly computed in multiclass classification
+  problems. :issue:`12715` by :user:`Nicolas Hug<NicolasHug>`.
+
 :mod:`sklearn.linear_model`
 ...........................
 
diff --git a/sklearn/ensemble/gradient_boosting.py b/sklearn/ensemble/gradient_boosting.py
@@ -1163,11 +1163,17 @@ def _fit_stage(self, i, X, y, y_pred, sample_weight, sample_mask,
         loss = self.loss_
         original_y = y
 
+        # Need to pass a copy of y_pred to negative_gradient() because y_pred
+        # is partially updated at the end of the loop in
+        # update_terminal_regions(), and gradients need to be evaluated at
+        # iteration i - 1.
+        y_pred_copy = y_pred.copy()
+
         for k in range(loss.K):
             if loss.is_multi_class:
                 y = np.array(original_y == k, dtype=np.float64)
 
-            residual = loss.negative_gradient(y, y_pred, k=k,
+            residual = loss.negative_gradient(y, y_pred_copy, k=k,
                                               sample_weight=sample_weight)
 
             # induce regression tree on residuals
