scikit-learn · jnothman · Mar 26, 2019 · Feb 14, 2019 · Feb 14, 2019 · Feb 14, 2019
diff --git a/doc/whats_new/v0.21.rst b/doc/whats_new/v0.21.rst
@@ -26,8 +26,8 @@ random sampling procedures.
   `max_leaf_nodes` are set. |Fix|
 - :class:`linear_model.LogisticRegression` and
   :class:`linear_model.LogisticRegressionCV` with 'saga' solver. |Fix|
-- :class:`ensemble.GradientBoostingClassifier` for multiclass
-  classification. |Fix|
+- :class:`ensemble.GradientBoostingClassifier` |Fix|
+- :class:`neural_network.MLPClassifier` |Fix|
 - :func:`svm.SVC.decision_function` and
   :func:`multiclass.OneVsOneClassifier.decision_function`. |Fix|
 
@@ -183,6 +183,10 @@ Support for Python 3.4 and below has been officially dropped.
   the gradients would be incorrectly computed in multiclass classification
   problems. :issue:`12715` by :user:`Nicolas Hug<NicolasHug>`.
 
+- |Fix| Fixed a bug in :class:`ensemble.GradientBoostingClassifier` where
+  validation sets for early stopping were not sampled with stratification.
+  :issue:`13164` by :user:`Nicolas Hug<NicolasHug>`.
+
 - |Fix| Fixed a bug in :class:`ensemble.GradientBoostingClassifier` where
   the default initial prediction of a multiclass classifier would predict the
   classes priors instead of the log of the priors. :issue:`12983` by
@@ -422,6 +426,11 @@ Support for Python 3.4 and below has been officially dropped.
   :class:`neural_network.MLPRegressor` where the option :code:`shuffle=False`
   was being ignored. :issue:`12582` by :user:`Sam Waterbury <samwaterbury>`.
 
+- |Fix| Fixed a bug in :class:`neural_network.MLPClassifier` where
+  validation sets for early stopping were not sampled with stratification. In
+  multilabel case however, splits are still not stratified.
+  :issue:`13164` by :user:`Nicolas Hug<NicolasHug>`.
+
 :mod:`sklearn.pipeline`
 .......................
 

diff --git a/sklearn/ensemble/gradient_boosting.py b/sklearn/ensemble/gradient_boosting.py
@@ -1447,10 +1447,12 @@ def fit(self, X, y, sample_weight=None, monitor=None):
         y = self._validate_y(y, sample_weight)
 
         if self.n_iter_no_change is not None:
+            stratify = y if is_classifier(self) else None
             X, X_val, y, y_val, sample_weight, sample_weight_val = (
                 train_test_split(X, y, sample_weight,
                                  random_state=self.random_state,
-                                 test_size=self.validation_fraction))
+                                 test_size=self.validation_fraction,
+                                 stratify=stratify))
             if is_classifier(self):
                 if self.n_classes_ != np.unique(y).shape[0]:
                     # We choose to error here. The problem is that the init
@@ -1933,7 +1935,7 @@ class GradientBoostingClassifier(BaseGradientBoosting, ClassifierMixin):
         number, it will set aside ``validation_fraction`` size of the training
         data as validation and terminate training when validation score is not
         improving in all of the previous ``n_iter_no_change`` numbers of
-        iterations.
+        iterations. The split is stratified.
 
         .. versionadded:: 0.20
 

diff --git a/sklearn/ensemble/tests/test_gradient_boosting.py b/sklearn/ensemble/tests/test_gradient_boosting.py
@@ -1265,8 +1265,8 @@ def test_gradient_boosting_early_stopping():
     X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                         random_state=42)
     # Check if early_stopping works as expected
-    for est, tol, early_stop_n_estimators in ((gbc, 1e-1, 24), (gbr, 1e-1, 13),
-                                              (gbc, 1e-3, 36),
+    for est, tol, early_stop_n_estimators in ((gbc, 1e-1, 28), (gbr, 1e-1, 13),
+                                              (gbc, 1e-3, 70),
                                               (gbr, 1e-3, 28)):
         est.set_params(tol=tol)
         est.fit(X_train, y_train)
@@ -1321,6 +1321,18 @@ def test_gradient_boosting_validation_fraction():
     assert gbc.n_estimators_ < gbc3.n_estimators_
 
 
+def test_early_stopping_stratified():
+    # Make sure data splitting for early stopping is stratified
+    X = [[1, 2], [2, 3], [3, 4], [4, 5]]
+    y = [0, 0, 0, 1]
+
+    gbc = GradientBoostingClassifier(n_iter_no_change=5)
+    with pytest.raises(
+            ValueError,
+            match='The least populated class in y has only 1 member'):
+        gbc.fit(X, y)
+
+
 class _NoSampleWeightWrapper(BaseEstimator):
     def __init__(self, est):
         self.est = est
@@ -1381,19 +1393,20 @@ def test_gradient_boosting_init_wrong_methods(estimator, missing_method):
 
 
 def test_early_stopping_n_classes():
-    # when doing early stopping (_, y_train, _, _ = train_test_split(X, y))
+    # when doing early stopping (_, , y_train, _ = train_test_split(X, y))
     # there might be classes in y that are missing in y_train. As the init
     # estimator will be trained on y_train, we need to raise an error if this
     # happens.
 
-    X = [[1, 2], [2, 3], [3, 4], [4, 5]]
-    y = [0, 1, 1, 1]
-    gb = GradientBoostingClassifier(n_iter_no_change=5, random_state=4)
+    X = [[1]] * 10
+    y = [0, 0] + [1] * 8  # only 2 negative class over 10 samples
+    gb = GradientBoostingClassifier(n_iter_no_change=5, random_state=0,
+                                    validation_fraction=8)
     with pytest.raises(
                 ValueError,
                 match='The training data after the early stopping split'):
         gb.fit(X, y)
 
-    # No error with another random seed
-    gb = GradientBoostingClassifier(n_iter_no_change=5, random_state=0)
-    gb.fit(X, y)
+    # No error if we let training data be big enough
+    gb = GradientBoostingClassifier(n_iter_no_change=5, random_state=0,
+                                    validation_fraction=4)
diff --git a/sklearn/linear_model/passive_aggressive.py b/sklearn/linear_model/passive_aggressive.py
@@ -37,8 +37,8 @@ class PassiveAggressiveClassifier(BaseSGDClassifier):
     early_stopping : bool, default=False
         Whether to use early stopping to terminate training when validation.
         score is not improving. If set to True, it will automatically set aside
-        a fraction of training data as validation and terminate training when
-        validation score is not improving by at least tol for
+        a stratified fraction of training data as validation and terminate
+        training when validation score is not improving by at least tol for
         n_iter_no_change consecutive epochs.
 
         .. versionadded:: 0.20
@@ -282,8 +282,8 @@ class PassiveAggressiveRegressor(BaseSGDRegressor):
     early_stopping : bool, default=False
         Whether to use early stopping to terminate training when validation.
         score is not improving. If set to True, it will automatically set aside
-        a fraction of training data as validation and terminate training when
-        validation score is not improving by at least tol for
+        a fraction of training data as validation and terminate
+        training when validation score is not improving by at least tol for
         n_iter_no_change consecutive epochs.
 
         .. versionadded:: 0.20

diff --git a/sklearn/linear_model/perceptron.py b/sklearn/linear_model/perceptron.py
@@ -62,8 +62,8 @@ class Perceptron(BaseSGDClassifier):
     early_stopping : bool, default=False
         Whether to use early stopping to terminate training when validation.
         score is not improving. If set to True, it will automatically set aside
-        a fraction of training data as validation and terminate training when
-        validation score is not improving by at least tol for
+        a stratified fraction of training data as validation and terminate
+        training when validation score is not improving by at least tol for
         n_iter_no_change consecutive epochs.
 
         .. versionadded:: 0.20

diff --git a/sklearn/linear_model/stochastic_gradient.py b/sklearn/linear_model/stochastic_gradient.py
@@ -828,8 +828,8 @@ class SGDClassifier(BaseSGDClassifier):
     early_stopping : bool, default=False
         Whether to use early stopping to terminate training when validation
         score is not improving. If set to True, it will automatically set aside
-        a fraction of training data as validation and terminate training when
-        validation score is not improving by at least tol for
+        a stratified fraction of training data as validation and terminate
+        training when validation score is not improving by at least tol for
         n_iter_no_change consecutive epochs.
 
         .. versionadded:: 0.20
@@ -1433,8 +1433,8 @@ class SGDRegressor(BaseSGDRegressor):
     early_stopping : bool, default=False
         Whether to use early stopping to terminate training when validation
         score is not improving. If set to True, it will automatically set aside
-        a fraction of training data as validation and terminate training when
-        validation score is not improving by at least tol for
+        a fraction of training data as validation and terminate
+        training when validation score is not improving by at least tol for
         n_iter_no_change consecutive epochs.
 
         .. versionadded:: 0.20

diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
@@ -484,9 +484,13 @@ def _fit_stochastic(self, X, y, activations, deltas, coef_grads,
         # early_stopping in partial_fit doesn't make sense
         early_stopping = self.early_stopping and not incremental
         if early_stopping:
+            # don't stratify in multilabel classification
+            should_stratify = is_classifier(self) and self.n_outputs_ == 1
+            stratify = y if should_stratify else None
             X, X_val, y, y_val = train_test_split(
                X, y, random_state=self._random_state,
-                test_size=self.validation_fraction)
+                test_size=self.validation_fraction,
+                stratify=stratify)
             if is_classifier(self):
                 y_val = self._label_binarizer.inverse_transform(y_val)
         else:
@@ -803,7 +807,8 @@ class MLPClassifier(BaseMultilayerPerceptron, ClassifierMixin):
         score is not improving. If set to true, it will automatically set
         aside 10% of training data as validation and terminate training when
         validation score is not improving by at least tol for
-        ``n_iter_no_change`` consecutive epochs.
+        ``n_iter_no_change`` consecutive epochs. The split is stratified,
+        except in a multilabel setting.
         Only effective when solver='sgd' or 'adam'
 
     validation_fraction : float, optional, default 0.1

diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
@@ -308,6 +308,11 @@ def test_multilabel_classification():
         mlp.partial_fit(X, y, classes=[0, 1, 2, 3, 4])
     assert_greater(mlp.score(X, y), 0.9)
 
+    # Make sure early stopping still work now that spliting is stratified by
+    # default (it is disabled for multilabel classification)
+    mlp = MLPClassifier(early_stopping=True)
+    mlp.fit(X, y).predict(X)
+
 
 @pytest.mark.filterwarnings('ignore: The default value of multioutput')  # 0.23
 def test_multioutput_regression():
@@ -663,3 +668,15 @@ def test_n_iter_no_change_inf():
 
     # validate _update_no_improvement_count() was always triggered
     assert_equal(clf._no_improvement_count, clf.n_iter_ - 1)
+
+
+def test_early_stopping_stratified():
+    # Make sure data splitting for early stopping is stratified
+    X = [[1, 2], [2, 3], [3, 4], [4, 5]]
+    y = [0, 0, 0, 1]
+
+    mlp = MLPClassifier(early_stopping=True)
+    with pytest.raises(
+            ValueError,
+            match='The least populated class in y has only 1 member'):
+        mlp.fit(X, y)