From 46f76cb25511130a1cd0354791ae3306b7f78ef0 Mon Sep 17 00:00:00 2001
From: Marvvxi <marvin.krawutschke@outlook.de>
Date: Fri, 20 Jan 2023 15:22:42 +0100
Subject: [PATCH 01/11] Bug: Add return for break when max iter is reached

---
 sklearn/neural_network/_multilayer_perceptron.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sklearn/neural_network/_multilayer_perceptron.py b/sklearn/neural_network/_multilayer_perceptron.py
index 7ed0ab33a0f29..d513ca3ea2227 100644
--- a/sklearn/neural_network/_multilayer_perceptron.py
+++ b/sklearn/neural_network/_multilayer_perceptron.py
@@ -687,6 +687,7 @@ def _fit_stochastic(
                         % self.max_iter,
                         ConvergenceWarning,
                     )
+                    return
         except KeyboardInterrupt:
             warnings.warn("Training interrupted by user.")
 

From d8bcaf9ac5f493a85ebb94d58c1cd01f336819a4 Mon Sep 17 00:00:00 2001
From: Marvvxi <marvin.krawutschke@outlook.de>
Date: Fri, 20 Jan 2023 18:09:01 +0100
Subject: [PATCH 02/11] Bug: Change return to break (out of loop)

---
 sklearn/neural_network/_multilayer_perceptron.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sklearn/neural_network/_multilayer_perceptron.py b/sklearn/neural_network/_multilayer_perceptron.py
index d513ca3ea2227..f950e6d5c649c 100644
--- a/sklearn/neural_network/_multilayer_perceptron.py
+++ b/sklearn/neural_network/_multilayer_perceptron.py
@@ -687,7 +687,7 @@ def _fit_stochastic(
                         % self.max_iter,
                         ConvergenceWarning,
                     )
-                    return
+                    break
         except KeyboardInterrupt:
             warnings.warn("Training interrupted by user.")
 

From f854bedbb298f5bab58339d5546d6f9c3411a187 Mon Sep 17 00:00:00 2001
From: Marvvxi <marvin.krawutschke@outlook.de>
Date: Tue, 24 Jan 2023 16:39:01 +0100
Subject: [PATCH 03/11] MLP: Make end condition explicit.

---
 sklearn/neural_network/_multilayer_perceptron.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sklearn/neural_network/_multilayer_perceptron.py b/sklearn/neural_network/_multilayer_perceptron.py
index f950e6d5c649c..80449d9fe0b01 100644
--- a/sklearn/neural_network/_multilayer_perceptron.py
+++ b/sklearn/neural_network/_multilayer_perceptron.py
@@ -607,7 +607,8 @@ def _fit_stochastic(
             batch_size = np.clip(self.batch_size, 1, n_samples)
 
         try:
-            for it in range(self.max_iter):
+            max_iter_remaining = self.max_iter - self.n_iter_
+            for it in range(max_iter_remaining):
                 if self.shuffle:
                     # Only shuffle the sample indices instead of X and y to
                     # reduce the memory footprint. These indices will be used
@@ -687,7 +688,6 @@ def _fit_stochastic(
                         % self.max_iter,
                         ConvergenceWarning,
                     )
-                    break
         except KeyboardInterrupt:
             warnings.warn("Training interrupted by user.")
 

From f249df94df25c24962ee67a7dfe427f66e2d24e0 Mon Sep 17 00:00:00 2001
From: Marvvxi <marvin.krawutschke@outlook.de>
Date: Tue, 24 Jan 2023 16:52:04 +0100
Subject: [PATCH 04/11] MLP: Add test warm start no convergence

---
 sklearn/neural_network/tests/test_mlp.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index a4d4831766170..321901fa98681 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -926,3 +926,22 @@ def test_mlp_warm_start_with_early_stopping(MLPEstimator):
     mlp.set_params(max_iter=20)
     mlp.fit(X_iris, y_iris)
     assert len(mlp.validation_scores_) > n_validation_scores
+
+
+@pytest.mark.parametrize("MLPEstimator", [MLPClassifier, MLPRegressor])
+def test_mlp_warm_start_no_convergence(MLPEstimator):
+    """Check that we stop the number of iteration at `max_iter` when warm starting.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/24764
+    """
+    model = MLPEstimator(warm_start=True, early_stopping=False, max_iter=10)
+
+    with pytest.warns(ConvergenceWarning):
+        model.fit(X_iris, y_iris)
+    assert model.n_iter_ == 10
+
+    model.set_params(max_iter=20)
+    with pytest.warns(ConvergenceWarning):
+        model.fit(X_iris, y_iris)
+    assert model.n_iter_ == 20

From 6dec290fc7c58028809d39ec12771a2db01241b5 Mon Sep 17 00:00:00 2001
From: Marvvxi <marvin.krawutschke@outlook.de>
Date: Wed, 25 Jan 2023 20:11:32 +0100
Subject: [PATCH 05/11] MLP: mlp.learning_rate_init is equal post_eta

---
 sklearn/neural_network/tests/test_mlp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index 321901fa98681..21a082d92c51d 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -354,7 +354,7 @@ def test_learning_rate_warmstart():
         if learning_rate == "constant":
             assert prev_eta == post_eta
         elif learning_rate == "invscaling":
-            assert mlp.learning_rate_init / pow(8 + 1, mlp.power_t) == post_eta
+            assert mlp.learning_rate_init == post_eta
 
 
 def test_multilabel_classification():

From 0d8413674e2053d25732ac29f8ad5a20ba6505dc Mon Sep 17 00:00:00 2001
From: Marvvxi <marvin.krawutschke@outlook.de>
Date: Wed, 25 Jan 2023 20:20:58 +0100
Subject: [PATCH 06/11] MLP: max_iter corresponds to now the real max iter

---
 sklearn/neural_network/tests/test_mlp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index 21a082d92c51d..9b22abd28c325 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -752,7 +752,7 @@ def test_warm_start_full_iteration(MLPEstimator):
     clf.fit(X, y)
     assert max_iter == clf.n_iter_
     clf.fit(X, y)
-    assert 2 * max_iter == clf.n_iter_
+    assert max_iter == clf.n_iter_
 
 
 def test_n_iter_no_change():

From 8415ba07a86a67233ad9dbb8429cccb46768443a Mon Sep 17 00:00:00 2001
From: Marvvxi <marvin.krawutschke@outlook.de>
Date: Wed, 25 Jan 2023 20:27:40 +0100
Subject: [PATCH 07/11] MLP: Decrease relative tolerance, NN was previously
 trained over max iter

---
 sklearn/neural_network/tests/test_mlp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index 9b22abd28c325..d34962f28e71a 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -891,7 +891,7 @@ def test_mlp_loading_from_joblib_partial_fit(tmp_path):
 
     # finetuned model learned the new target
     predicted_value = load_estimator.predict(fine_tune_features)
-    assert_allclose(predicted_value, fine_tune_target, rtol=1e-4)
+    assert_allclose(predicted_value, fine_tune_target, rtol=1e-3)
 
 
 @pytest.mark.parametrize("Estimator", [MLPClassifier, MLPRegressor])

From 35a36aeff538a90d249a644f3b4a213f106178d7 Mon Sep 17 00:00:00 2001
From: Marvvxi <marvin.krawutschke@outlook.de>
Date: Wed, 25 Jan 2023 20:48:54 +0100
Subject: [PATCH 08/11] MLP: update whats new

---
 doc/whats_new/v1.3.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/whats_new/v1.3.rst b/doc/whats_new/v1.3.rst
index a0ea72c23c8a3..377175ef00d5e 100644
--- a/doc/whats_new/v1.3.rst
+++ b/doc/whats_new/v1.3.rst
@@ -151,6 +151,10 @@ Changelog
   no longer raise warnings when fitting data with feature names.
   :pr:`24873` by :user:`Tim Head <betatim>`.
 
+- |Fix| :class:`neural_network.MLPRegressor`
+  MLPEstimator respect `max_iter` when `warm_start` is on.
+  :pr:`25443` by :user:`Marvin Krawutschke <Marvvxi>`.
+
 :mod:`sklearn.pipeline`
 .......................
 

From ba101f2d8c1ae7798f36e2ccafe9f1145a13ed32 Mon Sep 17 00:00:00 2001
From: Guillaume Lemaitre <g.lemaitre58@gmail.com>
Date: Tue, 31 Jan 2023 14:48:28 +0100
Subject: [PATCH 09/11] fixes

---
 doc/whats_new/v1.2.rst                           |  9 +++++++++
 doc/whats_new/v1.3.rst                           |  4 ----
 sklearn/neural_network/_multilayer_perceptron.py |  4 ++--
 sklearn/neural_network/tests/test_mlp.py         | 11 +++++++----
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/doc/whats_new/v1.2.rst b/doc/whats_new/v1.2.rst
index c572acf49370c..723ebf9540604 100644
--- a/doc/whats_new/v1.2.rst
+++ b/doc/whats_new/v1.2.rst
@@ -35,6 +35,15 @@ Changelog
   when the global configuration sets `transform_output="pandas"`.
   :pr:`25500` by :user:`Guillaume Lemaitre <glemaitre>`.
 
+:mod:`sklearn.neural_network`
+............................
+
+- |Fix| :class:`neural_network.MLPRegressor` and :class:`neural_network.MLPClassifier`
+  reports the right `n_iter_` when `warm_start=True`. It corresponds to the number
+  of iterations performed on the current call to `fit` instead of the total number
+  of iterations performed since the initialization of the estimator.
+  :pr:`25443` by :user:`Marvin Krawutschke <Marvvxi>`.
+
 .. _changes_1_2_1:
 
 Version 1.2.1
diff --git a/doc/whats_new/v1.3.rst b/doc/whats_new/v1.3.rst
index b08761dd0425d..3ef8a7653b5f7 100644
--- a/doc/whats_new/v1.3.rst
+++ b/doc/whats_new/v1.3.rst
@@ -169,10 +169,6 @@ Changelog
   dissimilarity is not a metric and cannot be supported by the BallTree.
   :pr:`25417` by :user:`Guillaume Lemaitre <glemaitre>`.
 
-- |Fix| :class:`neural_network.MLPRegressor`
-  MLPEstimator respect `max_iter` when `warm_start` is on.
-  :pr:`25443` by :user:`Marvin Krawutschke <Marvvxi>`.
-
 :mod:`sklearn.pipeline`
 .......................
 
diff --git a/sklearn/neural_network/_multilayer_perceptron.py b/sklearn/neural_network/_multilayer_perceptron.py
index 80449d9fe0b01..2f83468a230f7 100644
--- a/sklearn/neural_network/_multilayer_perceptron.py
+++ b/sklearn/neural_network/_multilayer_perceptron.py
@@ -607,8 +607,8 @@ def _fit_stochastic(
             batch_size = np.clip(self.batch_size, 1, n_samples)
 
         try:
-            max_iter_remaining = self.max_iter - self.n_iter_
-            for it in range(max_iter_remaining):
+            self.n_iter_ = 0
+            for it in range(self.max_iter):
                 if self.shuffle:
                     # Only shuffle the sample indices instead of X and y to
                     # reduce the memory footprint. These indices will be used
diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index d34962f28e71a..6db1f965dad7e 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -354,7 +354,7 @@ def test_learning_rate_warmstart():
         if learning_rate == "constant":
             assert prev_eta == post_eta
         elif learning_rate == "invscaling":
-            assert mlp.learning_rate_init == post_eta
+            assert mlp.learning_rate_init / pow(8 + 1, mlp.power_t) == post_eta
 
 
 def test_multilabel_classification():
@@ -891,7 +891,7 @@ def test_mlp_loading_from_joblib_partial_fit(tmp_path):
 
     # finetuned model learned the new target
     predicted_value = load_estimator.predict(fine_tune_features)
-    assert_allclose(predicted_value, fine_tune_target, rtol=1e-3)
+    assert_allclose(predicted_value, fine_tune_target, rtol=1e-4)
 
 
 @pytest.mark.parametrize("Estimator", [MLPClassifier, MLPRegressor])
@@ -929,13 +929,16 @@ def test_mlp_warm_start_with_early_stopping(MLPEstimator):
 
 
 @pytest.mark.parametrize("MLPEstimator", [MLPClassifier, MLPRegressor])
-def test_mlp_warm_start_no_convergence(MLPEstimator):
+@pytest.mark.parametrize("solver", ["sgd", "adam", "lbfgs"])
+def test_mlp_warm_start_no_convergence(MLPEstimator, solver):
     """Check that we stop the number of iteration at `max_iter` when warm starting.
 
     Non-regression test for:
     https://github.com/scikit-learn/scikit-learn/issues/24764
     """
-    model = MLPEstimator(warm_start=True, early_stopping=False, max_iter=10)
+    model = MLPEstimator(
+        solver=solver, warm_start=True, early_stopping=False, max_iter=10
+    )
 
     with pytest.warns(ConvergenceWarning):
         model.fit(X_iris, y_iris)

From f49c22f91bc3fd157de4b20e7470a514f9c2b56a Mon Sep 17 00:00:00 2001
From: Guillaume Lemaitre <g.lemaitre58@gmail.com>
Date: Tue, 31 Jan 2023 15:18:56 +0100
Subject: [PATCH 10/11] DOC fix sphinx issue

---
 doc/whats_new/v1.2.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/whats_new/v1.2.rst b/doc/whats_new/v1.2.rst
index 723ebf9540604..bc69287886160 100644
--- a/doc/whats_new/v1.2.rst
+++ b/doc/whats_new/v1.2.rst
@@ -36,7 +36,7 @@ Changelog
   :pr:`25500` by :user:`Guillaume Lemaitre <glemaitre>`.
 
 :mod:`sklearn.neural_network`
-............................
+.............................
 
 - |Fix| :class:`neural_network.MLPRegressor` and :class:`neural_network.MLPClassifier`
   reports the right `n_iter_` when `warm_start=True`. It corresponds to the number

From b60445fb722ebf84143d588a42551d62ca6cba49 Mon Sep 17 00:00:00 2001
From: Guillaume Lemaitre <g.lemaitre58@gmail.com>
Date: Mon, 20 Feb 2023 11:42:16 +0100
Subject: [PATCH 11/11] move entry from 1.2 to 1.3

---
 doc/whats_new/v1.2.rst | 9 ---------
 doc/whats_new/v1.3.rst | 9 +++++++++
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/doc/whats_new/v1.2.rst b/doc/whats_new/v1.2.rst
index bc69287886160..c572acf49370c 100644
--- a/doc/whats_new/v1.2.rst
+++ b/doc/whats_new/v1.2.rst
@@ -35,15 +35,6 @@ Changelog
   when the global configuration sets `transform_output="pandas"`.
   :pr:`25500` by :user:`Guillaume Lemaitre <glemaitre>`.
 
-:mod:`sklearn.neural_network`
-.............................
-
-- |Fix| :class:`neural_network.MLPRegressor` and :class:`neural_network.MLPClassifier`
-  reports the right `n_iter_` when `warm_start=True`. It corresponds to the number
-  of iterations performed on the current call to `fit` instead of the total number
-  of iterations performed since the initialization of the estimator.
-  :pr:`25443` by :user:`Marvin Krawutschke <Marvvxi>`.
-
 .. _changes_1_2_1:
 
 Version 1.2.1
diff --git a/doc/whats_new/v1.3.rst b/doc/whats_new/v1.3.rst
index 3ef8a7653b5f7..3fbd26c47dc8c 100644
--- a/doc/whats_new/v1.3.rst
+++ b/doc/whats_new/v1.3.rst
@@ -169,6 +169,15 @@ Changelog
   dissimilarity is not a metric and cannot be supported by the BallTree.
   :pr:`25417` by :user:`Guillaume Lemaitre <glemaitre>`.
 
+:mod:`sklearn.neural_network`
+.............................
+
+- |Fix| :class:`neural_network.MLPRegressor` and :class:`neural_network.MLPClassifier`
+  reports the right `n_iter_` when `warm_start=True`. It corresponds to the number
+  of iterations performed on the current call to `fit` instead of the total number
+  of iterations performed since the initialization of the estimator.
+  :pr:`25443` by :user:`Marvin Krawutschke <Marvvxi>`.
+
 :mod:`sklearn.pipeline`
 .......................