From 71e6d30cab26b5c2d9a7a038c64a1ed7908d8f0f Mon Sep 17 00:00:00 2001
From: Danny Perry <daniel.perry@gmail.com>
Date: Mon, 3 Jul 2017 14:46:34 -0600
Subject: [PATCH 01/19] Setting both maxiter and maxfun in call to lbfgs.

---
 sklearn/neural_network/multilayer_perceptron.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
index ec1196a3e2ac6..dcc8992954848 100644
--- a/sklearn/neural_network/multilayer_perceptron.py
+++ b/sklearn/neural_network/multilayer_perceptron.py
@@ -464,6 +464,7 @@ def _fit_lbfgs(self, X, y, activations, deltas, coef_grads,
             x0=packed_coef_inter,
             func=self._loss_grad_lbfgs,
             maxfun=self.max_iter,
+            maxiter=self.max_iter,
             iprint=iprint,
             pgtol=self.tol,
             args=(X, y, activations, deltas, coef_grads, intercept_grads))

From 646a7fdbc6898dbbf8f9df1700195bf3a3594a49 Mon Sep 17 00:00:00 2001
From: Danny Perry <dperry@sci.utah.edu>
Date: Thu, 7 Sep 2017 14:24:46 -0600
Subject: [PATCH 02/19] [MRG] adding max_fun parameter to MLP

---
 .../neural_network/multilayer_perceptron.py   | 35 +++++++++++++------
 sklearn/neural_network/tests/test_mlp.py      | 26 ++++++++++++++
 2 files changed, 50 insertions(+), 11 deletions(-)

diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
index 29447c7910d05..3631e32a638d7 100644
--- a/sklearn/neural_network/multilayer_perceptron.py
+++ b/sklearn/neural_network/multilayer_perceptron.py
@@ -49,7 +49,7 @@ class BaseMultilayerPerceptron(six.with_metaclass(ABCMeta, BaseEstimator)):
     @abstractmethod
     def __init__(self, hidden_layer_sizes, activation, solver,
                  alpha, batch_size, learning_rate, learning_rate_init, power_t,
-                 max_iter, loss, shuffle, random_state, tol, verbose,
+                 max_iter, max_fun, loss, shuffle, random_state, tol, verbose,
                  warm_start, momentum, nesterovs_momentum, early_stopping,
                  validation_fraction, beta_1, beta_2, epsilon):
         self.activation = activation
@@ -60,6 +60,7 @@ def __init__(self, hidden_layer_sizes, activation, solver,
         self.learning_rate_init = learning_rate_init
         self.power_t = power_t
         self.max_iter = max_iter
+        self.max_fun = max_fun
         self.loss = loss
         self.hidden_layer_sizes = hidden_layer_sizes
         self.shuffle = shuffle
@@ -177,7 +178,6 @@ def _loss_grad_lbfgs(self, packed_coef_inter, X, y, activations, deltas,
         self._unpack(packed_coef_inter)
         loss, coef_grads, intercept_grads = self._backprop(
             X, y, activations, deltas, coef_grads, intercept_grads)
-        self.n_iter_ += 1
         grad = _pack(coef_grads, intercept_grads)
         return loss, grad
 
@@ -389,6 +389,8 @@ def _validate_hyperparameters(self):
                              self.shuffle)
         if self.max_iter <= 0:
             raise ValueError("max_iter must be > 0, got %s." % self.max_iter)
+        if self.max_fun <= 0:
+            raise ValueError("max_fun must be > 0, got %s." % self.max_fun)
         if self.alpha < 0.0:
             raise ValueError("alpha must be >= 0, got %s." % self.alpha)
         if (self.learning_rate in ["constant", "invscaling", "adaptive"] and
@@ -464,11 +466,12 @@ def _fit_lbfgs(self, X, y, activations, deltas, coef_grads,
         optimal_parameters, self.loss_, d = fmin_l_bfgs_b(
             x0=packed_coef_inter,
             func=self._loss_grad_lbfgs,
-            maxfun=self.max_iter,
+            maxfun=self.max_fun,
             maxiter=self.max_iter,
             iprint=iprint,
             pgtol=self.tol,
             args=(X, y, activations, deltas, coef_grads, intercept_grads))
+        self.n_iter_ = d['nit']
 
         self._unpack(optimal_parameters)
 
@@ -769,6 +772,11 @@ class MLPClassifier(BaseMultilayerPerceptron, ClassifierMixin):
         (how many times each data point will be used), not the number of
         gradient steps.
 
+    max_fun : int, optional, default 200
+        Maximum number of function calls. The solver iterates until convergence
+        (determined by 'tol') or this number of function calls.
+        Only used when solver='lbfgs'.
+
     shuffle : bool, optional, default True
         Whether to shuffle samples in each iteration. Only used when
         solver='sgd' or 'adam'.
@@ -887,7 +895,7 @@ def __init__(self, hidden_layer_sizes=(100,), activation="relu",
                  solver='adam', alpha=0.0001,
                  batch_size='auto', learning_rate="constant",
                  learning_rate_init=0.001, power_t=0.5, max_iter=200,
-                 shuffle=True, random_state=None, tol=1e-4,
+                 max_fun=200, shuffle=True, random_state=None, tol=1e-4,
                  verbose=False, warm_start=False, momentum=0.9,
                  nesterovs_momentum=True, early_stopping=False,
                  validation_fraction=0.1, beta_1=0.9, beta_2=0.999,
@@ -898,9 +906,9 @@ def __init__(self, hidden_layer_sizes=(100,), activation="relu",
                      activation=activation, solver=solver, alpha=alpha,
                      batch_size=batch_size, learning_rate=learning_rate,
                      learning_rate_init=learning_rate_init, power_t=power_t,
-                     max_iter=max_iter, loss='log_loss', shuffle=shuffle,
-                     random_state=random_state, tol=tol, verbose=verbose,
-                     warm_start=warm_start, momentum=momentum,
+                     max_iter=max_iter, max_fun=max_fun, loss='log_loss',
+                     shuffle=shuffle, random_state=random_state, tol=tol,
+                     verbose=verbose, warm_start=warm_start, momentum=momentum,
                      nesterovs_momentum=nesterovs_momentum,
                      early_stopping=early_stopping,
                      validation_fraction=validation_fraction,
@@ -1146,6 +1154,11 @@ class MLPRegressor(BaseMultilayerPerceptron, RegressorMixin):
         (how many times each data point will be used), not the number of
         gradient steps.
 
+    max_fun : int, optional, default 200
+        Maximum number of function calls. The solver iterates until convergence
+        (determined by 'tol') or this number of function calls.
+        Only used when solver='lbfgs'.
+
     shuffle : bool, optional, default True
         Whether to shuffle samples in each iteration. Only used when
         solver='sgd' or 'adam'.
@@ -1261,7 +1274,7 @@ def __init__(self, hidden_layer_sizes=(100,), activation="relu",
                  solver='adam', alpha=0.0001,
                  batch_size='auto', learning_rate="constant",
                  learning_rate_init=0.001,
-                 power_t=0.5, max_iter=200, shuffle=True,
+                 power_t=0.5, max_iter=200, max_fun=200, shuffle=True,
                  random_state=None, tol=1e-4,
                  verbose=False, warm_start=False, momentum=0.9,
                  nesterovs_momentum=True, early_stopping=False,
@@ -1273,9 +1286,9 @@ def __init__(self, hidden_layer_sizes=(100,), activation="relu",
                      activation=activation, solver=solver, alpha=alpha,
                      batch_size=batch_size, learning_rate=learning_rate,
                      learning_rate_init=learning_rate_init, power_t=power_t,
-                     max_iter=max_iter, loss='squared_loss', shuffle=shuffle,
-                     random_state=random_state, tol=tol, verbose=verbose,
-                     warm_start=warm_start, momentum=momentum,
+                     max_iter=max_iter, max_fun=max_fun, loss='squared_loss',
+                     shuffle=shuffle, random_state=random_state, tol=tol,
+                     verbose=verbose, warm_start=warm_start, momentum=momentum,
                      nesterovs_momentum=nesterovs_momentum,
                      early_stopping=early_stopping,
                      validation_fraction=validation_fraction,
diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index 9c42b7c930cdf..201944e9dd47e 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -266,6 +266,32 @@ def test_lbfgs_regression():
             # Non linear models perform much better than linear bottleneck:
             assert_greater(mlp.score(X, y), 0.95)
 
+def test_lbfgs_maxfun():
+    # Test lbfgs parameter max_fun
+    # It should independently limit the 
+    # number of iterations for lbfgs
+    max_fun = 10
+    expected_num_iter = 10 
+    for X, y in classification_datasets:
+        X_train = X[:150]
+        y_train = y[:150]
+        X_test = X[150:]
+
+
+        for activation in ACTIVATION_TYPES:
+            mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50,
+                                max_iter=150, max_fun=max_fun, shuffle=True, 
+                                random_state=1, activation=activation)
+            mlp.fit(X_train, y_train)
+            assert_greater(expected_num_iter, mlp.n_iter_)
+    X = Xboston
+    y = yboston 
+    for activation in ACTIVATION_TYPES:
+        mlp = MLPRegressor(solver='lbfgs', hidden_layer_sizes=50,
+                           max_iter=150, max_fun=max_fun, shuffle=True, 
+                           random_state=1, activation=activation)
+        mlp.fit(X, y)
+        assert_greater(expected_num_iter, mlp.n_iter_)
 
 def test_learning_rate_warmstart():
     # Tests that warm_start reuse past solutions.

From 9e30730a218263ebb134ea65d95bb038a95d0a49 Mon Sep 17 00:00:00 2001
From: Danny Perry <dperry@sci.utah.edu>
Date: Fri, 8 Sep 2017 15:22:08 -0600
Subject: [PATCH 03/19] better comments, increasing default max_fun to 15000

---
 .../neural_network/multilayer_perceptron.py    | 18 +++++++++++-------
 sklearn/neural_network/tests/test_mlp.py       |  4 ++--
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
index 3631e32a638d7..d13dab7e6b372 100644
--- a/sklearn/neural_network/multilayer_perceptron.py
+++ b/sklearn/neural_network/multilayer_perceptron.py
@@ -772,10 +772,12 @@ class MLPClassifier(BaseMultilayerPerceptron, ClassifierMixin):
         (how many times each data point will be used), not the number of
         gradient steps.
 
-    max_fun : int, optional, default 200
+    max_fun : int, optional, default 15000
         Maximum number of function calls. The solver iterates until convergence
-        (determined by 'tol') or this number of function calls.
-        Only used when solver='lbfgs'.
+        (determined by 'tol'), number of iterations reaches max_iter, or this
+        number of function calls. Note that number of function calls will be
+        greater than or equal to the number of iterations for the
+        MLPClassifier. Only used when solver='lbfgs'.
 
     shuffle : bool, optional, default True
         Whether to shuffle samples in each iteration. Only used when
@@ -895,7 +897,7 @@ def __init__(self, hidden_layer_sizes=(100,), activation="relu",
                  solver='adam', alpha=0.0001,
                  batch_size='auto', learning_rate="constant",
                  learning_rate_init=0.001, power_t=0.5, max_iter=200,
-                 max_fun=200, shuffle=True, random_state=None, tol=1e-4,
+                 max_fun=15000, shuffle=True, random_state=None, tol=1e-4,
                  verbose=False, warm_start=False, momentum=0.9,
                  nesterovs_momentum=True, early_stopping=False,
                  validation_fraction=0.1, beta_1=0.9, beta_2=0.999,
@@ -1154,9 +1156,11 @@ class MLPRegressor(BaseMultilayerPerceptron, RegressorMixin):
         (how many times each data point will be used), not the number of
         gradient steps.
 
-    max_fun : int, optional, default 200
+    max_fun : int, optional, default 15000
         Maximum number of function calls. The solver iterates until convergence
-        (determined by 'tol') or this number of function calls.
+        (determined by 'tol'), number of iterations reaches max_iter, or this
+        number of function calls. Note that number of function calls will be
+        greater than or equal to the number of iterations for the MLPRegressor.
         Only used when solver='lbfgs'.
 
     shuffle : bool, optional, default True
@@ -1274,7 +1278,7 @@ def __init__(self, hidden_layer_sizes=(100,), activation="relu",
                  solver='adam', alpha=0.0001,
                  batch_size='auto', learning_rate="constant",
                  learning_rate_init=0.001,
-                 power_t=0.5, max_iter=200, max_fun=200, shuffle=True,
+                 power_t=0.5, max_iter=200, max_fun=15000, shuffle=True,
                  random_state=None, tol=1e-4,
                  verbose=False, warm_start=False, momentum=0.9,
                  nesterovs_momentum=True, early_stopping=False,
diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index 201944e9dd47e..e86adcff906cc 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -272,18 +272,18 @@ def test_lbfgs_maxfun():
     # number of iterations for lbfgs
     max_fun = 10
     expected_num_iter = 10 
+    # classification tests
     for X, y in classification_datasets:
         X_train = X[:150]
         y_train = y[:150]
         X_test = X[150:]
-
-
         for activation in ACTIVATION_TYPES:
             mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50,
                                 max_iter=150, max_fun=max_fun, shuffle=True, 
                                 random_state=1, activation=activation)
             mlp.fit(X_train, y_train)
             assert_greater(expected_num_iter, mlp.n_iter_)
+    # regression tests
     X = Xboston
     y = yboston 
     for activation in ACTIVATION_TYPES:

From 71daad8bfb6e9763af50cf3984d3eb8ad12f9576 Mon Sep 17 00:00:00 2001
From: Danny Perry <dperry@sci.utah.edu>
Date: Wed, 18 Oct 2017 15:02:39 -0600
Subject: [PATCH 04/19] better comments, fixing some PEP8 warnings

---
 .../neural_network/multilayer_perceptron.py   | 20 +++++++++----------
 sklearn/neural_network/tests/test_mlp.py      | 18 ++++++++---------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
index d13dab7e6b372..87b799d3f0a84 100644
--- a/sklearn/neural_network/multilayer_perceptron.py
+++ b/sklearn/neural_network/multilayer_perceptron.py
@@ -773,11 +773,11 @@ class MLPClassifier(BaseMultilayerPerceptron, ClassifierMixin):
         gradient steps.
 
     max_fun : int, optional, default 15000
-        Maximum number of function calls. The solver iterates until convergence
-        (determined by 'tol'), number of iterations reaches max_iter, or this
-        number of function calls. Note that number of function calls will be
-        greater than or equal to the number of iterations for the
-        MLPClassifier. Only used when solver='lbfgs'.
+        Only used when solver='lbfgs'. Maximum number of function calls.
+        The solver iterates until convergence (determined by 'tol'), number
+        of iterations reaches max_iter, or this  number of function calls.
+        Note that number of function calls will be greater than or equal to
+        the number of iterations for the MLPClassifier.
 
     shuffle : bool, optional, default True
         Whether to shuffle samples in each iteration. Only used when
@@ -1157,11 +1157,11 @@ class MLPRegressor(BaseMultilayerPerceptron, RegressorMixin):
         gradient steps.
 
     max_fun : int, optional, default 15000
-        Maximum number of function calls. The solver iterates until convergence
-        (determined by 'tol'), number of iterations reaches max_iter, or this
-        number of function calls. Note that number of function calls will be
-        greater than or equal to the number of iterations for the MLPRegressor.
-        Only used when solver='lbfgs'.
+        Only used when solver='lbfgs'. Maximum number of function calls.
+        The solver iterates until convergence (determined by 'tol'), number
+        of iterations reaches max_iter, or this number of function calls.
+        Note that number of function calls will be greater than or equal to
+        the number of iterations for the MLPRegressor.
 
     shuffle : bool, optional, default True
         Whether to shuffle samples in each iteration. Only used when
diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index e86adcff906cc..49d41489f6d91 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -266,12 +266,11 @@ def test_lbfgs_regression():
             # Non linear models perform much better than linear bottleneck:
             assert_greater(mlp.score(X, y), 0.95)
 
+
 def test_lbfgs_maxfun():
-    # Test lbfgs parameter max_fun
-    # It should independently limit the 
-    # number of iterations for lbfgs
+    # Test lbfgs parameter max_fun.
+    # It should independently limit the number of iterations for lbfgs.
     max_fun = 10
-    expected_num_iter = 10 
     # classification tests
     for X, y in classification_datasets:
         X_train = X[:150]
@@ -279,19 +278,20 @@ def test_lbfgs_maxfun():
         X_test = X[150:]
         for activation in ACTIVATION_TYPES:
             mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50,
-                                max_iter=150, max_fun=max_fun, shuffle=True, 
+                                max_iter=150, max_fun=max_fun, shuffle=True,
                                 random_state=1, activation=activation)
             mlp.fit(X_train, y_train)
-            assert_greater(expected_num_iter, mlp.n_iter_)
+            assert_greater(max_fun, mlp.n_iter_)
     # regression tests
     X = Xboston
-    y = yboston 
+    y = yboston
     for activation in ACTIVATION_TYPES:
         mlp = MLPRegressor(solver='lbfgs', hidden_layer_sizes=50,
-                           max_iter=150, max_fun=max_fun, shuffle=True, 
+                           max_iter=150, max_fun=max_fun, shuffle=True,
                            random_state=1, activation=activation)
         mlp.fit(X, y)
-        assert_greater(expected_num_iter, mlp.n_iter_)
+        assert_greater(max_fun, mlp.n_iter_)
+
 
 def test_learning_rate_warmstart():
     # Tests that warm_start reuse past solutions.

From 97268c171e67e1fc7ca71ebc9732e05dd8ef5109 Mon Sep 17 00:00:00 2001
From: Alexandre Gramfort <alexandre.gramfort@m4x.org>
Date: Mon, 25 Feb 2019 14:44:36 +0100
Subject: [PATCH 05/19] test exception

---
 sklearn/neural_network/tests/test_mlp.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index a57011760946c..45c2f0edfafbc 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -293,6 +293,9 @@ def test_lbfgs_maxfun():
         mlp.fit(X, y)
         assert max_fun >= mlp.n_iter_
 
+    mlp.max_fun = -1
+    assert_raises(ValueError, mlp.fit, X, y)
+
 
 def test_learning_rate_warmstart():
     # Tests that warm_start reuse past solutions.

From 3b721b8ebb0953e230bd23beb6bbda650f60f40f Mon Sep 17 00:00:00 2001
From: Alexandre Gramfort <alexandre.gramfort@m4x.org>
Date: Mon, 25 Feb 2019 14:48:56 +0100
Subject: [PATCH 06/19] update what's new

---
 doc/whats_new/v0.21.rst                         | 7 +++++++
 sklearn/neural_network/multilayer_perceptron.py | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/doc/whats_new/v0.21.rst b/doc/whats_new/v0.21.rst
index 36582d834c708..55190e09fd9c3 100644
--- a/doc/whats_new/v0.21.rst
+++ b/doc/whats_new/v0.21.rst
@@ -251,6 +251,13 @@ Support for Python 3.4 and below has been officially dropped.
   :class:`neural_network.MLPRegressor` where the option :code:`shuffle=False`
   was being ignored. :issue:`12582` by :user:`Sam Waterbury <samwaterbury>`.
 
+- |Feature| Add `max_fun` parameter in
+  :class:`neural_network.BaseMultilayerPerceptron`,
+  :class:`neural_network.MLPRegressor`, and
+  :class:`neural_network.MLPClassifier` to give control over
+  maximum number of function evaluation to not meet ``tol`` improvement.
+  :issue:`9274` by :user:`Daniel Perry <daniel-perry>`.
+
 :mod:`sklearn.pipeline`
 .......................
 
diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
index cdcc77be01d96..494b82a7cf502 100644
--- a/sklearn/neural_network/multilayer_perceptron.py
+++ b/sklearn/neural_network/multilayer_perceptron.py
@@ -835,7 +835,7 @@ class MLPClassifier(BaseMultilayerPerceptron, ClassifierMixin):
     max_fun : int, optional, default 15000
         Only used when solver='lbfgs'. Maximum number of function calls.
         The solver iterates until convergence (determined by 'tol'), number
-        of iterations reaches max_iter, or this  number of function calls.
+        of iterations reaches max_iter, or this number of function calls.
         Note that number of function calls will be greater than or equal to
         the number of iterations for the MLPClassifier.
 

From 77e9074d386c89ab56c9bdde1fbc67cd47c7af7e Mon Sep 17 00:00:00 2001
From: Alexandre Gramfort <alexandre.gramfort@m4x.org>
Date: Mon, 25 Feb 2019 17:34:28 +0100
Subject: [PATCH 07/19] doctests

---
 doc/modules/neural_networks_supervised.rst | 32 +++++++++++-----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/doc/modules/neural_networks_supervised.rst b/doc/modules/neural_networks_supervised.rst
index d3e3ac5710cb1..15026af9a4155 100644
--- a/doc/modules/neural_networks_supervised.rst
+++ b/doc/modules/neural_networks_supervised.rst
@@ -90,14 +90,14 @@ training samples::
     ...                     hidden_layer_sizes=(5, 2), random_state=1)
     ...
     >>> clf.fit(X, y)                         # doctest: +NORMALIZE_WHITESPACE
-    MLPClassifier(activation='relu', alpha=1e-05, batch_size='auto',
-                  beta_1=0.9, beta_2=0.999, early_stopping=False,
-                  epsilon=1e-08, hidden_layer_sizes=(5, 2),
-                  learning_rate='constant', learning_rate_init=0.001,
-                  max_iter=200, momentum=0.9, n_iter_no_change=10,
-                  nesterovs_momentum=True, power_t=0.5, random_state=1,
-                  shuffle=True, solver='lbfgs', tol=0.0001,
-                  validation_fraction=0.1, verbose=False, warm_start=False)
+    MLPClassifier(activation='relu', alpha=1e-05, batch_size='auto', beta_1=0.9,
+                  beta_2=0.999, early_stopping=False, epsilon=1e-08,
+                  hidden_layer_sizes=(5, 2), learning_rate='constant',
+                  learning_rate_init=0.001, max_fun=15000, max_iter=200,
+                  momentum=0.9, n_iter_no_change=10, nesterovs_momentum=True,
+                  power_t=0.5, random_state=1, shuffle=True, solver='lbfgs',
+                  tol=0.0001, validation_fraction=0.1, verbose=False,
+                  warm_start=False)
 
 After fitting (training), the model can predict labels for new samples::
 
@@ -139,14 +139,14 @@ indices where the value is `1` represents the assigned classes of that sample::
     ...                     hidden_layer_sizes=(15,), random_state=1)
     ...
     >>> clf.fit(X, y)                         # doctest: +NORMALIZE_WHITESPACE
-    MLPClassifier(activation='relu', alpha=1e-05, batch_size='auto',
-                  beta_1=0.9, beta_2=0.999, early_stopping=False,
-                  epsilon=1e-08, hidden_layer_sizes=(15,),
-                  learning_rate='constant', learning_rate_init=0.001,
-                  max_iter=200, momentum=0.9, n_iter_no_change=10,
-                  nesterovs_momentum=True, power_t=0.5,  random_state=1,
-                  shuffle=True, solver='lbfgs', tol=0.0001,
-                  validation_fraction=0.1, verbose=False, warm_start=False)
+    MLPClassifier(activation='relu', alpha=1e-05, batch_size='auto', beta_1=0.9,
+                  beta_2=0.999, early_stopping=False, epsilon=1e-08,
+                  hidden_layer_sizes=(15,), learning_rate='constant',
+                  learning_rate_init=0.001, max_fun=15000, max_iter=200,
+                  momentum=0.9, n_iter_no_change=10, nesterovs_momentum=True,
+                  power_t=0.5, random_state=1, shuffle=True, solver='lbfgs',
+                  tol=0.0001, validation_fraction=0.1, verbose=False,
+                  warm_start=False)
     >>> clf.predict([[1., 2.]])
     array([[1, 1]])
     >>> clf.predict([[0., 0.]])

From 06d1312504d317f9a20d49867c7b13fec5b0e4a7 Mon Sep 17 00:00:00 2001
From: Alexandre Gramfort <alexandre.gramfort@m4x.org>
Date: Tue, 26 Feb 2019 12:49:14 +0100
Subject: [PATCH 08/19] lint

---
 sklearn/neural_network/tests/test_mlp.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index 45c2f0edfafbc..7c979e1a1faea 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -273,14 +273,11 @@ def test_lbfgs_maxfun():
     max_fun = 10
     # classification tests
     for X, y in classification_datasets:
-        X_train = X[:150]
-        y_train = y[:150]
-        X_test = X[150:]
         for activation in ACTIVATION_TYPES:
             mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50,
                                 max_iter=150, max_fun=max_fun, shuffle=True,
                                 random_state=1, activation=activation)
-            mlp.fit(X_train, y_train)
+            mlp.fit(X, y)
             assert max_fun >= mlp.n_iter_
 
     # regression tests

From 8a276834e8c0af30605c5e7dae2dff59554c363e Mon Sep 17 00:00:00 2001
From: Danny Perry <daniel.perry@gmail.com>
Date: Sat, 11 May 2019 20:53:27 -0700
Subject: [PATCH 09/19] clarifying comments about function call

---
 sklearn/neural_network/multilayer_perceptron.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
index 494b82a7cf502..583d7684ec442 100644
--- a/sklearn/neural_network/multilayer_perceptron.py
+++ b/sklearn/neural_network/multilayer_perceptron.py
@@ -833,10 +833,10 @@ class MLPClassifier(BaseMultilayerPerceptron, ClassifierMixin):
         .. versionadded:: 0.20
 
     max_fun : int, optional, default 15000
-        Only used when solver='lbfgs'. Maximum number of function calls.
+        Only used when solver='lbfgs'. Maximum number of loss function calls.
         The solver iterates until convergence (determined by 'tol'), number
-        of iterations reaches max_iter, or this number of function calls.
-        Note that number of function calls will be greater than or equal to
+        of iterations reaches max_iter, or this number of loss function calls.
+        Note that number of loss function calls will be greater than or equal to
         the number of iterations for the MLPClassifier.
 
         .. versionadded:: 0.21

From 7864227adfbb5d91754459c271c1f9355e90217c Mon Sep 17 00:00:00 2001
From: Danny Perry <daniel.perry@gmail.com>
Date: Sat, 11 May 2019 21:10:19 -0700
Subject: [PATCH 10/19] fixing PEP8 errors

---
 sklearn/neural_network/multilayer_perceptron.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
index 583d7684ec442..de7af3b9cd704 100644
--- a/sklearn/neural_network/multilayer_perceptron.py
+++ b/sklearn/neural_network/multilayer_perceptron.py
@@ -836,8 +836,8 @@ class MLPClassifier(BaseMultilayerPerceptron, ClassifierMixin):
         Only used when solver='lbfgs'. Maximum number of loss function calls.
         The solver iterates until convergence (determined by 'tol'), number
         of iterations reaches max_iter, or this number of loss function calls.
-        Note that number of loss function calls will be greater than or equal to
-        the number of iterations for the MLPClassifier.
+        Note that number of loss function calls will be greater than or equal
+        to the number of iterations for the MLPClassifier.
 
         .. versionadded:: 0.21
 

From 60e5b0757129fda173cd64d018ccb2a3e06902df Mon Sep 17 00:00:00 2001
From: Danny Perry <daniel.perry@gmail.com>
Date: Tue, 14 May 2019 21:39:03 -0700
Subject: [PATCH 11/19] moving datasets to @pytest.mark.parameterize

---
 sklearn/neural_network/tests/test_mlp.py | 66 +++++++++++++-----------
 1 file changed, 35 insertions(+), 31 deletions(-)

diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index e3feedd82cb24..35f490c3ab90f 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -49,6 +49,8 @@
 Xboston = StandardScaler().fit_transform(boston.data)[: 200]
 yboston = boston.target[:200]
 
+regression_datasets = [(Xboston, yboston)]
+
 iris = load_iris()
 
 X_iris = iris.data
@@ -228,33 +230,31 @@ def loss_grad_fun(t):
                               (epsilon * 2.0))
             assert_almost_equal(numgrad, grad)
 
-
-def test_lbfgs_classification():
+@pytest.mark.parametrize('X,y', classification_datasets)
+def test_lbfgs_classification(X, y):
     # Test lbfgs on classification.
     # It should achieve a score higher than 0.95 for the binary and multi-class
     # versions of the digits dataset.
-    for X, y in classification_datasets:
-        X_train = X[:150]
-        y_train = y[:150]
-        X_test = X[150:]
-
-        expected_shape_dtype = (X_test.shape[0], y_train.dtype.kind)
-
-        for activation in ACTIVATION_TYPES:
-            mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50,
-                                max_iter=150, shuffle=True, random_state=1,
-                                activation=activation)
-            mlp.fit(X_train, y_train)
-            y_predict = mlp.predict(X_test)
-            assert_greater(mlp.score(X_train, y_train), 0.95)
-            assert_equal((y_predict.shape[0], y_predict.dtype.kind),
-                         expected_shape_dtype)
+    X_train = X[:150]
+    y_train = y[:150]
+    X_test = X[150:]
 
+    expected_shape_dtype = (X_test.shape[0], y_train.dtype.kind)
 
-def test_lbfgs_regression():
+    for activation in ACTIVATION_TYPES:
+        mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50,
+                            max_iter=150, shuffle=True, random_state=1,
+                            activation=activation)
+        mlp.fit(X_train, y_train)
+        y_predict = mlp.predict(X_test)
+        assert_greater(mlp.score(X_train, y_train), 0.95)
+        assert_equal((y_predict.shape[0], y_predict.dtype.kind),
+                      expected_shape_dtype)
+
+
+@pytest.mark.parametrize('X,y', regression_datasets)
+def test_lbfgs_regression(X, y):
     # Test lbfgs on the boston dataset, a regression problems.
-    X = Xboston
-    y = yboston
     for activation in ACTIVATION_TYPES:
         mlp = MLPRegressor(solver='lbfgs', hidden_layer_sizes=50,
                            max_iter=150, shuffle=True, random_state=1,
@@ -267,22 +267,26 @@ def test_lbfgs_regression():
             assert_greater(mlp.score(X, y), 0.95)
 
 
-def test_lbfgs_maxfun():
+@pytest.mark.parametrize('X,y', classification_datasets)
+def test_lbfgs_classification_maxfun(X, y):
     # Test lbfgs parameter max_fun.
     # It should independently limit the number of iterations for lbfgs.
     max_fun = 10
     # classification tests
-    for X, y in classification_datasets:
-        for activation in ACTIVATION_TYPES:
-            mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50,
-                                max_iter=150, max_fun=max_fun, shuffle=True,
-                                random_state=1, activation=activation)
-            mlp.fit(X, y)
-            assert max_fun >= mlp.n_iter_
+    for activation in ACTIVATION_TYPES:
+        mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50,
+                            max_iter=150, max_fun=max_fun, shuffle=True,
+                            random_state=1, activation=activation)
+        mlp.fit(X, y)
+        assert max_fun >= mlp.n_iter_
+
 
+@pytest.mark.parametrize('X,y', regression_datasets)
+def test_lbfgs_regression_maxfun(X, y):
+    # Test lbfgs parameter max_fun.
+    # It should independently limit the number of iterations for lbfgs.
+    max_fun = 10
     # regression tests
-    X = Xboston
-    y = yboston
     for activation in ACTIVATION_TYPES:
         mlp = MLPRegressor(solver='lbfgs', hidden_layer_sizes=50,
                            max_iter=150, max_fun=max_fun, shuffle=True,

From e056c9ccc1da1e9bef78c2d933c4ce76e8de9159 Mon Sep 17 00:00:00 2001
From: Danny Perry <daniel.perry@gmail.com>
Date: Tue, 14 May 2019 21:42:21 -0700
Subject: [PATCH 12/19] fixing PEP8 errors.

---
 sklearn/neural_network/tests/test_mlp.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index 35f490c3ab90f..84bec361e29b5 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -230,6 +230,7 @@ def loss_grad_fun(t):
                               (epsilon * 2.0))
             assert_almost_equal(numgrad, grad)
 
+
 @pytest.mark.parametrize('X,y', classification_datasets)
 def test_lbfgs_classification(X, y):
     # Test lbfgs on classification.
@@ -249,7 +250,7 @@ def test_lbfgs_classification(X, y):
         y_predict = mlp.predict(X_test)
         assert_greater(mlp.score(X_train, y_train), 0.95)
         assert_equal((y_predict.shape[0], y_predict.dtype.kind),
-                      expected_shape_dtype)
+                     expected_shape_dtype)
 
 
 @pytest.mark.parametrize('X,y', regression_datasets)

From 8084f405888172d89e26bb7ed2abd915d98215ae Mon Sep 17 00:00:00 2001
From: Daniel Perry <daniel-perry@users.noreply.github.com>
Date: Thu, 27 Jun 2019 09:10:04 -0700
Subject: [PATCH 13/19] Update sklearn/neural_network/multilayer_perceptron.py

Co-Authored-By: Adrin Jalali <adrin.jalali@gmail.com>
---
 sklearn/neural_network/multilayer_perceptron.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
index 4ec82f1150a58..9fdfe99f1a4f9 100644
--- a/sklearn/neural_network/multilayer_perceptron.py
+++ b/sklearn/neural_network/multilayer_perceptron.py
@@ -842,7 +842,7 @@ class MLPClassifier(BaseMultilayerPerceptron, ClassifierMixin):
         The solver iterates until convergence (determined by 'tol'), number
         of iterations reaches max_iter, or this number of loss function calls.
         Note that number of loss function calls will be greater than or equal
-        to the number of iterations for the MLPClassifier.
+        to the number of iterations for the `MLPClassifier`.
 
         .. versionadded:: 0.21
 

From 772a8281cc5dac86177a0b9c4187eec3cd41e395 Mon Sep 17 00:00:00 2001
From: Daniel Perry <daniel-perry@users.noreply.github.com>
Date: Thu, 27 Jun 2019 09:10:19 -0700
Subject: [PATCH 14/19] Update sklearn/neural_network/multilayer_perceptron.py

Co-Authored-By: Adrin Jalali <adrin.jalali@gmail.com>
---
 sklearn/neural_network/multilayer_perceptron.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
index 9fdfe99f1a4f9..1274d400e38b5 100644
--- a/sklearn/neural_network/multilayer_perceptron.py
+++ b/sklearn/neural_network/multilayer_perceptron.py
@@ -844,7 +844,7 @@ class MLPClassifier(BaseMultilayerPerceptron, ClassifierMixin):
         Note that number of loss function calls will be greater than or equal
         to the number of iterations for the `MLPClassifier`.
 
-        .. versionadded:: 0.21
+        .. versionadded:: 0.22
 
     Attributes
     ----------

From 211dbb131a1e5f30c55165444a0be741624607c0 Mon Sep 17 00:00:00 2001
From: Daniel Perry <daniel-perry@users.noreply.github.com>
Date: Thu, 27 Jun 2019 09:10:34 -0700
Subject: [PATCH 15/19] Update sklearn/neural_network/multilayer_perceptron.py

Co-Authored-By: Adrin Jalali <adrin.jalali@gmail.com>
---
 sklearn/neural_network/multilayer_perceptron.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
index 1274d400e38b5..8b1caaf58351b 100644
--- a/sklearn/neural_network/multilayer_perceptron.py
+++ b/sklearn/neural_network/multilayer_perceptron.py
@@ -1235,7 +1235,7 @@ class MLPRegressor(BaseMultilayerPerceptron, RegressorMixin):
         Note that number of function calls will be greater than or equal to
         the number of iterations for the MLPRegressor.
 
-        .. versionadded:: 0.21
+        .. versionadded:: 0.22
 
     Attributes
     ----------

From f65e2914ebc60c48d686562d37c10a1e71c7b177 Mon Sep 17 00:00:00 2001
From: Danny Perry <daniel.perry@gmail.com>
Date: Thu, 27 Jun 2019 09:19:17 -0700
Subject: [PATCH 16/19] moving feature announcement to v0.22.rst

---
 doc/whats_new/v0.21.rst |  7 -------
 doc/whats_new/v0.22.rst | 12 ++++++++++++
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/doc/whats_new/v0.21.rst b/doc/whats_new/v0.21.rst
index 41a1e2477f190..95f07bb911383 100644
--- a/doc/whats_new/v0.21.rst
+++ b/doc/whats_new/v0.21.rst
@@ -727,13 +727,6 @@ Support for Python 3.4 and below has been officially dropped.
   the multilabel case however, splits are still not stratified.
   :pr:`13164` by :user:`Nicolas Hug<NicolasHug>`.
 
-- |Feature| Add `max_fun` parameter in
-  :class:`neural_network.BaseMultilayerPerceptron`,
-  :class:`neural_network.MLPRegressor`, and
-  :class:`neural_network.MLPClassifier` to give control over
-  maximum number of function evaluation to not meet ``tol`` improvement.
-  :issue:`9274` by :user:`Daniel Perry <daniel-perry>`.
-
 :mod:`sklearn.pipeline`
 .......................
 
diff --git a/doc/whats_new/v0.22.rst b/doc/whats_new/v0.22.rst
index ac803d87a4524..fd2ddbc2cbb89 100644
--- a/doc/whats_new/v0.22.rst
+++ b/doc/whats_new/v0.22.rst
@@ -48,6 +48,18 @@ Changelog
   ``decision_function_shape='ovr'``, and the number of target classes > 2.
   :pr:`12557` by `Adrin Jalali`_.
 
+
+:mod:`sklearn.neural_network`
+.............................
+
+- |Feature| Add `max_fun` parameter in
+  :class:`neural_network.BaseMultilayerPerceptron`,
+  :class:`neural_network.MLPRegressor`, and
+  :class:`neural_network.MLPClassifier` to give control over
+  maximum number of function evaluation to not meet ``tol`` improvement.
+  :issue:`9274` by :user:`Daniel Perry <daniel-perry>`.
+
+
 Miscellaneous
 .............
 

From 28e73c9586b27d561412905a3984c615236c9202 Mon Sep 17 00:00:00 2001
From: Danny Perry <daniel.perry@gmail.com>
Date: Fri, 28 Jun 2019 09:26:45 -0700
Subject: [PATCH 17/19] addressing review comments.

---
 doc/whats_new/v0.22.rst | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/whats_new/v0.22.rst b/doc/whats_new/v0.22.rst
index c3e58419c216f..8269f336bbe57 100644
--- a/doc/whats_new/v0.22.rst
+++ b/doc/whats_new/v0.22.rst
@@ -19,8 +19,6 @@ random sampling procedures.
 
 - :class:`decomposition.SparsePCA` where `normalize_components` has no effect
   due to deprecation.
-..
-    TO FILL IN AS WE GO
 
 Details are listed in the changelog below.
 

From e11ca94ec4d43d9a1c97082491ae1b0fd89fb818 Mon Sep 17 00:00:00 2001
From: Danny Perry <daniel.perry@gmail.com>
Date: Fri, 28 Jun 2019 09:59:19 -0700
Subject: [PATCH 18/19] adding convergence warnings

---
 sklearn/neural_network/multilayer_perceptron.py | 17 +++++++++++++++++
 sklearn/neural_network/tests/test_mlp.py        | 10 ++++++----
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/sklearn/neural_network/multilayer_perceptron.py b/sklearn/neural_network/multilayer_perceptron.py
index 8b1caaf58351b..e5325ecda69f0 100644
--- a/sklearn/neural_network/multilayer_perceptron.py
+++ b/sklearn/neural_network/multilayer_perceptron.py
@@ -467,6 +467,23 @@ def _fit_lbfgs(self, X, y, activations, deltas, coef_grads,
             pgtol=self.tol,
             args=(X, y, activations, deltas, coef_grads, intercept_grads))
         self.n_iter_ = d['nit']
+        if d['warnflag'] == 1:
+            if d['nit'] >= self.max_iter:
+                warnings.warn(
+                    "LBFGS Optimizer: Maximum iterations (%d) "
+                    "reached and the optimization hasn't converged yet."
+                    % self.max_iter, ConvergenceWarning)
+            if d['funcalls'] >= self.max_fun:
+                warnings.warn(
+                    "LBFGS Optimizer: Maximum function evaluations (%d) "
+                    "reached and the optimization hasn't converged yet."
+                    % self.max_fun, ConvergenceWarning)
+        elif d['warnflag'] == 2:
+            warnings.warn(
+                "LBFGS Optimizer: Optimization hasn't converged yet, "
+                "cause of LBFGS stopping: %s."
+                % d['task'], ConvergenceWarning)
+
 
         self._unpack(optimal_parameters)
 
diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index 84bec361e29b5..acbcdef31a0a5 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -278,8 +278,9 @@ def test_lbfgs_classification_maxfun(X, y):
         mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50,
                             max_iter=150, max_fun=max_fun, shuffle=True,
                             random_state=1, activation=activation)
-        mlp.fit(X, y)
-        assert max_fun >= mlp.n_iter_
+        with pytest.warns(ConvergenceWarning):
+            mlp.fit(X, y)
+            assert max_fun >= mlp.n_iter_
 
 
 @pytest.mark.parametrize('X,y', regression_datasets)
@@ -292,8 +293,9 @@ def test_lbfgs_regression_maxfun(X, y):
         mlp = MLPRegressor(solver='lbfgs', hidden_layer_sizes=50,
                            max_iter=150, max_fun=max_fun, shuffle=True,
                            random_state=1, activation=activation)
-        mlp.fit(X, y)
-        assert max_fun >= mlp.n_iter_
+        with pytest.warns(ConvergenceWarning):
+            mlp.fit(X, y)
+            assert max_fun >= mlp.n_iter_
 
     mlp.max_fun = -1
     assert_raises(ValueError, mlp.fit, X, y)

From 72e38d19aad53eccc7dee07f5ec7a11d83884974 Mon Sep 17 00:00:00 2001
From: Roman Yurchak <rth.yurchak@pm.me>
Date: Tue, 2 Jul 2019 10:19:59 +0200
Subject: [PATCH 19/19] Lint

---
 sklearn/neural_network/tests/test_mlp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index eea003a285228..af646cbbd5432 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -246,9 +246,9 @@ def test_lbfgs_classification(X, y):
                             activation=activation)
         mlp.fit(X_train, y_train)
         y_predict = mlp.predict(X_test)
-        assert mlp.score(X_train, y_train) > 0.95)
+        assert mlp.score(X_train, y_train) > 0.95
         assert ((y_predict.shape[0], y_predict.dtype.kind) ==
-                     expected_shape_dtype)
+                expected_shape_dtype)
 
 
 @pytest.mark.parametrize('X,y', regression_datasets)