Prevent max_tasks_per_child use with a fork mp_context.

Also defaults to a spawn context when no mp_context is supplied for safe convenience.
python · gpshead · May 6, 2022 · Apr 16, 2022 · Apr 17, 2022 · Apr 17, 2022
commit 34f913eb5e3b93657eab5f74db04c4415d373792
diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst
@@ -254,8 +254,11 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
 
    *max_tasks_per_child* is an optional argument that specifies the maximum
    number of tasks a single process can execute before it will exit and be
-   replaced with a fresh worker process. The default *max_tasks_per_child* is
-   ``None`` which means worker processes will live as long as the pool.
+   replaced with a 
8000
fresh worker process. By default *max_tasks_per_child* is
+   ``None`` which means worker processes will live as long as the pool. When
+   a max is specified, the "spawn" multiprocessing start method will be used by
+   default in absense of a *mp_context* parameter. This feature is incompatible
+   with the "fork" start method.
 
    .. versionchanged:: 3.3
       When one of the worker processes terminates abruptly, a

diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
@@ -614,14 +614,16 @@ def __init__(self, max_workers=None, mp_context=None,
                 execute the given calls. If None or not given then as many
                 worker processes will be created as the machine has processors.
             mp_context: A multiprocessing context to launch the workers. This
-                object should provide SimpleQueue, Queue and Process.
+                object should provide SimpleQueue, Queue and Process. Useful
+                to allow specific multiprocessing start methods.
             initializer: A callable used to initialize worker processes.
             initargs: A tuple of arguments to pass to the initializer.
-            max_tasks_per_child: The maximum number of tasks a worker process can
-                complete before it will exit and be replaced with a fresh
-                worker process, to enable unused resources to be freed. The
-                default value is None, which means worker process will live
-                as long as the executor will live.
+            max_tasks_per_child: The maximum number of tasks a worker process
+                can complete before it will exit and be replaced with a fresh
+                worker process. The default of None means worker process will
+                live as long as the executor. Requires a non-'fork' mp_context
+                start method. When given, we default to using 'spawn' if no
+                mp_context is supplied.
         """
         _check_system_limits()
 
@@ -641,7 +643,10 @@ def __init__(self, max_workers=None, mp_context=None,
             self._max_workers = max_workers
 
         if mp_context is None:
-            mp_context = mp.get_context()
+            if max_tasks_per_child is not None:
+                mp_context = mp.get_context("spawn")
+            else:
+                mp_context = mp.get_context()
         self._mp_context = mp_context
 
         if initializer is not None and not callable(initializer):
@@ -654,6 +659,11 @@ def __init__(self, max_workers=None, mp_context=None,
                 raise TypeError("max_tasks_per_child must be an integer")
             elif max_tasks_per_child <= 0:
                 raise ValueError("max_tasks_per_child must be >= 1")
+            if self._mp_context.get_start_method(allow_none=False) == "fork":
+                # https://github.com/python/cpython/issues/90622
+                raise ValueError("max_tasks_per_child is incompatible with"
+                                 " the 'fork' multiprocessing start method;"
+                                 " supply a different mp_context.")
         self._max_tasks_per_child = max_tasks_per_child
 
         # Management thread

diff --git a/Misc/NEWS.d/next/Library/2022-04-15-18-32-38.gh-issue-90622.WQjFDe.rst b/Misc/NEWS.d/next/Library/2022-04-15-18-32-38.gh-issue-90622.WQjFDe.rst
@@ -0,0 +1,5 @@
+In ``concurrent.futures.process.ProcessPoolExecutor`` disallow the "fork"
+multiprocessing start method when the new ``max_tasks_per_child`` feature is
+used as the mix of threads+fork can hang the child processes. Default to
+using the safe "spawn" start method in that circumstance if no
+``mp_context`` was supplied.