8000 [3.12] gh-113964: Don't prevent new threads until all non-daemon thre… · python/cpython@9256433 · GitHub
[go: up one dir, main page]
Skip to content

Commit 9256433

Browse files
colesburycolesbury
authored
[3.12] gh-113964: Don't prevent new threads until all non-daemon threads exit (GH-116677) (#117029)
Starting in Python 3.12, we prevented calling fork() and starting new threads during interpreter finalization (shutdown). This has led to a number of regressions and flaky tests. We should not prevent starting new threads (or `fork()`) until all non-daemon threads exit and finalization starts in earnest. This changes the checks to use `_PyInterpreterState_GetFinalizing(interp)`, which is set immediately before terminating non-daemon threads. (cherry picked from commit 60e105c)
1 parent 4be9fa8 commit 9256433

File tree

8 files changed

+57
-27
lines changed

8 files changed

+57
-27
lines changed

‎Lib/test/test_os.py

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4840,20 +4840,21 @@ def test_fork_warns_when_non_python_thread_exists(self):
48404840
self.assertEqual(err.decode("utf-8"), "")
48414841
self.assertEqual(out.decode("utf-8"), "")
48424842

4843-
def test_fork_at_exit(self):
4843+
def test_fork_at_finalization(self):
48444844
code = """if 1:
48454845
import atexit
48464846
import os
48474847
4848-
def exit_handler():
4849-
pid = os.fork()
4850-
if pid != 0:
4851-
print("shouldn't be printed")
4852-
4853-
atexit.register(exit_handler)
4848+
class AtFinalization:
4849+
def __del__(self):
4850+
print("OK")
4851+
pid = os.fork()
4852+
if pid != 0:
4853+
print("shouldn't be printed")
4854+
at_finalization = AtFinalization()
48544855
"""
48554856
_, out, err = assert_python_ok("-c", code)
4856-
self.assertEqual(b"", out)
4857+
self.assertEqual(b"OK\n", out)
48574858
self.assertIn(b"can't fork at interpreter shutdown", err)
48584859

48594860

‎Lib/test/test_subprocess.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3413,14 +3413,15 @@ def test_preexec_at_exit(self):
34133413
def dummy():
34143414
pass
34153415
3416-
def exit_handler():
3417-
subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy)
3418-
print("shouldn't be printed")
3419-
3420-
atexit.register(exit_handler)
3416+
class AtFinalization:
3417+
def __del__(self):
3418+
print("OK")
3419+
subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy)
3420+
print("shouldn't be printed")
3421+
at_finalization = AtFinalization()
34213422
"""
34223423
_, out, err = assert_python_ok("-c", code)
3423-
self.assertEqual(out, b'')
3424+
self.assertEqual(out.strip(), b"OK")
34243425
self.assertIn(b"preexec_fn not supported at interpreter shutdown", err)
34253426

34263427

‎Lib/test/test_threading.py

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1133,21 +1133,21 @@ def import_threading():
11331133
self.assertEqual(out, b'')
11341134
self.assertEqual(err, b'')
11351135

1136-
def test_start_new_thread_at_exit(self):
1136+
def test_start_new_thread_at_finalization(self):
11371137
code = """if 1:
1138-
import atexit
11391138
import _thread
11401139
11411140
def f():
11421141
print("shouldn't be printed")
11431142
1144-
def exit_handler():
1145-
_thread.start_new_thread(f, ())
1146-
1147-
atexit.register(exit_handler)
1143+
class AtFinalization:
1144+
def __del__(self):
1145+
print("OK")
1146+
_thread.start_new_thread(f, ())
1147+
at_finalization = AtFinalization()
11481148
"""
11491149
_, out, err = assert_python_ok("-c", code)
1150-
self.assertEqual(out, b'')
1150+
self.assertEqual(out.strip(), b"OK")
11511151
self.assertIn(b"can't create new thread at interpreter shutdown", err)
11521152

11531153
class ThreadJoinOnShutdown(BaseTestCase):
@@ -1276,6 +1276,30 @@ def main():
12761276
rc, out, err = assert_python_ok('-c', script)
12771277
self.assertFalse(err)
12781278

1279+
def test_thread_from_thread(self):
1280+
script = """if True:
1281+
import threading
1282+
import time
1283+
1284+
def thread2():
1285+
time.sleep(0.05)
1286+
print("OK")
1287+
1288+
def thread1():
1289+
time.sleep(0.05)
1290+
t2 = threading.Thread(target=thread2)
1291+
t2.start()
1292+
1293+
t = threading.Thread(target=thread1)
1294+
t.start()
1295+
# do not join() -- the interpreter waits for non-daemon threads to
1296+
# finish.
1297+
"""
1298+
rc, out, err = assert_python_ok('-c', script)
1299+
self.assertEqual(err, b"")
1300+
self.assertEqual(out.strip(), b"OK")
1301+
self.assertEqual(rc, 0)
1302+
12791303
@skip_unless_reliable_fork
12801304
def test_reinit_tls_after_fork(self):
12811305
# Issue #13817: fork() would deadlock in a multithreaded program with

‎Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Starting new threads and process creation through :func:`os.fork` are now
2+
only prevented once all non-daemon threads exit.

‎Modules/_posixsubprocess.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -946,7 +946,9 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args,
946946
Py_ssize_t fds_to_keep_len = PyTuple_GET_SIZE(py_fds_to_keep);
947947

948948
PyInterpreterState *interp = PyInterpreterState_Get();
949-
if ((preexec_fn != Py_None) && interp->finalizing) {
949+
if ((preexec_fn != Py_None) &&
950+
_PyInterpreterState_GetFinalizing(interp) != NULL)
951+
{
950952
PyErr_SetString(PyExc_RuntimeError,
951953
"preexec_fn not supported at interpreter shutdown");
952954
return NULL;

‎Modules/_threadmodule.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1190,7 +1190,7 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs)
11901190
"thread is not supported for isolated subinterpreters");
11911191
return NULL;
11921192
}
1193-
if (interp->finalizing) {
1193+
if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
11941194
PyErr_SetString(PyExc_RuntimeError,
11951195
"can't create new thread at interpreter shutdown");
11961196
return NULL;

‎Modules/posixmodule.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7663,7 +7663,7 @@ os_fork1_impl(PyObject *module)
76637663
pid_t pid;
76647664

76657665
PyInterpreterState *interp = _PyInterpreterState_GET();
7666-
if (interp->finalizing) {
7666+
if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
76677667
PyErr_SetString(PyExc_RuntimeError,
76687668
"can't fork at interpreter shutdown");
76697669
return NULL;
@@ -7707,7 +7707,7 @@ os_fork_impl(PyObject *module)
77077707
{
77087708
pid_t pid;
77097709
PyInterpreterState *interp = _PyInterpreterState_GET();
7710-
if (interp->finalizing) {
7710+
if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
77117711
PyErr_SetString(PyExc_RuntimeError,
77127712
"can't fork at interpreter shutdown");
77137713
return NULL;
@@ -8391,7 +8391,7 @@ os_forkpty_impl(PyObject *module)
83918391
pid_t pid;
83928392

83938393
PyInterpreterState *interp = _PyInterpreterState_GET();
8394-
if (interp->finalizing) {
8394+
if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
83958395
PyErr_SetString(PyExc_RuntimeError,
83968396
"can't fork at interpreter shutdown");
83978397
return NULL;

‎Objects/unicodeobject.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,7 @@ unicode_check_encoding_errors(const char *encoding, const char *errors)
516516

517517
/* Disable checks during Python finalization. For example, it allows to
518518
call _PyObject_Dump() during finalization for debugging purpose. */
519-
if (interp->finalizing) {
519+
if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
520520
return 0;
521521
}
522522

0 commit comments

Comments
 (0)
0