8000 [3.6] bpo-30697: Fix PyErr_NormalizeException() when no memory (GH-23… · python/cpython@4b27d51 · GitHub
[go: up one dir, main page]
Skip to content

Commit 4b27d51

Browse files
xdegayexdegaye
authored
[3.6] bpo-30697: Fix PyErr_NormalizeException() when no memory (GH-2327). (#4135)
(cherry picked from commit 56d1f5c)
1 parent d94ef8f commit 4b27d51

File tree

8 files changed

+203
-53
lines changed

8 files changed

+203
-53
lines changed

‎Doc/whatsnew/3.6.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1852,6 +1852,11 @@ Build and C API Changes
18521852
* The :c:func:`PyUnicode_FSConverter` and :c:func:`PyUnicode_FSDecoder`
18531853
functions will now accept :term:`path-like objects <path-like object>`.
18541854

1855+
* The ``PyExc_RecursionErrorInst`` singleton that was part of the public API
1856+
has been removed as its members being never cleared may cause a segfault
1857+
during finalization of the interpreter. Contributed by Xavier de Gaye in
1858+
:issue:`22898` and :issue:`30697`.
1859+
18551860

18561861
Other Improvements
18571862
==================

‎Include/pyerrors.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,8 +219,6 @@ PyAPI_DATA(PyObject *) PyExc_IOError;
219219
PyAPI_DATA(PyObject *) PyExc_WindowsError;
220220
#endif
221221

222-
PyAPI_DATA(PyObject *) PyExc_RecursionErrorInst;
223-
224222
/* Predefined warning categories */
225223
PyAPI_DATA(PyObject *) PyExc_Warning;
226224
PyAPI_DATA(PyObject *) PyExc_UserWarning;

‎Lib/test/test_exceptions.py

Lines changed: 101 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010

1111
from test.support import (TESTFN, captured_stderr, check_impl_detail,
1212
check_warnings, cpython_only, gc_collect, run_unittest,
13-
no_tracing, unlink, import_module, script_helper)
14-
13+
no_tracing, unlink, import_module, script_helper,
14+
SuppressCrashReport)
1515
class NaiveException(Exception):
1616
def __init__(self, x):
1717
self.x = x
@@ -936,6 +936,105 @@ def g():
936936
self.assertTrue(isinstance(v, RecursionError), type(v))
937937
self.assertIn("maximum recursion depth exceeded", str(v))
938938

939+
@cpython_only
940+
def test_recursion_normalizing_exception(self):
941+
# Issue #22898.
942+
# Test that a RecursionError is raised when tstate->recursion_depth is
943+
# equal to recursion_limit in PyErr_NormalizeException() and check
944+
# that a ResourceWarning is printed.
945+
# Prior to #22898, the recursivity of PyErr_NormalizeException() was
946+
# controled by tstate->recursion_depth and a PyExc_RecursionErrorInst
947+
# singleton was being used in that case, that held traceback data and
948+
# locals indefinitely and would cause a segfault in _PyExc_Fini() upon
949+
# finalization of these locals.
950+
code = """if 1:
951+
import sys
952+
from _testcapi import get_recursion_depth
953+
954+
class MyException(Exception): pass
955+
956+
def setrecursionlimit(depth):
957+
while 1:
958+
try:
959+
sys.setrecursionlimit(depth)
960+
return depth
961+
except RecursionError:
962+
# sys.setrecursionlimit() raises a RecursionError if
963+
# the new recursion limit is too low (issue #25274).
964+
depth += 1
965+
966+
def recurse(cnt):
967+
cnt -= 1
968+
if cnt:
969+
recurse(cnt)
970+
else:
971+
generator.throw(MyException)
972+
973+
def gen():
974+
f = open(%a, mode='rb', buffering=0)
975+
yield
976+
977+
generator = gen()
978+
next(generator)
979+
recursionlimit = sys.getrecursionlimit()
980+
depth = get_recursion_depth()
981+
try:
982+
# Upon the last recursive invocation of recurse(),
983+
# tstate->recursion_depth is equal to (recursion_limit - 1)
984+
# and is equal to recursion_limit when _gen_throw() calls
985+
# PyErr_NormalizeException().
986+
recurse(setrecursionlimit(depth + 2) - depth - 1)
987+
finally:
988+
sys.setrecursionlimit(recursionlimit)
989+
print('Done.')
990+
""" % __file__
991+
rc, out, err = script_helper.assert_python_failure("-Wd", "-c", code)
992+
# Check that the program does not fail with SIGABRT.
993+
self.assertEqual(rc, 1)
994+
self.assertIn(b'RecursionError', err)
995+
self.assertIn(b'ResourceWarning', err)
996+
self.assertIn(b'Done.', out)
997+
998+
@cpython_only
999+
def test_recursion_normalizing_infinite_exception(self):
1000+
# Issue #30697. Test that a RecursionError is raised when
1001+
# PyErr_NormalizeException() maximum recursion depth has been
1002+
# exceeded.
1003+
code = """if 1:
1004+
import _testcapi
1005+
try:
1006+
raise _testcapi.RecursingInfinitelyError
1007+
finally:
1008+
print('Done.')
1009+
"""
1010+
rc, out, err = script_helper.assert_python_failure("-c", code)
1011+
self.assertEqual(rc, 1)
1012+
self.assertIn(b'RecursionError: maximum recursion depth exceeded '
1013+
b'while normalizing an exception', err)
1014+
self.assertIn(b'Done.', out)
1015+
1016+
@cpython_only
1017+
def test_recursion_normalizing_with_no_memory(self):
1018+
# Issue #30697. Test that in the abort that occurs when there is no
1019+
# memory left and the size of the Python frames stack is greater than
1020+
# the size of the list of preallocated MemoryError instances, the
1021+
# Fatal Python error message mentions MemoryError.
1022+
code = """if 1:
1023+
import _testcapi
1024+
class C(): pass
1025+
def recurse(cnt):
1026+
cnt -= 1
1027+
if cnt:
1028+
recurse(cnt)
1029+
else:
1030+
_testcapi.set_nomemory(0)
1031+
C()
1032+
recurse(16)
1033+
"""
1034+
with SuppressCrashReport():
1035+
rc, out, err = script_helper.assert_python_failure("-c", code)
1036+
self.assertIn(b'Fatal Python error: Cannot recover from '
1037+
b'MemoryErrors while normalizing exceptions.', err)
9391038

9401039
@cpython_only
9411040
def test_MemoryError(self):

‎Misc/NEWS.d/next/C API/2017-06-30-11-58-01.bpo-30697.Q3T_8n.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
The `PyExc_RecursionErrorInst` singleton is removed and
2+
`PyErr_NormalizeException()` does not use it anymore. This singleton is
3+
persistent and its members being never cleared may cause a segfault during
4+
finalization of the interpreter. See also issue #22898.

‎Modules/_testcapimodule.c

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4862,6 +4862,61 @@ static PyTypeObject awaitType = {
48624862
};
48634863

48644864

4865+
static int recurse_infinitely_error_init(PyObject *, PyObject *, PyObject *);
4866+
4867+
static PyTypeObject PyRecursingInfinitelyError_Type = {
4868+
PyVarObject_HEAD_INIT(NULL, 0)
4869+
"RecursingInfinitelyError", /* tp_name */
4870+
sizeof(PyBaseExceptionObject), /* tp_basicsize */
4871+
0, /* tp_itemsize */
4872+
0, /* tp_dealloc */
4873+
0, /* tp_print */
4874+
0, /* tp_getattr */
4875+
0, /* tp_setattr */
4876+
0, /* tp_reserved */
4877+
0, /* tp_repr */
4878+
0, /* tp_as_number */
4879+
0, /* tp_as_sequence */
4880+
0, /* tp_as_mapping */
4881+
0, /* tp_hash */
4882+
0, /* tp_call */
4883+
0, /* tp_str */
4884+
0, /* tp_getattro */
4885+
0, /* tp_setattro */
4886+
0, /* tp_as_buffer */
4887+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
4888+
"Instantiating this exception starts infinite recursion.", /* tp_doc */
4889+
0, /* tp_traverse */
4890+
0, /* tp_clear */
4891+
0, /* tp_richcompare */
4892+
0, /* tp_weaklistoffset */
4893+
0, /* tp_iter */
4894+
0, /* tp_iternext */
4895+
0, /* tp_methods */
4896+
0, /* tp_members */
4897+
0, /* tp_getset */
4898+
0, /* tp_base */
4899+
0, /* tp_dict */
4900+
0, /* tp_descr_get */
4901+
0, /* tp_descr_set */
4902+
0, /* tp_dictoffset */
4903+
(initproc)recurse_infinitely_error_init, /* tp_init */
4904+
0, /* tp_alloc */
4905+
0, /* tp_new */
4906+
};
4907+
4908+
static int
4909+
recurse_infinitely_error_init(PyObject *self, PyObject *args, PyObject *kwds)
4910+
{
4911+
PyObject *type = (PyObject *)&PyRecursingInfinitelyError_Type;
4912+
4913+
/* Instantiating this exception starts infinite recursion. */
4914+
Py_INCREF(type);
4915+
PyErr_SetObject(type, NULL);
4916+
return -1;
4917+
}
4918+
4919+
48654920
static struct PyModuleDef _testcapimodule = {
48664921
PyModuleDef_HEAD_INIT,
48674922
"_testcapi",
@@ -4903,6 +4958,14 @@ PyInit__testcapi(void)
49034958
Py_INCREF(&awaitType);
49044959
PyModule_AddObject(m, "awaitType", (PyObject *)&awaitType);
49054960

4961+
PyRecursingInfinitelyError_Type.tp_base = (PyTypeObject *)PyExc_Exception;
4962+
if (PyType_Ready(&PyRecursingInfinitelyError_Type) < 0) {
4963+
return NULL;
4964+
}
4965+
Py_INCREF(&PyRecursingInfinitelyError_Type);
4966+
PyModule_AddObject(m, "RecursingInfinitelyError",
4967+
(PyObject *)&PyRecursingInfinitelyError_Type);
4968+
49064969
PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX));
49074970
PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN));
49084971
PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX));

‎Objects/exceptions.c

Lines changed: 0 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2430,12 +2430,6 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning,
24302430

24312431

24322432

2433-
/* Pre-computed RecursionError instance for when recursion depth is reached.
2434-
Meant to be used when normalizing the exception for exceeding the recursion
2435-
depth will cause its own infinite recursion.
2436-
*/
2437-
PyObject *PyExc_RecursionErrorInst = NULL;
2438-
24392433
#define PRE_INIT(TYPE) \
24402434
if (!(_PyExc_ ## TYPE.tp_flags & Py_TPFLAGS_READY)) { \
24412435
if (PyType_Ready(&_PyExc_ ## TYPE) < 0) \
@@ -2697,37 +2691,11 @@ _PyExc_Init(PyObject *bltinmod)
26972691
ADD_ERRNO(TimeoutError, ETIMEDOUT);
26982692

26992693
preallocate_memerrors();
2700-
2701-
if (!PyExc_RecursionErrorInst) {
2702-
PyExc_RecursionErrorInst = BaseException_new(&_PyExc_RecursionError, NULL, NULL);
2703-
if (!PyExc_RecursionErrorInst)
2704-
Py_FatalError("Cannot pre-allocate RecursionError instance for "
2705-
"recursion errors");
2706-
else {
2707-
PyBaseExceptionObject *err_inst =
2708-
(PyBaseExceptionObject *)PyExc_RecursionErrorInst;
2709-
PyObject *args_tuple;
2710-
PyObject *exc_message;
2711-
exc_message = PyUnicode_FromString("maximum recursion depth exceeded");
2712-
if (!exc_message)
2713-
Py_FatalError("cannot allocate argument for RecursionError "
2714-
"pre-allocation");
2715-
args_tuple = PyTuple_Pack(1, exc_message);
2716-
if (!args_tuple)
2717-
Py_FatalError("cannot allocate tuple for RecursionError "
2718-
"pre-allocation");
2719-
Py_DECREF(exc_message);
2720-
if (BaseException_init(err_inst, args_tuple, NULL))
2721-
Py_FatalError("init of pre-allocated RecursionError failed");
2722-
Py_DECREF(args_tuple);
2723-
}
2724-
}
27252694
}
27262695

27272696
void
27282697
_PyExc_Fini(void)
27292698
{
2730-
Py_CLEAR(PyExc_RecursionErrorInst);
27312699
free_preallocated_memerrors();
27322700
Py_CLEAR(errnomap);
27332701
}

‎PC/python3.def

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,6 @@ EXPORTS
224224
PyExc_PermissionError=python36.PyExc_PermissionError DATA
225225
PyExc_ProcessLookupError=python36.PyExc_ProcessLookupError DATA
226226
PyExc_RecursionError=python36.PyExc_RecursionError DATA
227-
PyExc_RecursionErrorInst=python36.PyExc_RecursionErrorInst DATA
228227
PyExc_ReferenceError=python36.PyExc_ReferenceError DATA
229228
PyExc_ResourceWarning=python36.PyExc_ResourceWarning DATA
230229
PyExc_RuntimeError=python36.PyExc_RuntimeError DATA

‎Python/errors.c

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -217,20 +217,24 @@ PyErr_ExceptionMatches(PyObject *exc)
217217
}
218218

219219

220+
#ifndef Py_NORMALIZE_RECURSION_LIMIT
221+
#define Py_NORMALIZE_RECURSION_LIMIT 32
222+
#endif
223+
220224
/* Used in many places to normalize a raised exception, including in
221225
eval_code2(), do_raise(), and PyErr_Print()
222226
223227
XXX: should PyErr_NormalizeException() also call
224228
PyException_SetTraceback() with the resulting value and tb?
225229
*/
226-
void
227-
PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb)
230+
static void
231+
PyErr_NormalizeExceptionEx(PyObject **exc, PyObject **val,
232+
PyObject **tb, int recursion_depth)
228233
{
229234
PyObject *type = *exc;
230235
PyObject *value = *val;
231236
PyObject *inclass = NULL;
232237
PyObject *initial_tb = NULL;
233-
PyThreadState *tstate = NULL;
234238

235239
if (type == NULL) {
236240
/* There was no exception, so nothing to do. */
@@ -292,6 +296,10 @@ PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb)
292296
finally:
293297
Py_DECREF(type);
294298
Py_DECREF(value);
299+
if (recursion_depth + 1 == Py_NORMALIZE_RECURSION_LIMIT) {
300+
PyErr_SetString(PyExc_RecursionError, "maximum recursion depth "
301+
"exceeded while normalizing an exception");
302+
}
295303
/* If the new exception doesn't set a traceback and the old
296304
exception had a traceback, use the old traceback for the
297305
new exception. It's better than nothing.
@@ -304,20 +312,26 @@ PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb)
304312
else
305313
Py_DECREF(initial_tb);
306314
}
307-
/* normalize recursively */
308-
tstate = PyThreadState_GET();
309-
if (++tstate->recursion_depth > Py_GetRecursionLimit()) {
310-
--tstate->recursion_depth;
311-
/* throw away the old exception and use the recursion error instead */
312-
Py_INCREF(PyExc_RecursionError);
313-
Py_SETREF(*exc, PyExc_RecursionError);
314-
Py_INCREF(PyExc_RecursionErrorInst);
315-
Py_SETREF(*val, PyExc_RecursionErrorInst);
316-
/* just keeping the old traceback */
317-
return;
315+
/* Normalize recursively.
316+
* Abort when Py_NORMALIZE_RECURSION_LIMIT has been exceeded and the
317+
* corresponding RecursionError could not be normalized.*/
318+
if (++recursion_depth > Py_NORMALIZE_RECURSION_LIMIT) {
319+
if (PyErr_GivenExceptionMatches(*exc, PyExc_MemoryError)) {
320+
Py_FatalError("Cannot recover from MemoryErrors "
321+
"while normalizing exceptions.");
322+
}
323+
else {
324+
Py_FatalError("Cannot recover from the recursive normalization "
325+
"of an exception.");
326+
}
318327
}
319-
PyErr_NormalizeException(exc, val, tb);
320-
--tstate->recursion_depth;
328+
PyErr_NormalizeExceptionEx(exc, val, tb, recursion_depth);
329+
}
330+
331+
void
332+
PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb)
333+
{
334+
PyErr_NormalizeExceptionEx(exc, val, tb, 0);
321335
}
322336

323337

0 commit comments

Comments
 (0)
0