8000 bpo-30697: fix PyErr_NormalizeException() when no memory by xdegaye · Pull Request #2327 · python/cpython · GitHub
[go: up one dir, main page]
Skip to content

bpo-30697: fix PyErr_NormalizeException() when no memory #2327

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Oct 26, 2017
Merged

bpo-30697: fix PyErr_NormalizeException() when no memory #2327

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d190732
bpo-30697: fix PyErr_NormalizeException() when no memory
xdegaye Jun 22, 2017
56ebb33
Do not share the recursion_depth variable amongst all threads.
xdegaye Jun 23, 2017
f824885
Add the test cases not dealing with memory errors.
xdegaye Jun 24, 2017
5fc42a2
Fix the test case for non-debug builds.
xdegaye Jun 28, 2017
fb05d40
Merge remote-tracking branch 'origin/master' into bpo-30697
xdegaye Jun 29, 2017
e53b5a8
Add the Misc/NEWS and What's New entries.
xdegaye Jun 30, 2017
e1bd62b
Merge remote-tracking branch 'origin/master' into bpo-30697
xdegaye Jul 1, 2017
af6896e
Add the test case on memory exhaustion (last test)
xdegaye Jul 1, 2017
d81ef9c
Merge remote-tracking branch 'origin/master' into bpo-30697
xdegaye Oct 26, 2017
312ec15
Merge branch 'master' into bpo-30697
xdegaye Oct 26, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions Include/pyerrors.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,6 @@ PyAPI_DATA(PyObject *) PyExc_IOError;
PyAPI_DATA(PyObject *) PyExc_WindowsError;
#endif

PyAPI_DATA(PyObject *) PyExc_RecursionErrorInst;

/* Predefined warning categories */
PyAPI_DATA(PyObject *) PyExc_Warning;
PyAPI_DATA(PyObject *) PyExc_UserWarning;
Expand Down
63 changes: 63 additions & 0 deletions Modules/_testcapimodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4700,6 +4700,61 @@ static PyTypeObject awaitType = {
};


static int recurse_infinitely_error_init(PyObject *, PyObject *, PyObject *);

static PyTypeObject PyRecursingInfinitelyError_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"RecursingInfinitelyError", /* tp_name */
sizeof(PyBaseExceptionObject), /* tp_basicsize */
0, /* tp_itemsize */
0, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
"Instantiating this exception starts infinite recursion.", /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
(initproc)recurse_infinitely_error_init, /* tp_init */
0, /* tp_alloc */
0, /* tp_new */
};

static int
recurse_infinitely_error_init(PyObject *self, PyObject *args, PyObject *kwds)
{
PyObject *type = (PyObject *)&PyRecursingInfinitelyError_Type;

/* Instantiating this exception starts infinite recursion. */
Py_INCREF(type);
PyErr_SetObject(type, NULL);
return -1;
}


static struct PyModuleDef _testcapimodule = {
PyModuleDef_HEAD_INIT,
"_testcapi",
Expand Down Expand Up @@ -4741,6 +4796,14 @@ PyInit__testcapi(void)
Py_INCREF(&awaitType);
PyModule_AddObject(m, "awaitType", (PyObject *)&awaitType);

PyRecursingInfinitelyError_Type.tp_base = (PyTypeObject *)PyExc_Exception;
if (PyType_Ready(&PyRecursingInfinitelyError_Type) < 0) {
return NULL;
}
Py_INCREF(&PyRecursingInfinitelyError_Type);
PyModule_AddObject(m, "RecursingInfinitelyError",
(PyObject *)&PyRecursingInfinitelyError_Type);

PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX));
PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN));
PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX));
Expand Down
32 changes: 0 additions & 32 deletions Objects/exceptions.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2408,12 +2408,6 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning,



/* Pre-computed RecursionError instance for when recursion depth is reached.
Meant to be used when normalizing the exception for exceeding the recursion
depth will cause its own infinite recursion.
*/
PyObject *PyExc_RecursionErrorInst = NULL;

#define PRE_INIT(TYPE) \
if (!(_PyExc_ ## TYPE.tp_flags & Py_TPFLAGS_READY)) { \
if (PyType_Ready(&_PyExc_ ## TYPE) < 0) \
Expand Down Expand Up @@ -2673,37 +2667,11 @@ _PyExc_Init(PyObject *bltinmod)
ADD_ERRNO(TimeoutError, ETIMEDOUT)

preallocate_memerrors();

if (!PyExc_RecursionErrorInst) {
PyExc_RecursionErrorInst = BaseException_new(&_PyExc_RecursionError, NULL, NULL);
if (!PyExc_RecursionErrorInst)
Py_FatalError("Cannot pre-allocate RecursionError instance for "
"recursion errors");
else {
PyBaseExceptionObject *err_inst =
(PyBaseExceptionObject *)PyExc_RecursionErrorInst;
PyObject *args_tuple;
PyObject *exc_message;
exc_message = PyUnicode_FromString("maximum recursion depth exceeded");
if (!exc_message)
Py_FatalError("cannot allocate argument for RecursionError "
"pre-allocation");
args_tuple = PyTuple_Pack(1, exc_message);
if (!args_tuple)
Py_FatalError("cannot allocate tuple for RecursionError "
"pre-allocation");
Py_DECREF(exc_message);
if (BaseException_init(err_inst, args_tuple, NULL))
Py_FatalError("init of pre-allocated RecursionError failed");
Py_DECREF(args_tuple);
}
}
}

void
_PyExc_Fini(void)
{
Py_CLEAR(PyExc_RecursionErrorInst);
free_preallocated_memerrors();
Py_CLEAR(errnomap);
}
Expand Down
1 change: 0 additions & 1 deletion PC/python3.def
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,6 @@ EXPORTS
PyExc_PermissionError=python37.PyExc_PermissionError DATA
PyExc_ProcessLookupError=python37.PyExc_ProcessLookupError DATA
PyExc_RecursionError=python37.PyExc_RecursionError DATA
PyExc_RecursionErrorInst=python37.PyExc_RecursionErrorInst DATA
PyExc_ReferenceError=python37.PyExc_ReferenceError DATA
PyExc_ResourceWarning=python37.PyExc_ResourceWarning DATA
PyExc_RuntimeError=python37.PyExc_RuntimeError DATA
Expand Down
35 changes: 22 additions & 13 deletions Python/errors.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,10 @@ PyErr_ExceptionMatches(PyObject *exc)
}


#ifndef Py_NORMALIZE_RECURSION_LIMIT
#define Py_NORMALIZE_RECURSION_LIMIT 32
#endif

/* Used in many places to normalize a raised exception, including in
eval_code2(), do_raise(), and PyErr_Print()

Expand All @@ -230,7 +234,7 @@ PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb)
PyObject *value = *val;
PyObject *inclass = NULL;
PyObject *initial_tb = NULL;
PyThreadState *tstate = NULL;
static int recursion_depth = 0;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it right to use a static variable instead of a threadstate-attached variable? What if the GIL is released somewhere in-between and the wrong thread gets the recursion error (or, worse, triggers a fatal error)?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right of course. I will fix that.


if (type == NULL) {
/* There was no exception, so nothing to do. */
Expand Down Expand Up @@ -292,6 +296,10 @@ PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb)
finally:
Py_DECREF(type);
Py_DECREF(value);
if (recursion_depth + 1 == Py_NORMALIZE_RECURSION_LIMIT) {
PyErr_Format(PyExc_RecursionError, "maximum recursion depth exceeded "
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A nit, but you can use PyErr_SetString if you don't use the formatting capabilities.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or even PyErr_SetObject() with pre-created static string object (this is used in some critical cases).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The instantiation of the RecursionError object itself fails anyway upon its normalization when there is no memory (and a memory error is reported to the user and not the recursion error). So it seems there is no advantage in using PyErr_SetObject() with a pre-created static string object over PyErr_SetString() here.

"while normalizing an exception");
}
/* If the new exception doesn't set a traceback and the old
exception had a traceback, use the old traceback for the
new exception. It's better than nothing.
Expand All @@ -304,20 +312,21 @@ PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb)
else
Py_DECREF(initial_tb);
}
/* normalize recursively */
tstate = PyThreadState_GET();
if (++tstate->recursion_depth > Py_GetRecursionLimit()) {
--tstate->recursion_depth;
/* throw away the old exception and use the recursion error instead */
Py_INCREF(PyExc_RecursionError);
Py_SETREF(*exc, PyExc_RecursionError);
Py_INCREF(PyExc_RecursionErrorInst);
Py_SETREF(*val, PyExc_RecursionErrorInst);
/* just keeping the old traceback */
return;
/* Normalize recursively.
* Abort when Py_NORMALIZE_RECURSION_LIMIT has been exceeded and the
* corresponding RecursionError could not be normalized.*/
if (++recursion_depth > Py_NORMALIZE_RECURSION_LIMIT) {
if (PyErr_GivenExceptionMatches(*exc, PyExc_MemoryError)) {
Py_FatalError("Cannot recover from MemoryErrors "
"while normalizing exceptions.");
}
else {
Py_FatalError("Cannot recover from the recursive normalization "
"of an exception.");
}
}
PyErr_NormalizeException(exc, val, tb);
--tstate->recursion_depth;
--recursion_depth;
}


Expand Down
0