10000 GH-91079: Decouple C stack overflow checks from Python recursion chec… · python/cpython@7644935 · GitHub
[go: up one dir, main page]
Skip to content

Commit 7644935

Browse files
markshannonmarkshannon
authored
GH-91079: Decouple C stack overflow checks from Python recursion checks. (GH-96510)
1 parent 0ff8fd6 commit 7644935

22 files changed

+165
-99
lines changed

‎Include/cpython/pystate.h

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,10 @@ struct _ts {
9595
/* Was this thread state statically allocated? */
9696
int _static;
9797

98-
int recursion_remaining;
99-
int recursion_limit;
98+
int py_recursion_remaining;
99+
int py_recursion_limit;
100+
101+
int c_recursion_remaining;
100102
int recursion_headroom; /* Allow 50 more calls to handle any errors. */
101103

102104
/* 'tracing' keeps track of the execution depth when tracing/profiling.
@@ -202,6 +204,16 @@ struct _ts {
202204
_PyCFrame root_cframe;
203205
};
204206

207+
/* WASI has limited call stack. Python's recursion limit depends on code
208+
layout, optimization, and WASI runtime. Wasmtime can handle about 700
209+
recursions, sometimes less. 500 is a more conservative limit. */
210+
#ifndef C_RECURSION_LIMIT
211+
# ifdef __wasi__
212+
# define C_RECURSION_LIMIT 500
213+
# else
214+
# define C_RECURSION_LIMIT 800
215+
# endif
216+
#endif
205217

206218
/* other API */
207219

‎Include/internal/pycore_ceval.h

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,8 @@ extern "C" {
1212
struct pyruntimestate;
1313
struct _ceval_runtime_state;
1414

15-
/* WASI has limited call stack. Python's recursion limit depends on code
16-
layout, optimization, and WASI runtime. Wasmtime can handle about 700-750
17-
recursions, sometimes less. 600 is a more conservative limit. */
1815
#ifndef Py_DEFAULT_RECURSION_LIMIT
19-
# ifdef __wasi__
20-
# define Py_DEFAULT_RECURSION_LIMIT 600
21-
# else
22-
# define Py_DEFAULT_RECURSION_LIMIT 1000
23-
# endif
16+
# define Py_DEFAULT_RECURSION_LIMIT 1000
2417
#endif
2518

2619
#include "pycore_interp.h" // PyInterpreterState.eval_frame
@@ -118,19 +111,22 @@ extern void _PyEval_DeactivateOpCache(void);
118111
/* With USE_STACKCHECK macro defined, trigger stack checks in
119112
_Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */
120113
static inline int _Py_MakeRecCheck(PyThreadState *tstate) {
121-
return (tstate->recursion_remaining-- <= 0
122-
|| (tstate->recursion_remaining & 63) == 0);
114+
return (tstate->c_recursion_remaining-- <= 0
115+
|| (tstate->c_recursion_remaining & 63) == 0);
123116
}
124117
#else
125118
static inline int _Py_MakeRecCheck(PyThreadState *tstate) {
126-
return tstate->recursion_remaining-- <= 0;
119+
return tstate->c_recursion_remaining-- <= 0;
127120
}
128121
#endif
129122

130123
PyAPI_FUNC(int) _Py_CheckRecursiveCall(
131124
PyThreadState *tstate,
132125
const char *where);
133126

127+
int _Py_CheckRecursiveCallPy(
128+
PyThreadState *tstate);
129+
134130
static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate,
135131
const char *where) {
136132
return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where));
@@ -142,7 +138,7 @@ static inline int _Py_EnterRecursiveCall(const char *where) {
142138
}
143139

144140
static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) {
145-
tstate->recursion_remaining++;
141+
tstate->c_recursion_remaining++;
146142
}
147143

148144
static inline void _Py_LeaveRecursiveCall(void) {
@@ -157,6 +153,7 @@ extern PyObject* _Py_MakeCoro(PyFunctionObject *func);
157153
extern int _Py_HandlePending(PyThreadState *tstate);
158154

159155

156+
160157
#ifdef __cplusplus
161158
}
162159
#endif

‎Include/internal/pycore_runtime_init.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ extern "C" {
6868
#define _PyThreadState_INIT \
6969
{ \
7070
._static = 1, \
71-
.recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
71+
.py_recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
7272
.context_ver = 1, \
7373
}
7474

‎Lib/test/support/__init__.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@
6060
"run_with_tz", "PGO", "missing_compiler_executable",
6161
"ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST",
6262
"LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT",
63-
"Py_DEBUG",
63+
"Py_DEBUG", "EXCEEDS_RECURSION_LIMIT",
6464
]
6565

6666

@@ -2352,3 +2352,6 @@ def adjust_int_max_str_digits(max_digits):
23522352
yield
23532353
finally:
23542354
sys.set_int_max_str_digits(current)
2355+
2356+
#For recursion tests, easily exceeds default recursion limit
2357+
EXCEEDS_RECURSION_LIMIT = 5000

‎Lib/test/test_ast.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -825,9 +825,9 @@ def next(self):
825825

826826
@support.cpython_only
827827
def test_ast_recursion_limit(self):
828-
fail_depth = sys.getrecursionlimit() * 3
829-
crash_depth = sys.getrecursionlimit() * 300
830-
success_depth = int(fail_depth * 0.75)
828+
fail_depth = support.EXCEEDS_RECURSION_LIMIT
829+
crash_depth = 100_000
830+
success_depth = 1200
831831

832832
def check_limit(prefix, repeated):
833833
expect_ok = prefix + repeated * success_depth

‎Lib/test/test_call.py

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -864,6 +864,44 @@ def test_multiple_values(self):
864864
with self.check_raises_type_error(msg):
865865
A().method_two_args("x", "y", x="oops")
866866

867+
@cpython_only
868+
class TestRecursion(unittest.TestCase):
869+
870+
def test_super_deep(self):
871+
872+
def recurse(n):
873+
if n:
874+
recurse(n-1)
875+
876+
def py_recurse(n, m):
877+
if n:
878+
py_recurse(n-1, m)
879+
else:
880+
c_py_recurse(m-1)
881+
882+
def c_recurse(n):
883+
if n:
884+
_testcapi.pyobject_fastcall(c_recurse, (n-1,))
885+
886+
def c_py_recurse(m):
887+
if m:
888+
_testcapi.pyobject_fastcall(py_recurse, (1000, m))
889+
890+
depth = sys.getrecursionlimit()
891+
sys.setrecursionlimit(100_000)
892+
try:
893+
recurse(90_000)
894+
with self.assertRaises(RecursionError):
895+
recurse(101_000)
896+
c_recurse(100)
897+
with self.assertRaises(RecursionError):
898+
c_recurse(90_000)
899+
c_py_recurse(90)
900+
with self.assertRaises(RecursionError):
901+
c_py_recurse(100_000)
902+
finally:
903+
sys.setrecursionlimit(depth)
904+
867905

868906
if __name__ == "__main__":
869907
unittest.main()

‎Lib/test/test_collections.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -545,7 +545,7 @@ def test_odd_sizes(self):
545545
self.assertEqual(Dot(1)._replace(d=999), (999,))
546546
self.assertEqual(Dot(1)._fields, ('d',))
547547

548-
n = 5000
548+
n = support.EXCEEDS_RECURSION_LIMIT
549549
names = list(set(''.join([choice(string.ascii_letters)
550550
for j in range(10)]) for i in range(n)))
551551
n = len(names)

‎Lib/test/test_compile.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,7 @@ def __getitem__(self, key):
111111

112112
@unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
113113
def test_extended_arg(self):
114-
# default: 1000 * 2.5 = 2500 repetitions
115-
repeat = int(sys.getrecursionlimit() * 2.5)
114+
repeat = 2000
116115
longexpr = 'x = x or ' + '-x' * repeat
117116
g = {}
118117
code = '''

‎Lib/test/test_dynamic.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -140,11 +140,11 @@ class MyGlobals(dict):
140140
def __missing__(self, key):
141141
return int(key.removeprefix("_number_"))
142142

143-
# 1,000 on most systems
144-
limit = sys.getrecursionlimit()
145-
code = "lambda: " + "+".join(f"_number_{i}" for i in range(limit))
143+
# Need more than 256 variables to use EXTENDED_ARGS
144+
variables = 400
145+
code = "lambda: " + "+".join(f"_number_{i}" for i in range(variables))
146146
sum_func = eval(code, MyGlobals())
147-
expected = sum(range(limit))
147+
expected = sum(range(variables))
148148
# Warm up the the function for quickening (PEP 659)
149149
for _ in range(30):
150150
self.assertEqual(sum_func(), expected)

‎Lib/test/test_exceptions.py

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1372,6 +1372,7 @@ def test_recursion_normalizing_exception(self):
13721372
code = """if 1:
13731373
import sys
13741374
from _testinternalcapi import get_recursion_depth
1375+
from test import support
13751376
13761377
class MyException(Exception): pass
13771378
@@ -1399,13 +1400,8 @@ def gen():
13991400
generator = gen()
14001401
next(generator)
14011402
recursionlimit = sys.getrecursionlimit()
1402-
depth = get_recursion_depth()
14031403
try:
1404-
# Upon the last recursive invocation of recurse(),
1405-
# tstate->recursion_depth is equal to (recursion_limit - 1)
1406-
# and is equal to recursion_limit when _gen_throw() calls
1407-
# PyErr_NormalizeException().
1408-
recurse(setrecursionlimit(depth + 2) - depth)
1404+
recurse(support.EXCEEDS_RECURSION_LIMIT)
14091405
finally:
14101406
sys.setrecursionlimit(recursionlimit)
14111407
print('Done.')

‎Lib/test/test_isinstance.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from test import support
99

1010

11-
11+
1212
class TestIsInstanceExceptions(unittest.TestCase):
1313
# Test to make sure that an AttributeError when accessing the instance's
1414
# class's bases is masked. This was actually a bug in Python 2.2 and
@@ -97,7 +97,7 @@ def getclass(self):
9797
class D: pass
9898
self.assertRaises(RuntimeError, isinstance, c, D)
9999

100-
100+
101101
# These tests are similar to above, but tickle certain code paths in
102102
# issubclass() instead of isinstance() -- really PyObject_IsSubclass()
103103
# vs. PyObject_IsInstance().
@@ -147,7 +147,7 @@ def getbases(self):
147147
self.assertRaises(TypeError, issubclass, B, C())
148148

149149

150-
150+
151151
# meta classes for creating abstract classes and instances
152152
class AbstractClass(object):
153153
def __init__(self, bases):
@@ -179,7 +179,7 @@ class Super:
179179

180180
class Child(Super):
181181
pass
182-
182+
183183
class TestIsInstanceIsSubclass(unittest.TestCase):
184184
# Tests to ensure that isinstance and issubclass work on abstract
185185
# classes and instances. Before the 2.2 release, TypeErrors were
@@ -353,10 +353,10 @@ def blowstack(fxn, arg, compare_to):
353353
# Make sure that calling isinstance with a deeply nested tuple for its
354354
# argument will raise RecursionError eventually.
355355
tuple_arg = (compare_to,)
356-
for cnt in range(sys.getrecursionlimit()+5):
356+
for cnt in range(support.EXCEEDS_RECURSION_LIMIT):
357357
tuple_arg = (tuple_arg,)
358358
fxn(arg, tuple_arg)
359359

360-
360+
361361
if __name__ == '__main__':
362362
unittest.main()

‎Lib/test/test_marshal.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,8 @@ def test_code(self):
117117

118118
def test_many_codeobjects(self):
119119
# Issue2957: bad recursion count on code objects
120-
count = 5000 # more than MAX_MARSHAL_STACK_DEPTH
120+
# more than MAX_MARSHAL_STACK_DEPTH
121+
count = support.EXCEEDS_RECURSION_LIMIT
121122
codes = (ExceptionTestCase.test_exceptions.__code__,) * count
122123
marshal.loads(marshal.dumps(codes))
123124

‎Misc/NEWS.d/next/Core and Builtins/2022-09-05-09-56-32.gh-issue-91079.H4-DdU.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Separate Python recursion checking from C recursion checking which reduces
2+
the chance of C stack overflow and allows the recursion limit to be
3+
increased safely.

‎Modules/_testinternalcapi.c

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,7 @@ get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args))
4444
{
4545
PyThreadState *tstate = _PyThreadState_GET();
4646

47-
/* subtract one to ignore the frame of the get_recursion_depth() call */
48-
49-
return PyLong_FromLong(tstate->recursion_limit - tstate->recursion_remaining - 1);
47+
return PyLong_FromLong(tstate->py_recursion_limit - tstate->py_recursion_remaining);
5048
}
5149

5250

‎Parser/asdl_c.py

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1380,19 +1380,16 @@ class PartingShots(StaticVisitor):
13801380
return NULL;
13811381
}
13821382
1383-
int recursion_limit = Py_GetRecursionLimit();
13841383
int starting_recursion_depth;
13851384
/* Be careful here to prevent overflow. */
13861385
int COMPILER_STACK_FRAME_SCALE = 3;
13871386
PyThreadState *tstate = _PyThreadState_GET();
13881387
if (!tstate) {
13891388
return 0;
13901389
}
1391-
state->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
1392-
recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit;
1393-
int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
1394-
starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
1395-
recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth;
1390+
state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
1391+
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
1392+
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
13961393
state->recursion_depth = starting_recursion_depth;
13971394
13981395
PyObject *result = ast2obj_mod(state, t);

‎Python/Python-ast.c

Lines changed: 3 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/ast.c

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -975,7 +975,6 @@ _PyAST_Validate(mod_ty mod)
975975
int res = -1;
976976
struct validator state;
977977
PyThreadState *tstate;
978-
int recursion_limit = Py_GetRecursionLimit();
979978
int starting_recursion_depth;
980979

981980
/* Setup recursion depth check counters */
@@ -984,12 +983,10 @@ _PyAST_Validate(mod_ty mod)
984983
return 0;
985984
}
986985
/* Be careful here to prevent overflow. */
987-
int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
988-
starting_recursion_depth = (recursion_depth< INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
989-
recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth;
986+
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
987+
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
990988
state.recursion_depth = starting_recursion_depth;
991-
state.recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
992-
recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit;
989+
state.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
993990

994991
switch (mod->kind) {
995992
case Module_kind:

‎Python/ast_opt.c

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1080,7 +1080,6 @@ int
10801080
_PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state)
10811081
{
10821082
PyThreadState *tstate;
1083-
int recursion_limit = Py_GetRecursionLimit();
10841083
int starting_recursion_depth;
10851084

10861085
/* Setup recursion depth check counters */
@@ -1089,12 +1088,10 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state)
10891088
return 0;
10901089
}
10911090
/* Be careful here to prevent overflow. */
1092-
int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
1093-
starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
1094-
recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth;
1091+
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
1092+
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
10951093
state->recursion_depth = starting_recursion_depth;
1096-
state->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
1097-
recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit;
1094+
state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
10981095

10991096
int ret = astfold_mod(mod, arena, state);
11001097
assert(ret || PyErr_Occurred());

0 commit comments

Comments
 (0)
0