8000 [3.12] GH-112215: Backport C recursion changes (GH-115083) · python/cpython@4d87832 · GitHub
[go: up one dir, main page]
Skip to content

Commit 4d87832

Browse files
markshannonmarkshannon
authored
[3.12] GH-112215: Backport C recursion changes (GH-115083)
1 parent a30bb08 commit 4d87832

14 files changed

+64
-45
lines changed

‎Include/cpython/pystate.h

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -251,12 +251,24 @@ struct _ts {
251251
/* WASI has limited call stack. Python's recursion limit depends on code
252252
layout, optimization, and WASI runtime. Wasmtime can handle about 700
253253
recursions, sometimes less. 500 is a more conservative limit. */
254-
#ifndef C_RECURSION_LIMIT
255-
# ifdef __wasi__
254+
#ifdef Py_DEBUG
255+
# if defined(__wasi__)
256+
# define C_RECURSION_LIMIT 150
257+
# else
258+
# define C_RECURSION_LIMIT 500
259+
# endif
260+
#else
261+
# if defined(__wasi__)
256262
# define C_RECURSION_LIMIT 500
263+
# elif defined(__s390x__)
264+
# define C_RECURSION_LIMIT 800
265+
# elif defined(_WIN32)
266+
# define C_RECURSION_LIMIT 3000
267+
# elif defined(_Py_ADDRESS_SANITIZER)
268+
# define C_RECURSION_LIMIT 4000
257269
# else
258-
// This value is duplicated in Lib/test/support/__init__.py
259-
# define C_RECURSION_LIMIT 1500
270+
// This value is duplicated in Lib/test/support/__init__.py
271+
# define C_RECURSION_LIMIT 10000
260272
# endif
261273
#endif
262274

‎Lib/test/support/__init__.py

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2112,13 +2112,13 @@ def set_recursion_limit(limit):
21122112
finally:
21132113
sys.setrecursionlimit(original_limit)
21142114

2115-
def infinite_recursion(max_depth=100):
2116-
"""Set a lower limit for tests that interact with infinite recursions
2117-
(e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some
2118-
debug windows builds, due to not enough functions being inlined the
2119-
stack size might not handle the default recursion limit (1000). See
2120-
bpo-11105 for details."""
2121-
if max_depth < 3:
2115+
def infinite_recursion(max_depth=None):
2116+
if max_depth is None:
2117+
# Pick a number large enough to cause problems
2118+
# but not take too long for code that can handle
2119+
# very deep recursion.
2120+
max_depth = 20_000
2121+
elif max_depth < 3:
21222122
raise ValueError("max_depth must be at least 3, got {max_depth}")
21232123
depth = get_recursion_depth()
21242124
depth = max(depth - 1, 1) # Ignore infinite_recursion() frame.
@@ -2362,7 +2362,22 @@ def adjust_int_max_str_digits(max_digits):
23622362
EXCEEDS_RECURSION_LIMIT = 5000
23632363

23642364
# The default C recursion limit (from Include/cpython/pystate.h).
2365-
C_RECURSION_LIMIT = 1500
2365+
if Py_DEBUG:
2366+
if is_wasi:
2367+
C_RECURSION_LIMIT = 150
2368+
else:
2369+
C_RECURSION_LIMIT = 500
2370+
else:
2371+
if is_wasi:
2372+
C_RECURSION_LIMIT = 500
2373+
elif hasattr(os, 'uname') and os.uname().machine == 's390x':
2374+
C_RECURSION_LIMIT = 800
2375+
elif sys.platform.startswith('win'):
2376+
C_RECURSION_LIMIT = 3000
2377+
elif check_sanitizer(address=True):
2378+
C_RECURSION_LIMIT = 4000
2379+
else:
2380+
C_RECURSION_LIMIT = 10000
23662381

23672382
#Windows doesn't have os.uname() but it doesn't support s390x.
23682383
skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',

‎Lib/test/test_ast.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1087,9 +1087,9 @@ def next(self):
10871087
@unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
10881088
@support.cpython_only
10891089
def test_ast_recursion_limit(self):
1090-
fail_depth = support.EXCEEDS_RECURSION_LIMIT
1090+
fail_depth = support.C_RECURSION_LIMIT + 1
10911091
crash_depth = 100_000
1092-
success_depth = 1200
1092+
success_depth = int(support.C_RECURSION_LIMIT * 0.9)
10931093

10941094
def check_limit(prefix, repeated):
10951095
expect_ok = prefix + repeated * success_depth

‎Lib/test/test_call.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import unittest
2-
from test.support import cpython_only, requires_limited_api, skip_on_s390x
2+
from test.support import cpython_only, requires_limited_api, skip_on_s390x, is_wasi, Py_DEBUG
33
try:
44
import _testcapi
55
except ImportError:
@@ -932,6 +932,7 @@ def test_multiple_values(self):
932932
class TestRecursion(unittest.TestCase):
933933

934934
@skip_on_s390x
935+
@unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack")
935936
def test_super_deep(self):
936937

937938
def recurse(n):

‎Lib/test/test_compile.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -607,9 +607,9 @@ def test_compiler_recursion_limit(self):
607607
# Expected limit is C_RECURSION_LIMIT * 2
608608
# Duplicating the limit here is a little ugly.
609609
# Perhaps it should be exposed somewhere...
610-
fail_depth = C_RECURSION_LIMIT * 2 + 1
610+
fail_depth = C_RECURSION_LIMIT + 1
611611
crash_depth = C_RECURSION_LIMIT * 100
612-
success_depth = int(C_RECURSION_LIMIT * 1.8)
612+
success_depth = int(C_RECURSION_LIMIT * 0.9)
613613

614614
def check_limit(prefix, repeated, mode="single"):
615615
expect_ok = prefix + repeated * success_depth

‎Lib/test/test_isinstance.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,7 @@ def blowstack(fxn, arg, compare_to):
352352
# Make sure that calling isinstance with a deeply nested tuple for its
353353
# argument will raise RecursionError eventually.
354354
tuple_arg = (compare_to,)
355-
for cnt in range(support.EXCEEDS_RECURSION_LIMIT):
355+
for cnt in range(support.C_RECURSION_LIMIT * 2):
356356
tuple_arg = (tuple_arg,)
357357
fxn(arg, tuple_arg)
358358

‎Lib/test/test_plistlib.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -908,7 +908,7 @@ def test_cycles(self):
908908
self.assertIs(b['x'], b)
909909

910910
def test_deep_nesting(self):
911-
tests = [50, 100_000] if support.is_wasi else [50, 300, 100_000]
911+
tests = [50, 100_000] if support.is_wasi else [50, 600, 100_000]
912912
for N in tests:
913913
chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)]
914914
try:

‎Lib/test/test_sys_settrace.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2965,16 +2965,18 @@ def test_trace_unpack_long_sequence(self):
29652965
self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1})
29662966

29672967
def test_trace_lots_of_globals(self):
2968+
count = min(1000, int(support.C_RECURSION_LIMIT * 0.8))
2969+
29682970
code = """if 1:
29692971
def f():
29702972
return (
29712973
{}
29722974
)
2973-
""".format("\n+\n".join(f"var{i}\n" for i in range(1000)))
2974-
ns = {f"var{i}": i for i in range(1000)}
2975+
""".format("\n+\n".join(f"var{i}\n" for i in range(count)))
2976+
ns = {f"var{i}": i for i in range(count)}
29752977
exec(code, ns)
29762978
counts = self.count_traces(ns["f"])
2977-
self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1})
2979+
self.assertEqual(counts, {'call': 1, 'line': count * 2, 'return': 1})
29782980

29792981

29802982
class TestEdgeCases(unittest.TestCase):

‎Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Change the C recursion limits to more closely reflect the underlying
2+
platform limits.

‎Parser/asdl_c.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1393,15 +1393,14 @@ class PartingShots(StaticVisitor):
13931393
13941394
int starting_recursion_depth;
13951395
/* Be careful here to prevent overflow. */
1396-
int COMPILER_STACK_FRAME_SCALE = 2;
13971396
PyThreadState *tstate = _PyThreadState_GET();
13981397
if (!tstate) {
13991398
return NULL;
14001399
}
14011400
struct validator vstate;
1402-
vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
1401+
vstate.recursion_limit = C_RECURSION_LIMIT;
14031402
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
1404-
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
1403+
starting_recursion_depth = recursion_depth;
14051404
vstate.recursion_depth = starting_recursion_depth;
14061405
14071406
PyObject *result = ast2obj_mod(state, &vstate, t);

‎Python/Python-ast.c

Lines changed: 2 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/ast.c

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1038,9 +1038,6 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps)
10381038
}
10391039

10401040

1041-
/* See comments in symtable.c. */
1042-
#define COMPILER_STACK_FRAME_SCALE 2
1043-
10441041
int
10451042
_PyAST_Validate(mod_ty mod)
10461043
{
@@ -1057,9 +1054,9 @@ _PyAST_Validate(mod_ty mod)
10571054
}
10581055
/* Be careful here to prevent overflow. */
10591056
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
1060-
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
1057+
starting_recursion_depth = recursion_depth;
10611058
state.recursion_depth = starting_recursion_depth;
1062-
state.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
1059+
state.recursion_limit = C_RECURSION_LIMIT;
10631060

10641061
switch (mod->kind) {
10651062
case Module_kind:

‎Python/ast_opt.c

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1102,9 +1102,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat
11021102
#undef CALL_OPT
11031103
#undef CALL_SEQ
11041104

1105-
/* See comments in symtable.c. */
1106-
#define COMPILER_STACK_FRAME_SCALE 2
1107-
11081105
int
11091106
_PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state)
11101107
{
@@ -1118,9 +1115,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state)
11181115
}
11191116
/* Be careful here to prevent overflow. */
11201117
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
1121-
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
1118+
starting_recursion_depth = recursion_depth;
11221119
state->recursion_depth = starting_recursion_depth;
1123-
state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
1120+
state->recursion_limit = C_RECURSION_LIMIT;
11241121

11251122
int ret = astfold_mod(mod, arena, state);
11261123
assert(ret || PyErr_Occurred());

‎Python/symtable.c

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -281,11 +281,6 @@ symtable_new(void)
281281
return NULL;
282282
}
283283

284-
/* Using a scaling factor means this should automatically adjust when
285-
the recursion limit is adjusted for small or large C stack allocations.
286-
*/
287-
#define COMPILER_STACK_FRAME_SCALE 2
288-
289284
struct symtable *
290285
_PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
291286
{
@@ -312,9 +307,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
312307
}
313308
/* Be careful here to prevent overflow. */
314309
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
315-
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
310+
starting_recursion_depth = recursion_depth;
316311
st->recursion_depth = starting_recursion_depth;
317-
st->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
312+
st->recursion_limit = C_RECURSION_LIMIT;
318313

319314
/* Make the initial symbol information gathering pass */
320315
if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) {

0 commit comments

Comments
 (0)
0