From f07b697b12ee630b8c7b609a8940697ebda81102 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 4 Feb 2024 01:22:36 +0000 Subject: [PATCH 1/8] Backport C recursion limit changes to 3.12 --- Include/cpython/pystate.h | 20 ++++++++++++++++---- Lib/test/support/__init__.py | 15 ++++++++++++++- Lib/test/test_ast.py | 4 ++-- Lib/test/test_compile.py | 4 ++-- Lib/test/test_plistlib.py | 2 +- Lib/test/test_sys_settrace.py | 8 +++++--- Parser/asdl_c.py | 5 ++--- Python/Python-ast.c | 5 ++--- Python/ast.c | 7 ++----- Python/ast_opt.c | 7 ++----- Python/symtable.c | 9 ++------- 11 files changed, 50 insertions(+), 36 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 628f2e0996e469..5ef4ebf03069ec 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -251,12 +251,24 @@ struct _ts { /* WASI has limited call stack. Python's recursion limit depends on code layout, optimization, and WASI runtime. Wasmtime can handle about 700 recursions, sometimes less. 500 is a more conservative limit. */ -#ifndef C_RECURSION_LIMIT -# ifdef __wasi__ +#ifdef Py_DEBUG +# if defined(__wasi__) +# define C_RECURSION_LIMIT 150 +# else # define C_RECURSION_LIMIT 500 +# endif +#else +# if defined(__wasi__) +# define C_RECURSION_LIMIT 150 +# elif defined(__s390x__) +# define C_RECURSION_LIMIT 800 +# elif defined(_WIN32) +# define C_RECURSION_LIMIT 3000 +# elif defined(_Py_ADDRESS_SANITIZER) +# define C_RECURSION_LIMIT 4000 # else - // This value is duplicated in Lib/test/support/__init__.py -# define C_RECURSION_LIMIT 1500 + // This value is duplicated in Lib/test/support/__init__.py +# define C_RECURSION_LIMIT 10000 # endif #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 8c4b4e023f633f..ad73f2ffbd625c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2362,7 +2362,20 @@ def adjust_int_max_str_digits(max_digits): EXCEEDS_RECURSION_LIMIT = 5000 # The default C recursion limit (from Include/cpython/pystate.h). -C_RECURSION_LIMIT = 1500 +if Py_DEBUG: + if is_wasi: + C_RECURSION_LIMIT = 150 + else: + C_RECURSION_LIMIT = 500 +else: + if is_wasi: + C_RECURSION_LIMIT = 500 + elif hasattr(os, 'uname') and os.uname().machine == 's390x': + C_RECURSION_LIMIT = 800 + elif sys.platform.startswith('win'): + C_RECURSION_LIMIT = 3000 + else: + C_RECURSION_LIMIT = 10000 #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 3ba7cf7b04266f..9736208a92fc6e 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1087,9 +1087,9 @@ def next(self): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") @support.cpython_only def test_ast_recursion_limit(self): - fail_depth = support.EXCEEDS_RECURSION_LIMIT + fail_depth = support.C_RECURSION_LIMIT + 1 crash_depth = 100_000 - success_depth = 1200 + success_depth = int(support.C_RECURSION_LIMIT * 0.9) def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 42df670fe00e0a..ed23d31c0d3ecd 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -607,9 +607,9 @@ def test_compiler_recursion_limit(self): # Expected limit is C_RECURSION_LIMIT * 2 # Duplicating the limit here is a little ugly. # Perhaps it should be exposed somewhere... - fail_depth = C_RECURSION_LIMIT * 2 + 1 + fail_depth = C_RECURSION_LIMIT + 1 crash_depth = C_RECURSION_LIMIT * 100 - success_depth = int(C_RECURSION_LIMIT * 1.8) + success_depth = int(C_RECURSION_LIMIT * 0.9) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index 3f10f16d71996d..fa46050658afe0 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -908,7 +908,7 @@ def test_cycles(self): self.assertIs(b['x'], b) def test_deep_nesting(self): - tests = [50, 100_000] if support.is_wasi else [50, 300, 100_000] + tests = [50, 100_000] if support.is_wasi else [50, 600, 100_000] for N in tests: chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)] try: diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 7e16e94aa110b2..196fd60d1973f6 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2965,16 +2965,18 @@ def test_trace_unpack_long_sequence(self): self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1}) def test_trace_lots_of_globals(self): + count = min(1000, int(support.C_RECURSION_LIMIT * 0.8)) + code = """if 1: def f(): return ( {} ) - """.format("\n+\n".join(f"var{i}\n" for i in range(1000))) - ns = {f"var{i}": i for i in range(1000)} + """.format("\n+\n".join(f"var{i}\n" for i in range(count))) + ns = {f"var{i}": i for i in range(count)} exec(code, ns) counts = self.count_traces(ns["f"]) - self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1}) + self.assertEqual(counts, {'call': 1, 'line': count * 2, 'return': 1}) class TestEdgeCases(unittest.TestCase): diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index d42c26396d5de2..2c34f5c1bc2675 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1393,15 +1393,14 @@ class PartingShots(StaticVisitor): int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return NULL; } struct validator vstate; - vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + vstate.recursion_limit = C_RECURSION_LIMIT; int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; vstate.recursion_depth = starting_recursion_depth; PyObject *result = ast2obj_mod(state, &vstate, t); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 6c95f07c386fd0..ecaff2041f9565 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13152,15 +13152,14 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return NULL; } struct validator vstate; - vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + vstate.recursion_limit = C_RECURSION_LIMIT; int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; vstate.recursion_depth = starting_recursion_depth; PyObject *result = ast2obj_mod(state, &vstate, t); diff --git a/Python/ast.c b/Python/ast.c index 82d7beec0ee510..76f6556ded098b 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1038,9 +1038,6 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) } -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Validate(mod_ty mod) { @@ -1057,9 +1054,9 @@ _PyAST_Validate(mod_ty mod) } /* Be careful here to prevent overflow. */ int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = C_RECURSION_LIMIT; switch (mod->kind) { case Module_kind: diff --git a/Python/ast_opt.c b/Python/ast_opt.c index f8c4a9513236b9..e881b7fe2d4413 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1102,9 +1102,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_OPT #undef CALL_SEQ -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state) { @@ -1118,9 +1115,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state) } /* Be careful here to prevent overflow. */ int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state->recursion_depth = starting_recursion_depth; - state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state->recursion_limit = C_RECURSION_LIMIT; int ret = astfold_mod(mod, arena, state); assert(ret || PyErr_Occurred()); diff --git a/Python/symtable.c b/Python/symtable.c index a5c6b465b71ddd..4cf6ad3cf5ca6f 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -281,11 +281,6 @@ symtable_new(void) return NULL; } -/* Using a scaling factor means this should automatically adjust when - the recursion limit is adjusted for small or large C stack allocations. -*/ -#define COMPILER_STACK_FRAME_SCALE 2 - struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) { @@ -312,9 +307,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) } /* Be careful here to prevent overflow. */ int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; st->recursion_depth = starting_recursion_depth; - st->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + st->recursion_limit = C_RECURSION_LIMIT; /* Make the initial symbol information gathering pass */ if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) { From 876b7075699691df67e4cf05886c98f215bff2a1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 4 Feb 2024 01:26:32 +0000 Subject: [PATCH 2/8] Add news --- .../2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst new file mode 100644 index 00000000000000..4f6c356e414928 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst @@ -0,0 +1,2 @@ +Change the C recursion limits to more closely reflect the underlying +platform limits. From 8c1dae9429eb1f0ac372ab7318d72ab4545a9fc3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 4 Feb 2024 02:15:01 +0000 Subject: [PATCH 3/8] Make infinite_recursion do what it claims to do --- Lib/test/support/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index ad73f2ffbd625c..8037019b9d7ef8 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2112,13 +2112,13 @@ def set_recursion_limit(limit): finally: sys.setrecursionlimit(original_limit) -def infinite_recursion(max_depth=100): - """Set a lower limit for tests that interact with infinite recursions - (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some - debug windows builds, due to not enough functions being inlined the - stack size might not handle the default recursion limit (1000). See - bpo-11105 for details.""" - if max_depth < 3: +def infinite_recursion(max_depth=None): + if max_depth is None: + # Pick a number large enough to cause problems + # but not take too long for code that can handle + # very deep recursion. + max_depth = 20_000 + elif max_depth < 3: raise ValueError("max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() depth = max(depth - 1, 1) # Ignore infinite_recursion() frame. From 3175e968bf5ecd19345e646fb74f00b35ae7f7d7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 4 Feb 2024 02:25:52 +0000 Subject: [PATCH 4/8] Account for address santizer when setting C recursion limit --- Lib/test/support/__init__.py | 2 ++ Lib/test/test_isinstance.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 8037019b9d7ef8..cb5a84aa74e05f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2374,6 +2374,8 @@ def adjust_int_max_str_digits(max_digits): C_RECURSION_LIMIT = 800 elif sys.platform.startswith('win'): C_RECURSION_LIMIT = 3000 + elif check_sanitizer(address=True): + C_RECURSION_LIMIT = 4000 else: C_RECURSION_LIMIT = 10000 diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index bf9332e40aeaf2..b3e317bd7990fd 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -352,7 +352,7 @@ def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its # argument will raise RecursionError eventually. tuple_arg = (compare_to,) - for cnt in range(support.EXCEEDS_RECURSION_LIMIT): + for cnt in range(support.C_RECURSION_LIMIT * 2): tuple_arg = (tuple_arg,) fxn(arg, tuple_arg) From efede6689c9e6c93d6095dec9cdbe8c0e3a3be09 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 4 Feb 2024 05:57:11 +0000 Subject: [PATCH 5/8] Add wasi test exception --- Lib/test/test_call.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index ec8dc29d36c16a..9e1e14f55513d2 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -932,6 +932,7 @@ def test_multiple_values(self): class TestRecursion(unittest.TestCase): @skip_on_s390x + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") def test_super_deep(self): def recurse(n): From acaf0d59cf132fa1a3db390067d5787f5da6ad02 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 4 Feb 2024 06:01:05 +0000 Subject: [PATCH 6/8] Add wasi test exception --- Lib/test/test_call.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 9e1e14f55513d2..5266ca3fb146aa 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,5 +1,5 @@ import unittest -from test.support import cpython_only, requires_limited_api, skip_on_s390x +from test.support import cpython_only, requires_limited_api, skip_on_s390x, is_wasi try: import _testcapi except ImportError: From 35c99cd65c21a7982a4f3ebc86b1f4c2bdd637ad Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 4 Feb 2024 09:39:16 +0000 Subject: [PATCH 7/8] Use correct value for wasi non-debug --- Include/cpython/pystate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 5ef4ebf03069ec..95fad893786d8d 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -259,7 +259,7 @@ struct _ts { # endif #else # if defined(__wasi__) -# define C_RECURSION_LIMIT 150 +# define C_RECURSION_LIMIT 500 # elif defined(__s390x__) # define C_RECURSION_LIMIT 800 # elif defined(_WIN32) From 8a1982128fde659295552775a9a6e2fba94a9275 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 4 Feb 2024 10:37:32 +0000 Subject: [PATCH 8/8] Add missing import --- Lib/test/test_call.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 5266ca3fb146aa..46abf40605f491 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,5 +1,5 @@ import unittest -from test.support import cpython_only, requires_limited_api, skip_on_s390x, is_wasi +from test.support import cpython_only, requires_limited_api, skip_on_s390x, is_wasi, Py_DEBUG try: import _testcapi except ImportError: