10000 GH-107263: Increase C stack limit for most functions, except `_PyEval… · python/cpython@fa45958 · GitHub
[go: up one dir, main page]
Skip to content

Commit fa45958

Browse files
markshannonmarkshannon
authored
GH-107263: Increase C stack limit for most functions, except _PyEval_EvalFrameDefault() (GH-107535)
* Set C recursion limit to 1500, set cost of eval loop to 2 frames, and compiler mutliply to 2.
1 parent 0bd784b commit fa45958

21 files changed

+57
-45
lines changed

‎Doc/whatsnew/3.12.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -801,6 +801,11 @@ sys
801801
exception instance, rather than to a ``(typ, exc, tb)`` tuple.
802802
(Contributed by Irit Katriel in :gh:`103176`.)
803803

804+
* :func:`sys.setrecursionlimit` and :func:`sys.getrecursionlimit`.
805+
The recursion limit now applies only to Python code. Builtin functions do
806+
not use the recursion limit, but are protected by a different mechanism
807+
that prevents recursion from causing a virtual machine crash.
808+
804809
tempfile
805810
--------
806811

‎Include/cpython/pystate.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,8 @@ struct _ts {
222222
# ifdef __wasi__
223223
# define C_RECURSION_LIMIT 500
224224
# else
225-
# define C_RECURSION_LIMIT 800
225+
// This value is duplicated in Lib/test/support/__init__.py
226+
# define C_RECURSION_LIMIT 1500
226227
# endif
227228
#endif
228229

‎Lib/test/list_tests.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from functools import cmp_to_key
77

88
from test import seq_tests
9-
from test.support import ALWAYS_EQ, NEVER_EQ
9+
from test.support import ALWAYS_EQ, NEVER_EQ, C_RECUR 8000 SION_LIMIT
1010

1111

1212
class CommonTest(seq_tests.CommonTest):
@@ -61,7 +61,7 @@ def test_repr(self):
6161

6262
def test_repr_deep(self):
6363
a = self.type2test([])
64-
for i in range(sys.getrecursionlimit() + 100):
64+
for i in range(C_RECURSION_LIMIT + 1):
6565
a = self.type2test([a])
6666
self.assertRaises(RecursionError, repr, a)
6767

‎Lib/test/mapping_tests.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import unittest
33
import collections
44
import sys
5+
from test.support import C_RECURSION_LIMIT
56

67

78
class BasicTestMappingProtocol(unittest.TestCase):
@@ -624,7 +625,7 @@ def __repr__(self):
624625

625626
def test_repr_deep(self):
626627
d = self._empty_mapping()
627-
for i in range(sys.getrecursionlimit() + 100):
628+
for i in range(C_RECURSION_LIMIT + 1):
628629
d0 = d
629630
d = self._empty_mapping()
630631
d[1] = d0

‎Lib/test/support/__init__.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@
6464
"run_with_tz", "PGO", "missing_compiler_executable",
6565
"ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST",
6666
"LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT",
67-
"Py_DEBUG", "EXCEEDS_RECURSION_LIMIT",
67+
"Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT",
68+
"skip_on_s390x",
6869
]
6970

7071

@@ -2460,3 +2461,10 @@ def adjust_int_max_str_digits(max_digits):
24602461

24612462
#For recursion tests, easily exceeds default recursion limit
24622463
EXCEEDS_RECURSION_LIMIT = 5000
2464+
2465+
# The default C recursion limit (from Include/cpython/pystate.h).
2466+
C_RECURSION_LIMIT = 1500
2467+
2468+
#Windows doesn't have os.uname() but it doesn't support s390x.
2469+
skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',
2470+
'skipped on s390x')

‎Lib/test/test_ast.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1084,6 +1084,7 @@ def next(self):
10841084
return self
10851085
enum._test_simple_enum(_Precedence, ast._Precedence)
10861086

1087+
@unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
10871088
@support.cpython_only
10881089
def test_ast_recursion_limit(self):
10891090
fail_depth = support.EXCEEDS_RECURSION_LIMIT

‎Lib/test/test_call.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import unittest
2-
from test.support import cpython_only, requires_limited_api
2+
from test.support import cpython_only, requires_limited_api, skip_on_s390x
33
try:
44
import _testcapi
55
except ImportError:
@@ -918,6 +918,7 @@ def test_multiple_values(self):
918918
@cpython_only
919919
class TestRecursion(unittest.TestCase):
920920

921+
@skip_on_s390x
921922
def test_super_deep(self):
922923

923924
def recurse(n):

‎Lib/test/test_compile.py

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,9 @@
1111
import warnings
1212
from test import support
1313
from test.support import (script_helper, requires_debug_ranges,
14-
requires_specialization)
14+
requires_specialization, C_RECURSION_LIMIT)
1515
from test.support.os_helper import FakePath
1616

17-
1817
class TestSpecifics(unittest.TestCase):
1918

2019
def compile_single(self, source):
@@ -112,7 +111,7 @@ def __getitem__(self, key):
112111

113112
@unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
114113
def test_extended_arg(self):
115-
repeat = 2000
114+
repeat = int(C_RECURSION_LIMIT * 0.9)
116115
longexpr = 'x = x or ' + '-x' * repeat
117116
g = {}
118117
code = textwrap.dedent('''
@@ -558,16 +557,12 @@ def test_yet_more_evil_still_undecodable(self):
558557
@support.cpython_only
559558
@unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
560559
def test_compiler_recursion_limit(self):
561-
# Expected limit is sys.getrecursionlimit() * the scaling factor
562-
# in symtable.c (currently 3)
563-
# We expect to fail *at* that limit, because we use up some of
564-
# the stack depth limit in the test suite code
565-
# So we check the expected limit and 75% of that
566-
# XXX (ncoghlan): duplicating the scaling factor here is a little
567-
# ugly. Perhaps it should be exposed somewhere...
568-
fail_depth = sys.getrecursionlimit() * 3
569-
crash_depth = sys.getrecursionlimit() * 300
570-
success_depth = int(fail_depth * 0.75)
560+
# Expected limit is C_RECURSION_LIMIT * 2
561+
# Duplicating the limit here is a little ugly.
562+
# Perhaps it should be exposed somewhere...
563+
fail_depth = C_RECURSION_LIMIT * 2 + 1
564+
crash_depth = C_RECURSION_LIMIT * 100
565+
success_depth = int(C_RECURSION_LIMIT * 1.8)
571566

572567
def check_limit(prefix, repeated, mode="single"):
573568
expect_ok = prefix + repeated * success_depth

‎Lib/test/test_dict.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import unittest
99
import weakref
1010
from test import support
11-
from test.support import import_helper
11+
from test.support import import_helper, C_RECURSION_LIMIT
1212

1313

1414
class DictTest(unittest.TestCase):
@@ -596,7 +596,7 @@ def __repr__(self):
596596

597597
def test_repr_deep(self):
598598
d = {}
599-
for i in range(sys.getrecursionlimit() + 100):
599+
for i in range(C_RECURSION_LIMIT + 1):
600600
d = {1: d}
601601
self.assertRaises(RecursionError, repr, d)
602602

‎Lib/test/test_dictviews.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import pickle
44
import sys
55
import unittest
6+
from test.support import C_RECURSION_LIMIT
67

78
class DictSetTest(unittest.TestCase):
89

@@ -279,7 +280,7 @@ def test_recursive_repr(self):
279280

280281
def test_deeply_nested_repr(self):
281282
d = {}
282-
for i in range(sys.getrecursionlimit() + 100):
283+
for i in range(C_RECURSION_LIMIT//2 + 100):
283284
d = {42: d.values()}
284285
self.assertRaises(RecursionError, repr, d)
285286

‎Lib/test/test_exception_group.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import collections.abc
22
import types
33
import unittest
4-
4+
from test.support import C_RECURSION_LIMIT
55

66
class TestExceptionGroupTypeHierarchy(unittest.TestCase):
77
def test_exception_group_types(self):
@@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self):
460460
class DeepRecursionInSplitAndSubgroup(unittest.TestCase):
461461
def make_deep_eg(self):
462462
e = TypeError(1)
463-
for i in range(2000):
463+
for i in range(C_RECURSION_LIMIT + 1):
464464
e = ExceptionGroup('eg', [e])
465465
return e
466466

‎Lib/test/test_zlib.py

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import pickle
88
import random
99
import sys
10-
from test.support import bigmemtest, _1G, _4G
10+
from test.support import bigmemtest, _1G, _4G, skip_on_s390x
1111

1212

1313
zlib = import_helper.import_module('zlib')
@@ -44,10 +44,7 @@
4444
# zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data
4545
#
4646
# Make the assumption that s390x always has an accelerator to simplify the skip
47-
# condition. Windows doesn't have os.uname() but it doesn't support s390x.
48-
skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',
49-
'skipped on s390x')
50-
47+
# condition.
5148

5249
class VersionTestCase(unittest.TestCase):
5350

‎Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Increase C recursion limit for functions other than the main interpreter
2+
from 800 to 1500. This should allow functions like ``list.__repr__`` and
3+
``json.dumps`` to handle all the inputs that they could prior to 3.12

‎Parser/asdl_c.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1393,7 +1393,7 @@ class PartingShots(StaticVisitor):
13931393
13941394
int starting_recursion_depth;
13951395
/* Be careful here to prevent overflow. */
1396-
int COMPILER_STACK_FRAME_SCALE = 3;
1396+
int COMPILER_STACK_FRAME_SCALE = 2;
13971397
PyThreadState *tstate = _PyThreadState_GET();
13981398
if (!tstate) {
13991399
return 0;

‎Python/Python-ast.c

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/ast.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1029,7 +1029,7 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps)
10291029

10301030

10311031
/* See comments in symtable.c. */
1032-
#define COMPILER_STACK_FRAME_SCALE 3
1032+
#define COMPILER_STACK_FRAME_SCALE 2
10331033

10341034
int
10351035
_PyAST_Validate(mod_ty mod)

‎Python/ast_opt.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1112,7 +1112,7 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat
11121112
#undef CALL_SEQ
11131113

11141114
/* See comments in symtable.c. */
1115-
#define COMPILER_STACK_FRAME_SCALE 3
1115+
#define COMPILER_STACK_FRAME_SCALE 2
11161116

11171117
int
11181118
_PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features)

‎Python/bytecodes.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -741,7 +741,7 @@ dummy_func(
741741
tstate->cframe = cframe.previous;
742742
assert(tstate->cframe->current_frame == frame->previous);
743743
assert(!_PyErr_Occurred(tstate));
744-
_Py_LeaveRecursiveCallTstate(tstate);
744+
tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS;
745745
return retval;
746746
}
747747

‎Python/ceval.c

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -624,6 +624,11 @@ extern const struct _PyCode_DEF(8) _Py_InitCleanup;
624624
# pragma warning(disable:4102)
625625
#endif
626626

627+
628+
/* _PyEval_EvalFrameDefault() is a *big* function,
629+
* so consume 3 units of C stack */
630+
#define PY_EVAL_C_STACK_UNITS 2
631+
627632
PyObject* _Py_HOT_FUNCTION
628633
_PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
629634
{
@@ -676,6 +681,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
676681
frame->previous = &entry_frame;
677682
cframe.current_frame = frame;
678683

684+
tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1);
679685
if (_Py_EnterRecursiveCallTstate(tstate, "")) {
680686
tstate->c_recursion_remaining--;
681687
tstate->py_recursion_remaining--;
@@ -907,7 +913,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
907913
/* Restore previous cframe and exit */
908914
tstate->cframe = cframe.previous;
909915
assert(tstate->cframe->current_frame == frame->previous);
910-
_Py_LeaveRecursiveCallTstate(tstate);
916+
tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS;
911917
return NULL;
912918
}
913919

‎Python/generated_cases.c.h

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/symtable.c

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -282,17 +282,10 @@ symtable_new(void)
282282
return NULL;
283283
}
284284

285-
/* When compiling the use of C stack is probably going to be a lot
286-
lighter than when executing Python code but still can overflow
287-
and causing a Python crash if not checked (e.g. eval("()"*300000)).
288-
Using the current recursion limit for the compiler seems too
289-
restrictive (it caused at least one test to fail) so a factor is
290-
used to allow deeper recursion when compiling an expression.
291-
292-
Using a scaling factor means this should automatically adjust when
285+
/* Using a scaling factor means this should automatically adjust when
293286
the recursion limit is adjusted for small or large C stack allocations.
294287
*/
295-
#define COMPILER_STACK_FRAME_SCALE 3
288+
#define COMPILER_STACK_FRAME_SCALE 2
296289

297290
struct symtable *
298291
_PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)

0 commit comments

Comments
 (0)
0