8000 GH-91079: Revert "GH-91079: Implement C stack limits using addresses,… · python/cpython@ef29104 · GitHub
[go: up one dir, main page]
Skip to content

Commit ef29104

Browse files
encukouencukou
authored
GH-91079: Revert "GH-91079: Implement C stack limits using addresses, not counters. (GH-130007)" for now (GH130413)
Revert "GH-91079: Implement C stack limits using addresses, not counters. (GH-130007)" for now Unfortunatlely, the change broke some buildbots. This reverts commit 2498c22.
1 parent 0ff1611 commit ef29104

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1464
-1218
lines changed

‎Doc/c-api/exceptions.rst

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -921,7 +921,11 @@ because the :ref:`call protocol <call>` takes care of recursion handling.
921921
922922
Marks a point where a recursive C-level call is about to be performed.
923923
924-
The function then checks if the stack limit is reached. If this is the
924+
If :c:macro:`!USE_STACKCHECK` is defined, this function checks if the OS
925+
stack overflowed using :c:func:`PyOS_CheckStack`. If this is the case, it
926+
sets a :exc:`MemoryError` and returns a nonzero value.
927+
928+
The function then checks if the recursion limit is reached. If this is the
925929
case, a :exc:`RecursionError` is set and a nonzero value is returned.
926930
Otherwise, zero is returned.
927931

‎Include/cpython/object.h

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -487,19 +487,18 @@ PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate);
487487
* we have headroom above the trigger limit */
488488
#define Py_TRASHCAN_HEADROOM 50
489489

490-
/* Helper function for Py_TRASHCAN_BEGIN */
491-
PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count);
492-
493490
#define Py_TRASHCAN_BEGIN(op, dealloc) \
494491
do { \
495492
PyThreadState *tstate = PyThreadState_Get(); \
496-
if (_Py_ReachedRecursionLimitWithMargin(tstate, 1) && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \
493+
if (tstate->c_recursion_remaining <= Py_TRASHCAN_HEADROOM && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \
497494
_PyTrash_thread_deposit_object(tstate, (PyObject *)op); \
498495
break; \
499-
}
496+
} \
497+
tstate->c_recursion_remaining--;
500498
/* The body of the deallocator is here. */
501499
#define Py_TRASHCAN_END \
502-
if (tstate->delete_later && !_Py_ReachedRecursionLimitWithMargin(tstate, 2)) { \
500+
tstate->c_recursion_remaining++; \
501+
if (tstate->delete_later && tstate->c_recursion_remaining > (Py_TRASHCAN_HEADROOM*2)) { \
503502
_PyTrash_thread_destroy_chain(tstate); \
504503
} \
505504
} while (0);

‎Include/cpython/pystate.h

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ struct _ts {
112112
int py_recursion_remaining;
113113
int py_recursion_limit;
114114

115-
int c_recursion_remaining; /* Retained for backwards compatibility. Do not use */
115+
int c_recursion_remaining;
116116
int recursion_headroom; /* Allow 50 more calls to handle any errors. */
117117

118118
/* 'tracing' keeps track of the execution depth when tracing/profiling.
@@ -202,7 +202,36 @@ struct _ts {
202202
PyObject *threading_local_sentinel;
203203
};
204204

205-
# define Py_C_RECURSION_LIMIT 5000
205+
#ifdef Py_DEBUG
206+
// A debug build is likely built with low optimization level which implies
207+
// higher stack memory usage than a release build: use a lower limit.
208+
# define Py_C_RECURSION_LIMIT 500
209+
#elif defined(__s390x__)
210+
# define Py_C_RECURSION_LIMIT 800
211+
#elif defined(_WIN32) && defined(_M_ARM64)
212+
# define Py_C_RECURSION_LIMIT 1000
213+
#elif defined(_WIN32)
214+
# define Py_C_RECURSION_LIMIT 3000
215+
#elif defined(__ANDROID__)
216+
// On an ARM64 emulator, API level 34 was OK with 10000, but API level 21
217+
// crashed in test_compiler_recursion_limit.
218+
# define Py_C_RECURSION_LIMIT 3000
219+
#elif defined(_Py_ADDRESS_SANITIZER)
220+
# define Py_C_RECURSION_LIMIT 4000
221+
#elif defined(__sparc__)
222+
// test_descr crashed on sparc64 with >7000 but let's keep a margin of error.
223+
# define Py_C_RECURSION_LIMIT 4000
224+
#elif defined(__wasi__)
225+
// Based on wasmtime 16.
226+
# define Py_C_RECURSION_LIMIT 5000
227+
#elif defined(__hppa__) || defined(__powerpc64__)
228+
// test_descr crashed with >8000 but let's keep a margin of error.
229+
# define Py_C_RECURSION_LIMIT 5000
230+
#else
231+
// This value is duplicated in Lib/test/support/__init__.py
232+
# define Py_C_RECURSION_LIMIT 10000
233+
#endif
234+
206235

207236
/* other API */
208237

@@ -217,6 +246,7 @@ _PyThreadState_UncheckedGet(void)
217246
return PyThreadState_GetUnchecked();
218247
}
219248

249+
220250
// Disable tracing and profiling.
221251
PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate);
222252

‎Include/internal/pycore_ceval.h

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -193,12 +193,18 @@ extern void _PyEval_DeactivateOpCache(void);
193193

194194
/* --- _Py_EnterRecursiveCall() ----------------------------------------- */
195195

196+
#ifdef USE_STACKCHECK
197+
/* With USE_STACKCHECK macro defined, trigger stack checks in
198+
_Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */
196199
static inline int _Py_MakeRecCheck(PyThreadState *tstate) {
197-
char here;
198-
uintptr_t here_addr = (uintptr_t)&here;
199-
_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
200-
return here_addr < _tstate->c_stack_soft_limit;
200+
return (tstate->c_recursion_remaining-- < 0
201+
|| (tstate->c_recursion_remaining & 63) == 0);
201202
}
203+
#else
204+
static inline int _Py_MakeRecCheck(PyThreadState *tstate) {
205+
return tstate->c_recursion_remaining-- < 0;
206+
}
207+
#endif
202208

203209
// Export for '_json' shared extension, used via _Py_EnterRecursiveCall()
204210
// static inline function.
@@ -214,31 +220,23 @@ static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate,
214220
return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where));
215221
}
216222

223+
static inline void _Py_EnterRecursiveCallTstateUnchecked(PyThreadState *tstate) {
224+
assert(tstate->c_recursion_remaining > 0);
225+
tstate->c_recursion_remaining--;
226+
}
227+
217228
static inline int _Py_EnterRecursiveCall(const char *where) {
218229
PyThreadState *tstate = _PyThreadState_GET();
219230
return _Py_EnterRecursiveCallTstate(tstate, where);
220231
}
221232

222-
static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) {
223-
(void)tstate;
224-
}
225-
226-
PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate);
227-
228-
static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) {
229-
char here;
230-
uintptr_t here_addr = (uintptr_t)&here;
231-
_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
232-
if (here_addr > _tstate->c_stack_soft_limit) {
233-
return 0;
234-
}
235-
if (_tstate->c_stack_hard_limit == 0) {
236-
_Py_InitializeRecursionLimits(tstate);
237-
}
238-
return here_addr <= _tstate->c_stack_soft_limit;
233+
static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) {
234+
tstate->c_recursion_remaining++;
239235
}
240236

241237
static inline void _Py_LeaveRecursiveCall(void) {
238+
PyThreadState *tstate = _PyThreadState_GET();
239+
_Py_LeaveRecursiveCallTstate(tstate);
242240
}
243241

244242
extern struct _PyInterpreterFrame* _PyEval_GetFrame(void);
@@ -329,6 +327,7 @@ void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
329327

330328
PyAPI_FUNC(PyObject *) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value);
331329

330+
332331
#ifdef __cplusplus
333332
}
334333
#endif

‎Include/internal/pycore_symtable.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ struct symtable {
8282
PyObject *st_private; /* name of current class or NULL */
8383
_PyFutureFeatures *st_future; /* module's future features that affect
8484
the symbol table */
85+
int recursion_depth; /* current recursion depth */
86+
int recursion_limit; /* recursion limit */
8587
};
8688

8789
typedef struct _symtable_entry {

‎Include/internal/pycore_tstate.h

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,6 @@ typedef struct _PyThreadStateImpl {
2121
// semi-public fields are in PyThreadState.
2222
PyThreadState base;
2323

24-
// These are addresses, but we need to convert to ints to avoid UB.
25-
uintptr_t c_stack_top;
26-
uintptr_t c_stack_soft_limit;
27-
uintptr_t c_stack_hard_limit;
28-
2924
PyObject *asyncio_running_loop; // Strong reference
3025
PyObject *asyncio_running_task; // Strong reference
3126

‎Include/pythonrun.h

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,14 @@ PyAPI_FUNC(void) PyErr_DisplayException(PyObject *);
2121
/* Stuff with no proper home (yet) */
2222
PyAPI_DATA(int) (*PyOS_InputHook)(void);
2323

24-
/* Stack size, in "pointers". This must be large enough, so
25-
* no two calls to check recursion depth are more than this far
26-
* apart. In practice, that means it must be larger than the C
27-
* stack consumption of PyEval_EvalDefault */
28-
#if defined(Py_DEBUG) && defined(WIN32)
29-
# define PYOS_STACK_MARGIN 3072
30-
#else
31-
# define PYOS_STACK_MARGIN 2048
32-
#endif
33-
#define PYOS_STACK_MARGIN_BYTES (PYOS_STACK_MARGIN * sizeof(void *))
34-
35-
#if defined(WIN32)
24+
/* Stack size, in "pointers" (so we get extra safety margins
25+
on 64-bit platforms). On a 32-bit platform, this translates
26+
to an 8k margin. */
27+
#define PYOS_STACK_MARGIN 2048
28+
29+
#if defined(WIN32) && !defined(MS_WIN64) && !defined(_M_ARM) && defined(_MSC_VER) && _MSC_VER >= 1300
30+
/* Enable stack checking under Microsoft C */
31+
// When changing the platforms, ensure PyOS_CheckStack() docs are still correct
3632
#define USE_STACKCHECK
3733
#endif
3834

‎Lib/test/list_tests.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@
66
from functools import cmp_to_key
77

88
from test import seq_tests
9-
from test.support import ALWAYS_EQ, NEVER_EQ
10-
from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow
9+
from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit, skip_emscripten_stack_overflow
1110

1211

1312
class CommonTest(seq_tests.CommonTest):
@@ -60,11 +59,10 @@ def test_repr(self):
6059
self.assertEqual(str(a2), "[0, 1, 2, [...], 3]")
6160
self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]")
6261

63-
@skip_wasi_stack_overflow()
6462
@skip_emscripten_stack_overflow()
6563
def test_repr_deep(self):
6664
a = self.type2test([])
67-
for i in range(100_000):
65+
for i in range(get_c_recursion_limit() + 1):
6866
a = self.type2test([a])
6967
self.assertRaises(RecursionError, repr, a)
7068

‎Lib/test/mapping_tests.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# tests common to dict and UserDict
22
import unittest
33
import collections
4-
from test import support
4+
from test.support import get_c_recursion_limit, skip_emscripten_stack_overflow
55

66

77
class BasicTestMappingProtocol(unittest.TestCase):
@@ -622,11 +622,10 @@ def __repr__(self):
622622
d = self._full_mapping({1: BadRepr()})
623623
self.assertRaises(Exc, repr, d)
624624

625-
@support.skip_wasi_stack_overflow()
626-
@support.skip_emscripten_stack_overflow()
625+
@skip_emscripten_stack_overflow()
627626
def test_repr_deep(self):
628627
d = self._empty_mapping()
629-
for i in range(support.exceeds_recursion_limit()):
628+
for i in range(get_c_recursion_limit() + 1):
630629
d0 = d
631630
d = self._empty_mapping()
632631
d[1] = d0

‎Lib/test/pythoninfo.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -684,6 +684,7 @@ def collect_testcapi(info_add):
684684
for name in (
685685
'LONG_MAX', # always 32-bit on Windows, 64-bit on 64-bit Unix
686686
'PY_SSIZE_T_MAX',
687+
'Py_C_RECURSION_LIMIT',
687688
'SIZEOF_TIME_T', # 32-bit or 64-bit depending on the platform
688689
'SIZEOF_WCHAR_T', # 16-bit or 32-bit depending on the platform
689690
):

‎Lib/test/support/__init__.py

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@
5656
"run_with_tz", "PGO", "missing_compiler_executable",
5757
"ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST",
5858
"LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT",
59-
"Py_DEBUG", "exceeds_recursion_limit", "skip_on_s390x",
59+
"Py_DEBUG", "exceeds_recursion_limit", "get_c_recursion_limit",
60+
"skip_on_s390x",
6061
"requires_jit_enabled",
6162
"requires_jit_disabled",
6263
"force_not_colorized",
@@ -557,9 +558,6 @@ def skip_android_selinux(name):
557558
def skip_emscripten_stack_overflow():
558559
return unittest.skipIf(is_emscripten, "Exhausts limited stack on Emscripten")
559560

560-
def skip_wasi_stack_overflow():
561-
return unittest.skipIf(is_wasi, "Exhausts stack on WASI")
562-
563561
is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"}
564562
is_apple = is_apple_mobile or sys.platform == "darwin"
565563

@@ -2626,9 +2624,17 @@ def adjust_int_max_str_digits(max_digits):
26262624
sys.set_int_max_str_digits(current)
26272625

26282626

2627+
def get_c_recursion_limit():
2628+
try:
2629+
import _testcapi
2630+
return _testcapi.Py_C_RECURSION_LIMIT
2631+
except ImportError:
2632+
raise unittest.SkipTest('requires _testcapi')
2633+
2634+
26292635
def exceeds_recursion_limit():
26302636
"""For recursion tests, easily exceeds default recursion limit."""
2631-
return 100_000
2637+
return get_c_recursion_limit() * 3
26322638

26332639

26342640
# Windows doesn't have os.uname() but it doesn't support s390x.

‎Lib/test/test_ast/test_ast.py

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@
1818
_testinternalcapi = None
1919

2020
from test import support
21-
from test.support import os_helper, script_helper
22-
from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow
21+
from test.support import os_helper, script_helper, skip_emscripten_stack_overflow
2322
from test.support.ast_helper import ASTTestMixin
2423
from test.test_ast.utils import to_tuple
2524
from test.test_ast.snippets import (
@@ -751,25 +750,25 @@ def next(self):
751750
enum._test_simple_enum(_Precedence, ast._Precedence)
752751

753752
@support.cpython_only
754-
@skip_wasi_stack_overflow()
755753
@skip_emscripten_stack_overflow()
756754
def test_ast_recursion_limit(self):
757-
crash_depth = 200_000
758-
success_depth = 200
755+
fail_depth = support.exceeds_recursion_limit()
756+
crash_depth = 100_000
757+
success_depth = int(support.get_c_recursion_limit() * 0.8)
759758
if _testinternalcapi is not None:
760759
remaining = _testinternalcapi.get_c_recursion_remaining()
761760
success_depth = min(success_depth, remaining)
762761

763762
def check_limit(prefix, repeated):
764763
expect_ok = prefix + repeated * success_depth
765764
ast.parse(expect_ok)
766-
767-
broken = prefix + repeated * crash_depth
768-
details = "Compiling ({!r} + {!r} * {})".format(
769-
prefix, repeated, crash_depth)
770-
with self.assertRaises(RecursionError, msg=details):
771-
with support.infinite_recursion():
772-
ast.parse(broken)
765+
for depth in (fail_depth, crash_depth):
766+
broken = prefix + repeated * depth
767+
details = "Compiling ({!r} + {!r} * {})".format(
768+
prefix, repeated, depth)
769+
with self.assertRaises(RecursionError, msg=details):
770+
with support.infinite_recursion():
771+
ast.parse(broken)
773772

774773
check_limit("a", "()")
775774
check_limit("a", ".b")

‎Lib/test/test_call.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import unittest
22
from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG,
3-
set_recursion_limit, skip_on_s390x, exceeds_recursion_limit, skip_emscripten_stack_overflow,
3+
set_recursion_limit, skip_on_s390x, skip_emscripten_stack_overflow,
44
skip_if_sanitizer, import_helper)
55
try:
66
import _testcapi
@@ -1064,10 +1064,10 @@ def c_py_recurse(m):
10641064
recurse(90_000)
10651065
with self.assertRaises(RecursionError):
10661066
recurse(101_000)
1067-
c_recurse(50)
1067+
c_recurse(100)
10681068
with self.assertRaises(RecursionError):
10691069
c_recurse(90_000)
1070-
c_py_recurse(50)
1070+
c_py_recurse(90)
10711071
with self.assertRaises(RecursionError):
10721072
c_py_recurse(100_000)
10731073

‎Lib/test/test_capi/test_misc.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@ def test_trashcan_subclass(self):
408408
# activated when its tp_dealloc is being called by a subclass
409409
from _testcapi import MyList
410410
L = None
411-
for i in range(100):
411+
for i in range(1000):
412412
L = MyList((L,))
413413

414414
@support.requires_resource('cpu')

0 commit comments

Comments
 (0)
0