8000 gh-132775: Add _PyCode_VerifyStateless() (gh-133221) · python/cpython@d270bb5 · GitHub
[go: up one dir, main page]
Skip to content

Commit d270bb5

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-132775: Add _PyCode_VerifyStateless() (gh-133221)
"Stateless" code is a function or code object which does not rely on external state or internal state. It may rely on arguments and builtins, but not globals or a closure. I've left a comment in pycore_code.h that provides more detail. We also add _PyFunction_VerifyStateless(). The new functions will be used in several later changes that facilitate "sharing" functions and code objects between interpreters.
1 parent f610bbd commit d270bb5

File tree

8 files changed

+441
-37
lines changed

8 files changed

+441
-37
lines changed

‎Include/internal/pycore_code.h‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -621,6 +621,47 @@ PyAPI_FUNC(int) _PyCode_SetUnboundVarCounts(
621621
PyObject *globalsns,
622622
PyObject *builtinsns);
623623

624+
625+
/* "Stateless" code is a function or code object which does not rely on
626+
* external state or internal state. It may rely on arguments and
627+
* builtins, but not globals or a closure. Thus it does not rely
628+
* on __globals__ or __closure__, and a stateless function
629+
* is equivalent to its code object.
630+
*
631+
* Stateless code also does not keep any persistent state
632+
* of its own, so it can't have any executors, monitoring,
633+
* instrumentation, or "extras" (i.e. co_extra).
634+
*
635+
* Stateless code may create nested functions, including closures.
636+
* However, nested functions must themselves be stateless, except they
637+
* *can* close on the enclosing locals.
638+
*
639+
* Stateless code may return any value, including nested functions and closures.
640+
*
641+
* Stateless code that takes no arguments and doesn't return anything
642+
* may be treated like a script.
643+
*
644+
* We consider stateless code to be "portable" if it does not return any
645+
* any object that holds a reference to any of the code's locals. Thus
646+
* generators and coroutines are not portable. Likewise a function
647+
* that returns a closure is not portable. The concept of
648+
* portability is useful in cases where the code is run
649+
* in a different execution context than where
650+
* the return value will be used. */
651+
652+
PyAPI_FUNC(int) _PyCode_CheckNoInternalState(PyCodeObject *, const char **);
653+
PyAPI_FUNC(int) _PyCode_CheckNoExternalState(
654+
PyCodeObject *,
655+
_PyCode_var_counts_t *,
656+
const char **);
657+
PyAPI_FUNC(int) _PyCode_VerifyStateless(
658+
PyThreadState *,
659+
PyCodeObject *,
660+
PyObject *globalnames,
661+
PyObject *globalsns,
662+
PyObject *builtinsns);
663+
664+
PyAPI_FUNC(int) _PyCode_CheckPureFunction(PyCodeObject *, const char **);
624665
PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *);
625666

626667

‎Include/internal/pycore_function.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,13 @@ PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_cod
3535
extern PyObject *_Py_set_function_type_params(
3636
PyThreadState* unused, PyObject *func, PyObject *type_params);
3737

38+
39+
/* See pycore_code.h for explanation about what "stateless" means. */
40+
41+
PyAPI_FUNC(int)
42+
_PyFunction_VerifyStateless(PyThreadState *, PyObject *);
43+
44+
3845
#ifdef __cplusplus
3946
}
4047
#endif

‎Include/internal/pycore_opcode_utils.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ extern "C" {
5656

5757
#define IS_RETURN_OPCODE(opcode) \
5858
(opcode == RETURN_VALUE)
59+
#define IS_RAISE_OPCODE(opcode) \
60+
(opcode == RAISE_VARARGS || opcode == RERAISE)
5961

6062

6163
/* Flags used in the oparg for MAKE_FUNCTION */

‎Lib/test/_code_definitions.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,40 @@ def spam_minimal():
1212
return
1313

1414

15+
def spam_with_builtins():
16+
x = 42
17+
values = (42,)
18+
checks = tuple(callable(v) for v in values)
19+
res = callable(values), tuple(values), list(values), checks
20+
print(res)
21+
22+
23+
def spam_with_globals_and_builtins():
24+
func1 = spam
25+
func2 = spam_minimal
26+
funcs = (func1, func2)
27+
checks = tuple(callable(f) for f in funcs)
28+
res = callable(funcs), tuple(funcs), list(funcs), checks
29+
print(res)
30+
31+
32+
def spam_returns_arg(x):
33+
return x
34+
35+
36+
def spam_with_inner_not_closure():
37+
def eggs():
38+
pass
39+
eggs()
40+
41+
42+
def spam_with_inner_closure():
43+
x = 42
44+
def eggs():
45+
print(x)
46+
eggs()
47+
48+
1549
def spam_full(a, b, /, c, d:int=1, *args, e, f:object=None, **kwargs) -> tuple:
1650
# arg defaults, kwarg defaults
1751
# annotations
@@ -98,6 +132,11 @@ def ham_C_closure(z):
98132
TOP_FUNCTIONS = [
99133
# shallow
100134
spam_minimal,
135+
spam_with_builtins,
136+
spam_with_globals_and_builtins,
137+
spam_returns_arg,
138+
spam_with_inner_not_closure,
139+
spam_with_inner_closure,
101140
spam_full,
102141
spam,
103142
# outer func
@@ -127,6 +166,30 @@ def ham_C_closure(z):
127166
*NESTED_FUNCTIONS,
128167
]
129168

169+
STATELESS_FUNCTIONS = [
170+
spam,
171+
spam_minimal,
172+
spam_with_builtins,
173+
spam_returns_arg,
174+
spam_with_inner_not_closure,
175+
spam_with_inner_closure,
176+
spam_N,
177+
spam_C,
178+
spam_NN,
179+
spam_NC,
180+
spam_CN,
181+
spam_CC,
182+
eggs_nested,
183+
eggs_nested_N,
184+
ham_nested,
185+
ham_C_nested
186+
]
187+
STATELESS_CODE = [
188+
*STATELESS_FUNCTIONS,
189+
spam_with_globals_and_builtins,
190+
spam_full,
191+
]
192+
130193

131194
# generators
132195

‎Lib/test/test_code.py‎

Lines changed: 75 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,7 @@
220220
import _testinternalcapi
221221
except ModuleNotFoundError:
222222
_testinternalcapi = None
223+
import test._code_definitions as defs
223224

224225
COPY_FREE_VARS = opmap['COPY_FREE_VARS']
225226

@@ -671,9 +672,31 @@ def test_local_kinds(self):
671672
VARARGS = CO_FAST_LOCAL | CO_FAST_ARG_VAR | CO_FAST_ARG_POS
672673
VARKWARGS = CO_FAST_LOCAL | CO_FAST_ARG_VAR | CO_FAST_ARG_KW
673674

674-
import test._code_definitions as defs
675675
funcs = {
676676
defs.spam_minimal: {},
677+
defs.spam_with_builtins: {
678+
'x': CO_FAST_LOCAL,
679+
'values': CO_FAST_LOCAL,
680+
'checks': CO_FAST_LOCAL,
681+
'res': CO_FAST_LOCAL,
682+
},
683+
defs.spam_with_globals_and_builtins: {
684+
'func1': CO_FAST_LOCAL,
685+
'func2': CO_FAST_LOCAL,
686+
'funcs': CO_FAST_LOCAL,
687+
'checks': CO_FAST_LOCAL,
688+
'res': CO_FAST_LOCAL,
689+
},
690+
defs.spam_returns_arg: {
691+
'x': POSORKW,
692+
},
693+
defs.spam_with_inner_not_closure: {
694+
'eggs': CO_FAST_LOCAL,
695+
},
696+
defs.spam_with_inner_closure: {
697+
'x': CO_FAST_CELL,
698+
'eggs': CO_FAST_LOCAL,
699+
},
677700
defs.spam_full: {
678701
'a': POSONLY,
679702
'b': POSONLY,
@@ -859,9 +882,26 @@ def new_var_counts(*,
859882
},
860883
}
861884

862-
import test._code_definitions as defs
863885
funcs = {
864886
defs.spam_minimal: new_var_counts(),
887+
defs.spam_with_builtins: new_var_counts(
888+
purelocals=4,
889+
globalvars=4,
890+
),
891+
defs.spam_with_globals_and_builtins: new_var_counts(
892+
purelocals=5,
893+
globalvars=6,
894+
),
895+
defs.spam_returns_arg: new_var_counts(
896+
posorkw=1,
897+
),
898+
defs.spam_with_inner_not_closure: new_var_counts(
899+
purelocals=1,
900+
),
901+
defs.spam_with_inner_closure: new_var_counts(
902+
othercells=1,
903+
purelocals=1,
904+
),
865905
defs.spam_full: new_var_counts(
866906
posonly=2,
867907
posorkw=2,
@@ -958,55 +998,70 @@ def new_var_counts(*,
958998
counts = _testinternalcapi.get_code_var_counts(func.__code__)
959999
self.assertEqual(counts, expected)
9601000

961-
def func_with_globals_and_builtins():
962-
mod1 = _testinternalcapi
963-
mod2 = dis
964-
mods = (mod1, mod2)
965-
checks = tuple(callable(m) for m in mods)
966-
return callable(mod2), tuple(mods), list(mods), checks
967-
968-
func = func_with_globals_and_builtins
1001+
func = defs.spam_with_globals_and_builtins
9691002
with self.subTest(f'{func} code'):
9701003
expected = new_var_counts(
971-
purelocals=4,
972-
globalvars=5,
1004+
purelocals=5,
1005+
globalvars=6,
9731006
)
9741007
counts = _testinternalcapi.get_code_var_counts(func.__code__)
9751008
self.assertEqual(counts, expected)
9761009

9771010
with self.subTest(f'{func} with own globals and builtins'):
9781011
expected = new_var_counts(
979-
purelocals=4,
980-
globalvars=(2, 3),
1012+
purelocals=5,
1013+
globalvars=(2, 4),
9811014
)
9821015
counts = _testinternalcapi.get_code_var_counts(func)
9831016
self.assertEqual(counts, expected)
9841017

9851018
with self.subTest(f'{func} without globals'):
9861019
expected = new_var_counts(
987-
purelocals=4,
988-
globalvars=(0, 3, 2),
1020+
purelocals=5,
1021+
globalvars=(0, 4, 2),
9891022
)
9901023
counts = _testinternalcapi.get_code_var_counts(func, globalsns={})
9911024
self.assertEqual(counts, expected)
9921025

9931026
with self.subTest(f'{func} without both'):
9941027
expected = new_var_counts(
995-
purelocals=4,
996-
globalvars=5,
1028+
purelocals=5,
1029+
globalvars=6,
9971030
)
9981031
counts = _testinternalcapi.get_code_var_counts(func, globalsns={},
9991032
builtinsns={})
10001033
self.assertEqual(counts, expected)
10011034

10021035
with self.subTest(f'{func} without builtins'):
10031036
expected = new_var_counts(
1004-
purelocals=4,
1005-
globalvars=(2, 0, 3),
1037+
purelocals=5,
1038+
globalvars=(2, 0, 4),
10061039
)
10071040
counts = _testinternalcapi.get_code_var_counts(func, builtinsns={})
10081041
self.assertEqual(counts, expected)
10091042

1043+
@unittest.skipIf(_testinternalcapi is None, "missing _testinternalcapi")
1044+
def test_stateless(self):
1045+
self.maxDiff = None
1046 4EB8 +
1047+
for func in defs.STATELESS_CODE:
1048+
with self.subTest((func, '(code)')):
1049+
_testinternalcapi.verify_stateless_code(func.__code__)
1050+
for func in defs.STATELESS_FUNCTIONS:
1051+
with self.subTest((func, '(func)')):
1052+
_testinternalcapi.verify_stateless_code(func)
1053+
1054+
for func in defs.FUNCTIONS:
1055+
if func not in defs.STATELESS_CODE:
1056+
with self.subTest((func, '(code)')):
1057+
with self.assertRaises(Exception):
1058+
_testinternalcapi.verify_stateless_code(func.__code__)
1059+
1060+
if func not in defs.STATELESS_FUNCTIONS:
1061+
with self.subTest((func, '(func)')):
1062+
with self.assertRaises(Exception):
1063+
_testinternalcapi.verify_stateless_code(func)
1064+
10101065

10111066
def isinterned(s):
10121067
return s is sys.intern(('_' + s + '_')[1:-1])

‎Modules/_testinternalcapi.c‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1165,6 +1165,48 @@ get_code_var_counts(PyObject *self, PyObject *_args, PyObject *_kwargs)
11651165
return NULL;
11661166
}
11671167

1168+
static PyObject *
1169+
verify_stateless_code(PyObject *self, PyObject *args, PyObject *kwargs)
1170+
{
1171+
PyThreadState *tstate = _PyThreadState_GET();
1172+
PyObject *codearg;
1173+
PyObject *globalnames = NULL;
1174+
PyObject *globalsns = NULL;
1175+
PyObject *builtinsns = NULL;
1176+
static char *kwlist[] = {"code", "globalnames",
1177+
"globalsns", "builtinsns", NULL};
1178+
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
1179+
"O|O!O!O!:get_code_var_counts", kwlist,
1180+
&codearg, &PySet_Type, &globalnames,
1181+
&PyDict_Type, &globalsns, &PyDict_Type, &builtinsns))
1182+
{
1183+
return NULL;
1184+
}
1185+
if (PyFunction_Check(codearg)) {
1186+
if (globalsns == NULL) {
1187+
globalsns = PyFunction_GET_GLOBALS(codearg);
1188+
}
1189+
if (builtinsns == NULL) {
1190+
builtinsns = PyFunction_GET_BUILTINS(codearg);
1191+
}
1192+
codearg = PyFunction_GET_CODE(codearg);
1193+
}
1194+
else if (!PyCode_Check(codearg)) {
1195+
PyErr_SetString(PyExc_TypeError,
1196+
"argument must be a code object or a function");
1197+
return NULL;
1198+
}
1199+
PyCodeObject *code = (PyCodeObject *)codearg;
1200+
1201+
if (_PyCode_VerifyStateless(
1202+
tstate, code, globalnames, globalsns, builtinsns) < 0)
1203+
{
1204+
return NULL;
1205+
}
1206+
Py_RETURN_NONE;
1207+
}
1208+
1209+
11681210
static PyObject *
11691211
jit_enabled(PyObject *self, PyObject *arg)
11701212
{
@@ -2293,6 +2335,8 @@ static PyMethodDef module_functions[] = {
22932335
{"get_co_localskinds", get_co_localskinds, METH_O, NULL},
22942336
{"get_code_var_counts", _PyCFunction_CAST(get_code_var_counts),
22952337
METH_VARARGS | METH_KEYWORDS, NULL},
2338+
{"verify_stateless_code", _PyCFunction_CAST(verify_stateless_code),
2339+
METH_VARARGS | METH_KEYWORDS, NULL},
22962340
{"jit_enabled", jit_enabled, METH_NOARGS, NULL},
22972341
#ifdef _Py_TIER2
22982342
{"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL},

0 commit comments

Comments
 (0)
0