8000 gh-98608: Change _Py_NewInterpreter() to _Py_NewInterpreterFromConfig… · python/cpython@f323694 · GitHub
[go: up one dir, main page]
Skip to content

Commit f323694

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-98608: Change _Py_NewInterpreter() to _Py_NewInterpreterFromConfig() (gh-98609)
(see #98608) This change does the following: 1. change the argument to a new `_PyInterpreterConfig` struct 2. rename the function to `_Py_NewInterpreterFromConfig()`, inspired by `Py_InitializeFromConfig()` (takes a `_PyInterpreterConfig` instead of `isolated_subinterpreter`) 3. split up the boolean `isolated_subinterpreter` into the corresponding multiple granular settings * allow_fork * allow_subprocess * allow_threads 4. add `PyInterpreterState.feature_flags` to store those settings 5. add a function for checking if a feature is enabled on an opaque `PyInterpreterState *` 6. drop `PyConfig._isolated_interpreter` The existing default (see `Py_NewInterpeter()` and `Py_Initialize*()`) allows fork, subprocess, and threads and the optional "isolated" interpreter (see the `_xxsubinterpreters` module) disables all three. None of that changes here; the defaults are preserved. Note that the given `_PyInterpreterConfig` will not be used outside `_Py_NewInterpreterFromConfig()`, nor preserved. This contrasts with how `PyConfig` is currently preserved, used, and even modified outside `Py_InitializeFromConfig()`. I'd rather just avoid that mess from the start for `_PyInterpreterConfig`. We can preserve it later if we find an actual need. This change allows us to follow up with a number of improvements (e.g. stop disallowing subprocess and support disallowing exec instead). (Note that this PR adds "private" symbols. We'll probably make them public, and add docs, in a separate change.)
1 parent 24c56b4 commit f323694

21 files changed

+295
-39
lines changed

‎Doc/c-api/init_config.rst

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1571,8 +1571,6 @@ Private provisional API:
15711571
15721572
* :c:member:`PyConfig._init_main`: if set to ``0``,
15731573
:c:func:`Py_InitializeFromConfig` stops at the "Core" initialization phase.
1574-
* :c:member:`PyConfig._isolated_interpreter`: if non-zero,
1575-
disallow threads, subprocesses and fork.
15761574
15771575
.. c:function:: PyStatus _Py_InitializeMain(void)
15781576

‎Include/cpython/initconfig.h

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -213,10 +213,6 @@ typedef struct PyConfig {
213213
// If equal to 0, stop Python initialization before the "main" phase.
214214
int _init_main;
215215

216-
// If non-zero, disallow threads, subprocesses, and fork.
217-
// Default: 0.
218-
int _isolated_interpreter;
219-
220216
// If non-zero, we believe we're running from a source tree.
221217
int _is_python_build;
222218
} PyConfig;
@@ -245,6 +241,21 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config,
245241
Py_ssize_t length, wchar_t **items);
246242

247243

244+
/* --- PyInterpreterConfig ------------------------------------ */
245+
246+
typedef struct {
247+
int allow_fork;
248+
int allow_subprocess;
249+
int allow_threads;
250+
} _PyInterpreterConfig;
251+
252+
#define _PyInterpreterConfig_LEGACY_INIT \
253+
{ \
254+
.allow_fork = 1, \
255+
.allow_subprocess = 1, \
256+
.allow_threads = 1, \
257+
}
258+
248259
/* --- Helper functions --------------------------------------- */
249260

250261
/* Get the original command line arguments, before Python modified them.

‎Include/cpython/pylifecycle.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,4 +62,5 @@ PyAPI_FUNC(int) _Py_CoerceLegacyLocale(int warn);
6262
PyAPI_FUNC(int) _Py_LegacyLocaleDetected(int warn);
6363
PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category);
6464

65-
PyAPI_FUNC(PyThreadState *) _Py_NewInterpreter(int isolated_subinterpreter);
65+
PyAPI_FUNC(PyThreadState *) _Py_NewInterpreterFromConfig(
66+
const _PyInterpreterConfig *);

‎Include/cpython/pystate.h

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,38 @@
33
#endif
44

55

6+
/*
7+
Runtime Feature Flags
8+
9+
Each flag indicate whether or not a specific runtime feature
10+
is available in a given context. For example, forking the process
11+
might not be allowed in the current interpreter (i.e. os.fork() would fail).
12+
*/
13+
14+
// We leave the first 10 for less-specific features.
15+
16+
/* Set if threads are allowed. */
17+
#define Py_RTFLAGS_THREADS (1UL << 10)
18+
19+
/* Set if os.fork() is allowed. */
20+
#define Py_RTFLAGS_FORK (1UL << 15)
21+
22+
/* Set if subprocesses are allowed. */
23+
#define Py_RTFLAGS_SUBPROCESS (1UL << 16)
24+
25+
26+
PyAPI_FUNC(int) _PyInterpreterState_HasFeature(PyInterpreterState *interp,
27+
unsigned long feature);
28+
29+
30+
/* private interpreter helpers */
31+
632
PyAPI_FUNC(int) _PyInterpreterState_RequiresIDRef(PyInterpreterState *);
733
PyAPI_FUNC(void) _PyInterpreterState_RequireIDRef(PyInterpreterState *, int);
834

935
PyAPI_FUNC(PyObject *) _PyInterpreterState_GetMainModule(PyInterpreterState *);
1036

37+
1138
/* State unique per thread */
1239

1340
/* Py_tracefunc return -1 when raising an exception, or 0 for success. */

‎Include/internal/pycore_interp.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ struct _is {
143143
#ifdef HAVE_DLOPEN
144144
int dlopenflags;
145145
#endif
146+
unsigned long feature_flags;
146147

147148
PyObject *dict; /* Stores per-interpreter state */
148149

‎Lib/test/_test_embed_set_config.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,6 @@ def test_set_invalid(self):
8484
'skip_source_first_line',
8585
'_install_importlib',
8686
'_init_main',
87-
'_isolated_interpreter',
8887
]
8988
if MS_WINDOWS:
9089
options.append('legacy_windows_stdio')

‎Lib/test/support/__init__.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1793,6 +1793,22 @@ def run_in_subinterp(code):
17931793
Run code in a subinterpreter. Raise unittest.SkipTest if the tracemalloc
17941794
module is enabled.
17951795
"""
1796+
_check_tracemalloc()
1797+
import _testcapi
1798+
return _testcapi.run_in_subinterp(code)
1799+
1800+
1801+
def run_in_subinterp_with_config(code, **config):
1802+
"""
1803+
Run code in a subinterpreter. Raise unittest.SkipTest if the tracemalloc
1804+
module is enabled.
1805+
"""
1806+
_check_tracemalloc()
1807+
import _testcapi
1808+
return _testcapi.run_in_subinterp_with_config(code, **config)
1809+
1810+
1811+
def _check_tracemalloc():
17961812
# Issue #10915, #15751: PyGILState_*() functions don't work with
17971813
# sub-interpreters, the tracemalloc module uses these functions internally
17981814
try:
@@ -1804,8 +1820,6 @@ def run_in_subinterp(code):
18041820
raise unittest.SkipTest("run_in_subinterp() cannot be used "
18051821
"if tracemalloc module is tracing "
18061822
"memory allocations")
1807-
import _testcapi
1808-
return _testcapi.run_in_subinterp(code)
18091823

18101824

18111825
def check_free_after_iterating(test, iter, cls, args=()):

‎Lib/test/test_capi.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1096,6 +1096,45 @@ def test_py_config_isoloated_per_interpreter(self):
10961096
# test fails, assume that the environment in this process may
10971097
# be altered and suspect.
10981098

1099+
@unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
1100+
def test_configured_settings(self):
1101+
"""
1102+
The config with which an interpreter is created corresponds
1103+
1-to-1 with the new interpreter's settings. This test verifies
1104+
that they match.
1105+
"""
1106+
import json
1107+
1108+
THREADS = 1<<10
1109+
FORK = 1<<15
1110+
SUBPROCESS = 1<<16
1111+
1112+
features = ['fork', 'subprocess', 'threads']
1113+
kwlist = [f'allow_{n}' for n in features]
1114+
for config, expected in {
1115+
(True, True, True): FORK | SUBPROCESS | THREADS,
1116+
(False, False, False): 0,
1117+
(False, True, True): SUBPROCESS | THREADS,
1118+
}.items():
1119+
kwargs = dict(zip(kwlist, config))
1120+
expected = {
1121+
'feature_flags': expected,
1122+
}
1123+
with self.subTest(config):
1124+
r, w = os.pipe()
1125+
script = textwrap.dedent(f'''
1126+
import _testinternalcapi, json, os
1127+
settings = _testinternalcapi.get_interp_settings()
1128+
with os.fdopen({w}, "w") as stdin:
1129+
json.dump(settings, stdin)
1130+
''')
1131+
with os.fdopen(r) as stdout:
1132+
support.run_in_subinterp_with_config(script, **kwargs)
1133+
out = stdout.read()
1134+
settings = json.loads(out)
1135+
1136+
self.assertEqual(settings, expected)
1137+
10991138
def test_mutate_exception(self):
11001139
"""
11011140
Exceptions saved in global module state get shared between

‎Lib/test/test_embed.py

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -496,7 +496,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
496496
'check_hash_pycs_mode': 'default',
497497
'pathconfig_warnings': 1,
498498
'_init_main': 1,
499-
'_isolated_interpreter': 0,
500499
'use_frozen_modules': not support.Py_DEBUG,
501500
'safe_path': 0,
502501
'_is_python_build': IGNORE_CONFIG,
@@ -881,8 +880,6 @@ def test_init_from_config(self):
881880

882881
'check_hash_pycs_mode': 'always',
883882
'pathconfig_warnings': 0,
884-
885-
'_isolated_interpreter': 1,
886883
}
887884
self.check_all_configs("test_init_from_config", config, preconfig,
888885
api=API_COMPAT)
@@ -1650,6 +1647,25 @@ def test_init_use_frozen_modules(self):
16501647
self.check_all_configs("test_init_use_frozen_modules", config,
16511648
api=API_PYTHON, env=env)
16521649

1650+
def test_init_main_interpreter_settings(self):
1651+
THREADS = 1<<10
1652+
FORK = 1<<15
1653+
SUBPROCESS = 1<<16
1654+
expected = {
1655+
# All optional features should be enabled.
1656+
'feature_flags': THREADS | FORK | SUBPROCESS,
1657+
}
1658+
out, err = self.run_embedded_interpreter(
1659+
'test_init_main_interpreter_settings',
1660+
)
1661+
self.assertEqual(err, '')
1662+
try:
1663+
out = json.loads(out)
1664+
except json.JSONDecodeError:
1665+
self.fail(f'fail to decode stdout: {out!r}')
1666+
1667+
self.assertEqual(out, expected)
1668+
16531669

16541670
class SetConfigTests(unittest.TestCase):
16551671
def test_set_config(self):

‎Misc/NEWS.d/next/C API/2022-10-24-11-26-55.gh-issue-98608._Q2lNV.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
A ``_PyInterpreterConfig`` has been added and ``_Py_NewInterpreter()`` has
2+
been renamed to ``_Py_NewInterpreterFromConfig()``. The
3+
"isolated_subinterpreters" argument is now a granular config that captures
4+
the previous behavior. Note that this is all "private" API.

‎Modules/_posixsubprocess.c

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -842,8 +842,7 @@ subprocess_fork_exec(PyObject *module, PyObject *args)
842842
}
843843

844844
PyInterpreterState *interp = PyInterpreterState_Get();
845-
const PyConfig *config = _PyInterpreterState_GetConfig(interp);
846-
if (config->_isolated_interpreter) {
845+
if (!_PyInterpreterState_HasFeature(interp, Py_RTFLAGS_SUBPROCESS)) {
847846
PyErr_SetString(PyExc_RuntimeError,
848847
"subprocess not supported for isolated subinterpreters");
849848
return NULL;

‎Modules/_testcapimodule.c

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3225,6 +3225,66 @@ run_in_subinterp(PyObject *self, PyObject *args)
32253225
return PyLong_FromLong(r);
32263226
}
32273227

3228+
/* To run some code in a sub-interpreter. */
3229+
static PyObject *
3230+
run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
3231+
{
3232+
const char *code;
3233+
int allow_fork = -1;
3234+
int allow_subprocess = -1;
3235+
int allow_threads = -1;
3236+
int r;
3237+
PyThreadState *substate, *mainstate;
3238+
/* only initialise 'cflags.cf_flags' to test backwards compatibility */
3239+
PyCompilerFlags cflags = {0};
3240+
3241+
static char *kwlist[] = {"code",
3242+
"allow_fork", "allow_subprocess", "allow_threads",
3243+
NULL};
3244+
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
3245+
"s$ppp:run_in_subinterp_with_config", kwlist,
3246+
&code, &allow_fork, &allow_subprocess, &allow_threads)) {
3247+
return NULL;
3248+
}
3249+
if (allow_fork < 0) {
3250+
PyErr_SetString(PyExc_ValueError, "missing allow_fork");
3251+
return NULL;
3252+
}
3253+
if (allow_subprocess < 0) {
3254+
PyErr_SetString(PyExc_ValueError, "missing allow_subprocess");
3255+
return NULL;
3256+
}
3257+
if (allow_threads < 0) {
3258+
PyErr_SetString(PyExc_ValueError, "missing allow_threads");
3259+
return NULL;
3260+
}
3261+
3262+
mainstate = PyThreadState_Get();
3263+
3264+
PyThreadState_Swap(NULL);
3265+
3266+
const _PyInterpreterConfig config = {
3267+
.allow_fork = allow_fork,
3268+
.allow_subprocess = allow_subprocess,
3269+
.allow_threads = allow_threads,
3270+
};
3271+
substate = _Py_NewInterpreterFromConfig(&config);
3272+
if (substate == NULL) {
3273+
/* Since no new thread state was created, there is no exception to
3274+
propagate; raise a fresh one after swapping in the old thread
3275+
state. */
3276+
PyThreadState_Swap(mainstate);
3277+
PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed");
3278+
return NULL;
3279+
}
3280+
r = PyRun_SimpleStringFlags(code, &cflags);
3281+
Py_EndInterpreter(substate);
3282+
3283+
PyThreadState_Swap(mainstate);
3284+
3285+
return PyLong_FromLong(r);
3286+
}
3287+
32283288
static int
32293289
check_time_rounding(int round)
32303290
{
@@ -5998,6 +6058,9 @@ static PyMethodDef TestMethods[] = {
59986058
METH_NOARGS},
59996059
{"crash_no_current_thread", crash_no_current_thread, METH_NOARGS},
60006060
{"run_in_subinterp", run_in_subinterp, METH_VARARGS},
6061+
{"run_in_subinterp_with_config",
6062+
_PyCFunction_CAST(run_in_subinterp_with_config),
6063+
METH_VARARGS | METH_KEYWORDS},
60016064
{"pytime_object_to_time_t", test_pytime_object_to_time_t, METH_VARARGS},
60026065
{"pytime_object_to_timeval", test_pytime_object_to_timeval, METH_VARARGS},
60036066
{"pytime_object_to_timespec", test_pytime_object_to_timespec, METH_VARARGS},

‎Modules/_testinternalcapi.c

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -550,6 +550,51 @@ _testinternalcapi_optimize_cfg_impl(PyObject *module, PyObject *instructions,
550550
}
551551

552552

553+
static PyObject *
554+
get_interp_settings(PyObject *self, PyObject *args)
555+
{
556+
int interpid = -1;
557+
if (!PyArg_ParseTuple(args, "|i:get_interp_settings", &interpid)) {
558+
return NULL;
559+
}
560+
561+
PyInterpreterState *interp = NULL;
562+
if (interpid < 0) {
563+
PyThreadState *tstate = _PyThreadState_GET();
564+
interp = tstate ? tstate->interp : _PyInterpreterState_Main();
565+
}
566+
else if (interpid == 0) {
567+
interp = _PyInterpreterState_Main();
568+
}
569+
else {
570+
PyErr_Format(PyExc_NotImplementedError,
571+
"%zd", interpid);
572+
return NULL;
573+
}
574+
assert(interp != NULL);
575+
576+
PyObject *settings = PyDict_New();
577+
if (settings == NULL) {
578+
return NULL;
579+
}
580+
581+
/* Add the feature flags. */
582+
PyObject *flags = PyLong_FromUnsignedLong(interp->feature_flags);
583+
if (flags == NULL) {
584+
Py_DECREF(settings);
585+
return NULL;
586+
}
587+
int res = PyDict_SetItemString(settings, "feature_flags", flags);
588+
Py_DECREF(flags);
589+
if (res != 0) {
590+
Py_DECREF(settings);
591+
return NULL;
592+
}
593+
594+
return settings;
595+
}
596+
597+
553598
static PyMethodDef TestMethods[] = {
554599
{"get_configs", get_configs, METH_NOARGS},
555600
{"get_recursion_depth", get_recursion_depth, METH_NOARGS},
@@ -569,6 +614,7 @@ static PyMethodDef TestMethods[] = {
569614
{"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL},
570615
{"set_eval_frame_record", set_eval_frame_record, METH_O, NULL},
571616
_TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF
617+
{"get_interp_settings", get_interp_settings, METH_VARARGS, NULL},
572618
{NULL, NULL} /* sentinel */
573619
};
574620

‎Modules/_threadmodule.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1128,7 +1128,7 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs)
11281128
}
11291129

11301130
PyInterpreterState *interp = _PyInterpreterState_GET();
1131-
if (interp->config._isolated_interpreter) {
1131+
if (!_PyInterpreterState_HasFeature(interp, Py_RTFLAGS_THREADS)) {
11321132
PyErr_SetString(PyExc_RuntimeError,
11331133
"thread is not supported for isolated subinterpreters");
11341134
return NULL;

‎Modules/_winapi.c

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1090,8 +1090,7 @@ _winapi_CreateProcess_impl(PyObject *module,
10901090
}
10911091

10921092
PyInterpreterState *interp = PyInterpreterState_Get();
1093-
const PyConfig *config = _PyInterpreterState_GetConfig(interp);
1094-
if (config->_isolated_interpreter) {
1093+
if (!_PyInterpreterState_HasFeature(interp, Py_RTFLAGS_SUBPROCESS)) {
10951094
PyErr_SetString(PyExc_RuntimeError,
10961095
"subprocess not supported for isolated subinterpreters");
10971096
return NULL;

0 commit comments

Comments
 (0)
0