10000 gh-98627: Add an Optional Check for Extension Module Subinterpreter Compatibility by ericsnowcurrently · Pull Request #99040 · python/cpython · GitHub
[go: up one dir, main page]

Skip to content

gh-98627: Add an Optional Check for Extension Module Subinterpreter Compatibility #99040

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
eb9d241
Add tests for extension module subinterpreter compatibility.
ericsnowcurrently Oct 24, 2022
49b4895
Add _PyInterpreterConfig.check_multi_interp_extensions and Py_RTFLAGS…
ericsnowcurrently Oct 22, 2022
ed505a6
Add _PyImport_CheckSubinterpIncompatibleExtensionAllowed().
ericsnowcurrently Oct 24, 2022
64e1dc8
Raise ImportError in subinterpreters for incompatible single-phase in…
ericsnowcurrently Oct 24, 2022
72ab9b6
Add a NEWS entry.
ericsnowcurrently Nov 3, 2022
9c24b34
Add PyInterpreterState.override_multi_interp_extensions_check.
ericsnowcurrently Nov 21, 2022
3c084eb
Add check_multi_interp_extensions().
ericsnowcurrently Nov 21, 2022
a3d3a65
Add _imp._override_multi_interp_extensions_check().
ericsnowcurrently Nov 21, 2022
ad3fe36
Add test.support.import_helper.multi_interp_extensions_check().
ericsnowcurrently Nov 21, 2022
1defec3
Add a test.
ericsnowcurrently Nov 21, 2022
99f3371
Merge branch 'main' into HEAD
ericsnowcurrently Jan 12, 2023
3c3ed2b
Fix a typo.
ericsnowcurrently Jan 12, 2023
de6c791
Add some extra diagnostic info.
ericsnowcurrently Jan 12, 2023
af114f2
Clarify various names (e.g. data keys) in the test.
ericsnowcurrently Jan 12, 2023
282e6d3
Allow long test output.
ericsnowcurrently Jan 12, 2023
3cb8645
Do not show the noop values unless different.
ericsnowcurrently Jan 12, 2023
81abbfb
Add comments to the expected values.
ericsnowcurrently Jan 12, 2023
db5d35a
Tweak the subtest labels.
ericsnowcurrently Jan 12, 2023
e0c55ad
Fix the expected results.
ericsnowcurrently Jan 12, 2023
3b2dd6d
Add a test just for how the setting is used.
ericsnowcurrently Jan 12, 2023
d648a7b
Revert "Add a test just for how the setting is used."
ericsnowcurrently Jan 12, 2023
dc8d877
Add a test for the various settings and overrides for a singlephase e…
ericsnowcurrently Jan 12, 2023
5fba674
Fix check_config.py.
ericsnowcurrently Feb 3, 2023
35d322d
Merge branch 'main' into interpreter-multi-interp-extensions-check
ericsnowcurrently Feb 6, 2023
ddf01fb
Merge branch 'main' into interpreter-multi-interp-extensions-check
ericsnowcurrently Feb 15, 2023
ee2cd3c
Merge branch 'main' into interpreter-multi-interp-extensions-check
ericsnowcurrently Feb 15, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add _PyInterpreterConfig.check_multi_interp_extensions and Py_RTFLAGS…
…_MULTI_INTERP_EXTENSIONS.
  • Loading branch information
ericsnowcurrently committed Jan 11, 2023
commit 49b4895e6c18414ea0c4c7e43d0197d958928c1c
3 changes: 3 additions & 0 deletions Include/cpython/initconfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ typedef struct {
int allow_exec;
int allow_threads;
int allow_daemon_threads;
int check_multi_interp_extensions;
} _PyInterpreterConfig;

#define _PyInterpreterConfig_INIT \
Expand All @@ -256,6 +257,7 @@ typedef struct {
.allow_exec = 0, \
.allow_threads = 1, \
.allow_daemon_threads = 0, \
.check_multi_interp_extensions = 1, \
}

#define _PyInterpreterConfig_LEGACY_INIT \
Expand All @@ -264,6 +266,7 @@ typedef struct {
.allow_exec = 1, \
.allow_threads = 1, \
.allow_daemon_threads = 1, \
.check_multi_interp_extensions = 0, \
}

/* --- Helper functions --------------------------------------- */
Expand Down
3 changes: 3 additions & 0 deletions Include/cpython/pystate.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ is available in a given context. For example, forking the process
might not be allowed in the current interpreter (i.e. os.fork() would fail).
*/

/* Set if import should check a module for subinterpreter support. */
#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8)

/* Set if threads are allowed. */
#define Py_RTFLAGS_THREADS (1UL << 10)

Expand Down
11 changes: 7 additions & 4 deletions Lib/test/test_capi/test_misc.py
Original file line number Diff line number Diff line change
Expand Up @@ -1297,17 +1297,20 @@ def test_configured_settings(self):
"""
import json

EXTENSIONS = 1<<8
THREADS = 1<<10
DAEMON_THREADS = 1<<11
FORK = 1<<15
EXEC = 1<<16

features = ['fork', 'exec', 'threads', 'daemon_threads']
features = ['fork', 'exec', 'threads', 'daemon_threads', 'extensions']
kwlist = [f'allow_{n}' for n in features]
kwlist[-1] = 'check_multi_interp_extensions'
for config, expected in {
(True, True, True, True): FORK | EXEC | THREADS | DAEMON_THREADS,
(False, False, False, False): 0,
(False, False, True, False): THREADS,
(True, True, True, True, True):
FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
(False, False, False, False, False): 0,
(False, False, True, False, True): THREADS | EXTENSIONS,
}.items():
kwargs = dict(zip(kwlist, config))
expected = {
Expand Down
4 changes: 3 additions & 1 deletion Lib/test/test_embed.py
Original file line number Diff line number Diff line change
Expand Up @@ -1652,13 +1652,15 @@ def test_init_use_frozen_modules(self):
api=API_PYTHON, env=env)

def test_init_main_interpreter_settings(self):
EXTENSIONS = 1<<8
THREADS = 1<<10
DAEMON_THREADS = 1<<11
FORK = 1<<15
EXEC = 1<<16
expected = {
# All optional features should be enabled.
'feature_flags': FORK | EXEC | THREADS | DAEMON_THREADS,
'feature_flags':
FORK | EXEC | THREADS | DAEMON_THREADS,
}
out, err = self.run_embedded_interpreter(
'test_init_main_interpreter_settings',
Expand Down
68 changes: 33 additions & 35 deletions Lib/test/test_import/__init__.py
9E88
Original file line number Diff line number Diff line change
Expand Up @@ -1430,13 +1430,17 @@ def import_script(self, name, fd):
os.write({fd}, text.encode('utf-8'))
''')

def check_compatible_shared(self, name):
def check_compatible_shared(self, name, *, strict=False):
# Verify that the named module may be imported in a subinterpreter.
#
# The subinterpreter will be in the current process.
# The module will have already been imported in the main interpreter.
# Thus, for extension/builtin modules, the module definition will
# have been loaded already and cached globally.
#
# "strict" determines whether or not the interpreter will be
# configured to check for modules that are not compatible
# with use in multiple interpreters.

# This check should always pass for all modules if not strict.

Expand All @@ -1446,12 +1450,13 @@ def check_compatible_shared(self, name):
ret = run_in_subinterp_with_config(
self.import_script(name, w),
**self.RUN_KWARGS,
check_multi_interp_extensions=strict,
)
self.assertEqual(ret, 0)
out = os.read(r, 100)
self.assertEqual(out, b'okay')

def check_compatible_isolated(self, name):
def check_compatible_isolated(self, name, *, strict=False):
# Differences from check_compatible_shared():
# * subinterpreter in a new process
# * module has never been imported before in that process
Expand All @@ -1465,67 +1470,60 @@ def check_compatible_isolated(self, name):
ret = _testcapi.run_in_subinterp_with_config(
{self.import_script(name, "sys.stdout.fileno()")!r},
**{self.RUN_KWARGS},
check_multi_interp_extensions={strict},
)
assert ret == 0, ret
'''))
self.assertEqual(err, b'')
self.assertEqual(out, b'okay')

def check_incompatible_isolated(self, name):
# Differences from check_compatible_isolated():
# * verify that import fails
_, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f'''
import _testcapi, sys
assert {name!r} not in sys.modules, {name!r}
ret = _testcapi.run_in_subinterp_with_config(
{self.import_script(name, "sys.stdout.fileno()")!r},
**{self.RUN_KWARGS},
)
assert ret == 0, ret
'''))
self.assertEqual(err, b'')
self.assertEqual(
out.decode('utf-8'),
f'ImportError: module {name} does not support loading in subinterpreters',
)

def test_builtin_compat(self):
module = 'sys'
with self.subTest(f'{module}: shared'):
self.check_compatible_shared(module)
with self.subTest(f'{module}: not strict'):
self.check_compatible_shared(module, strict=False)
with self.subTest(f'{module}: strict, shared'):
self.check_compatible_shared(module, strict=True)

@cpython_only
def test_frozen_compat(self):
module = '_frozen_importlib'
if __import__(module).__spec__.origin != 'frozen':
raise unittest.SkipTest(f'{module} is unexpectedly not frozen')
with self.subTest(f'{module}: shared'):
self.check_compatible_shared(module)
with self.subTest(f'{module}: not strict'):
self.check_compatible_shared(module, strict=False)
with self.subTest(f'{module}: strict, shared'):
self.check_compatible_shared(module, strict=True)

@unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module")
@unittest.skipIf(_testsinglephase is None, "test requires _testsinglphase module")
def test_single_init_extension_compat(self):
module = '_testsinglephase'
with self.subTest(f'{module}: shared'):
with self.subTest(f'{module}: not strict'):
self.check_compatible_shared(module, strict=False)
with self.subTest(f'{module}: strict, shared'):
self.check_compatible_shared(module)
with self.subTest(f'{module}: isolated'):
with self.subTest(f'{module}: strict, isolated'):
self.check_compatible_isolated(module)

@unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module")
def test_multi_init_extension_compat(self):
module = '_testmultiphase'
with self.subTest(f'{module}: shared'):
self.check_compatible_shared(module)
with self.subTest(f'{module}: isolated'):
self.check_compatible_isolated(module)
with self.subTest(f'{module}: not strict'):
self.check_compatible_shared(module, strict=False)
with self.subTest(f'{module}: strict, shared'):
self.check_compatible_shared(module, strict=True)
with self.subTest(f'{module}: strict, isolated'):
self.check_compatible_isolated(module, strict=True)

def test_python_compat(self):
module = 'threading'
if __import__(module).__spec__.origin == 'frozen':
raise unittest.SkipTest(f'{module} is unexpectedly frozen')
with self.subTest(f'{module}: shared'):
self.check_compatible_shared(module)
with self.subTest(f'{module}: isolated'):
self.check_compatible_isolated(module)
with self.subTest(f'{module}: not strict'):
self.check_compatible_shared(module, strict=False)
with self.subTest(f'{module}: strict, shared'):
self.check_compatible_shared(module, strict=True)
with self.subTest(f'{module}: strict, isolated'):
self.check_compatible_isolated(module, strict=True)


if __name__ == '__main__':
Expand Down
1 change: 1 addition & 0 deletions Lib/test/test_threading.py
Original file line number Diff line number Diff line change
Expand Up @@ -1347,6 +1347,7 @@ def func():
allow_exec=True,
allow_threads={allowed},
allow_daemon_threads={daemon_allowed},
check_multi_interp_extensions=False,
)
""")
with test.support.SuppressCrashReport():
Expand Down
12 changes: 10 additions & 2 deletions Modules/_testcapimodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -1578,6 +1578,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
int allow_exec = -1;
int allow_threads = -1;
int allow_daemon_threads = -1;
int check_multi_interp_extensions = -1;
int r;
PyThreadState *substate, *mainstate;
/* only initialise 'cflags.cf_flags' to test backwards compatibility */
Expand All @@ -1588,11 +1589,13 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
"allow_exec",
"allow_threads",
"allow_daemon_threads",
"check_multi_interp_extensions",
NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
"s$pppp:run_in_subinterp_with_config", kwlist,
"s$ppppp:run_in_subinterp_with_config", kwlist,
&code, &allow_fork, &allow_exec,
&allow_threads, &allow_daemon_threads)) {
&allow_threads, &allow_daemon_threads,
&check_multi_interp_extensions)) {
return NULL;
}
if (allow_fork < 0) {
Expand All @@ -1611,6 +1614,10 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads");
return NULL;
}
if (check_multi_interp_extensions < 0) {
PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions");
return NULL;
}

mainstate = PyThreadState_Get();

Expand All @@ -1621,6 +1628,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
.allow_exec = allow_exec,
.allow_threads = allow_threads,
.allow_daemon_threads = allow_daemon_threads,
.check_multi_interp_extensions = check_multi_interp_extensions,
};
substate = _Py_NewInterpreterFromConfig(&config);
if (substate == NULL) {
Expand Down
4 changes: 4 additions & 0 deletions Python/pylifecycle.c
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,10 @@ init_interp_settings(PyInterpreterState *interp, const _PyInterpreterConfig *con
if (config->allow_daemon_threads) {
interp->feature_flags |= Py_RTFLAGS_DAEMON_THREADS;
}

if (config->check_multi_interp_extensions) {
interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS;
}
}


Expand Down
0