8000 gh-76785: Add PyInterpreterConfig Helpers (gh-117170) · python/cpython@f341d60 · GitHub
[go: up one dir, main page]
Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit f341d60

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-76785: Add PyInterpreterConfig Helpers (gh-117170)
These helpers make it easier to customize and inspect the config used to initialize interpreters. This is especially valuable in our tests. I found inspiration from the PyConfig API for the PyInterpreterConfig dict conversion stuff. As part of this PR I've also added a bunch of tests.
1 parent cae4cdd commit f341d60

File tree

13 files changed

+754
-86
lines changed

13 files changed

+754
-86
lines changed

‎Include/internal/pycore_pylifecycle.h

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,22 @@ PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category);
116116
// Export for special main.c string compiling with source tracebacks
117117
int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags);
118118

119+
120+
/* interpreter config */
121+
122+
// Export for _testinternalcapi shared extension
123+
PyAPI_FUNC(int) _PyInterpreterConfig_InitFromState(
124+
PyInterpreterConfig *,
125+
PyInterpreterState *);
126+
PyAPI_FUNC(PyObject *) _PyInterpreterConfig_AsDict(PyInterpreterConfig *);
127+
PyAPI_FUNC(int) _PyInterpreterConfig_InitFromDict(
128+
PyInterpreterConfig *,
129+
PyObject *);
130+
PyAPI_FUNC(int) _PyInterpreterConfig_UpdateFromDict(
131+
PyInterpreterConfig *,
132+
PyObject *);
133+
134+
119135
#ifdef __cplusplus
120136
}
121137
#endif

‎Lib/test/support/__init__.py

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1734,8 +1734,19 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config):
17341734
raise unittest.SkipTest("requires _testinternalcapi")
17351735
if own_gil is not None:
17361736
assert 'gil' not in config, (own_gil, config)
1737-
config['gil'] = 2 if own_gil else 1
1738-
return _testinternalcapi.run_in_subinterp_with_config(code, **config)
1737+
config['gil'] = 'own' if own_gil else 'shared'
1738+
else:
1739+
gil = config['gil']
1740+
if gil == 0:
1741+
config['gil'] = 'default'
1742+
elif gil == 1:
1743+
config['gil'] = 'shared'
1744+
elif gil == 2:
1745+
config['gil'] = 'own'
1746+
else:
1747+
raise NotImplementedError(gil)
1748+
config = types.SimpleNamespace(**config)
1749+
return _testinternalcapi.run_in_subinterp_with_config(code, config)
17391750

17401751

17411752
def _check_tracemalloc():

‎Lib/test/test_capi/test_misc.py

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2204,6 +2204,257 @@ def test_module_state_shared_in_global(self):
22042204
self.assertEqual(main_attr_id, subinterp_attr_id)
22052205

22062206

2207+
class InterpreterConfigTests(unittest.TestCase):
2208+
2209+
supported = {
2210+
'isolated': types.SimpleNamespace(
2211+
use_main_obmalloc=False,
2212+
allow_fork=False,
2213+
allow_exec=False,
2214+
allow_threads=True,
2215+
allow_daemon_threads=False,
2216+
check_multi_interp_extensions=True,
2217+
gil='own',
2218+
),
2219+
'legacy': types.SimpleNamespace(
2220+
use_main_obmalloc=True,
2221+
allow_fork=True,
2222+
allow_exec=True,
2223+
allow_threads=True,
2224+
allow_daemon_threads=True,
2225+
check_multi_interp_extensions=False,
2226+
gil='shared',
2227+
),
2228+
'empty': types.SimpleNamespace(
2229+
use_main_obmalloc=False,
2230+
allow_fork=False,
2231+
allow_exec=False,
2232+
allow_threads=False,
2233+
allow_daemon_threads=False,
2234+
check_multi_interp_extensions=False,
2235+
gil='default',
2236+
),
2237+
}
2238+
gil_supported = ['default', 'shared', 'own']
2239+
2240+
def iter_all_configs(self):
2241+
for use_main_obmalloc in (True, False):
2242+
for allow_fork in (True, False):
2243+
for allow_exec in (True, False):
2244+
for allow_threads in (True, False):
2245+
for allow_daemon in (True, False):
2246+
for checkext in (True, False):
2247+
for gil in ('shared', 'own', 'default'):
2248+
yield types.SimpleNamespace(
2249+
use_main_obmalloc=use_main_obmalloc,
2250+
allow_fork=allow_fork,
2251+
allow_exec=allow_exec,
2252+
allow_threads=allow_threads,
2253+
allow_daemon_threads=allow_daemon,
2254+
check_multi_interp_extensions=checkext,
2255+
gil=gil,
2256+
)
2257+
2258+
def assert_ns_equal(self, ns1, ns2, msg=None):
2259+
# This is mostly copied from TestCase.assertDictEqual.
2260+
self.assertEqual(type(ns1), type(ns2))
2261+
if ns1 == ns2:
2262+
return
2263+
2264+
import difflib
2265+
import pprint
2266+
from unittest.util import _common_shorten_repr
2267+
standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2)
2268+
diff = ('\n' + '\n'.join(difflib.ndiff(
2269+
pprint.pformat(vars(ns1)).splitlines(),
2270+
pprint.pformat(vars(ns2)).splitlines())))
2271+
diff = f'namespace({diff})'
2272+
standardMsg = self._truncateMessage(standardMsg, diff)
2273+
self.fail(self._formatMessage(msg, standardMsg))
2274+
2275+
def test_predefined_config(self):
2276+
def check(name, expected):
2277+
expected = self.supported[expected]
2278+
args = (name,) if name else ()
2279+
2280+
config1 = _testinternalcapi.new_interp_config(*args)
2281+
self.assert_ns_equal(config1, expected)
2282+
self.assertIsNot(config1, expected)
2283+
2284+
config2 = _testinternalcapi.new_interp_config(*args)
2285+
self.assert_ns_equal(config2, expected)
2286+
self.assertIsNot(config2, expected)
2287+
self.assertIsNot(config2, config1)
2288+
2289+
with self.subTest('default'):
2290+
check(None, 'isolated')
2291+
2292+
for name in self.supported:
2293+
with self.subTest(name):
2294+
check(name, name)
2295+
2296+
def test_update_from_dict(self):
2297+
for name, vanilla in self.supported.items():
2298+
with self.subTest(f'noop ({name})'):
2299+
expected = vanilla
2300+
overrides = vars(vanilla)
2301+
config = _testinternalcapi.new_interp_config(name, **overrides)
2302+
self.assert_ns_equal(config, expected)
2303+
2304+
with self.subTest(f'change all ({name})'):
2305+
overrides = {k: not v for k, v in vars(vanilla).items()}
2306+
for gil in self.gil_supported:
2307+
if vanilla.gil == gil:
2308+
continue
2309+
overrides['gil'] = gil
2310+
expected = types.SimpleNamespace(**overrides)
2311+
config = _testinternalcapi.new_interp_config(
2312+
name, **overrides)
2313+
self.assert_ns_equal(config, expected)
2314+
2315+
# Override individual fields.
2316+
for field, old in vars(vanilla).items():
2317+
if field == 'gil':
2318+
values = [v for v in self.gil_supported if v != old]
2319+
else:
2320+
values = [not old]
2321+
for val in values:
2322+
with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'):
2323+
overrides = {field: val}
2324+
expected = types.SimpleNamespace(
2325+
**dict(vars(vanilla), **overrides),
2326+
)
2327+
config = _testinternalcapi.new_interp_config(
2328+
name, **overrides)
2329+
self.assert_ns_equal(config, expected)
2330+
2331+
with self.subTest('unsupported field'):
2332+
for name in self.supported:
2333+
with self.assertRaises(ValueError):
2334+
_testinternalcapi.new_interp_config(name, spam=True)
2335+
2336+
# Bad values for bool fields.
2337+
for field, value in vars(self.supported['empty']).items():
2338+
if field == 'gil':
2339+
continue
2340+
assert isinstance(value, bool)
2341+
for value in [1, '', 'spam', 1.0, None, object()]:
2342+
with self.subTest(f'unsupported value ({field}={value!r})'):
2343+
with self.assertRaises(TypeError):
2344+
_testinternalcapi.new_interp_config(**{field: value})
2345+
2346+
# Bad values for .gil.
2347+
for value in [True, 1, 1.0, None, object()]:
2348+
with self.subTest(f'unsupported value(gil={value!r})'):
2349+
with self.assertRaises(TypeError):
2350+
_testinternalcapi.new_interp_config(gil=value)
2351+
for value in ['', 'spam']:
2352+
with self.subTest(f'unsupported value (gil={value!r})'):
2353+
with self.assertRaises(ValueError):
2354+
_testinternalcapi.new_interp_config(gil=value)
2355+
2356+
@requires_subinterpreters
2357+
def test_interp_init(self):
2358+
questionable = [
2359+
# strange
2360+
dict(
2361+
allow_fork=True,
2362+
allow_exec=False,
2363+
),
2364+
dict(
2365+
gil='shared',
2366+
use_main_obmalloc=False,
2367+
),
2368+
# risky
2369+
dict(
2370+
allow_fork=True,
2371+
allow_threads=True,
2372+
),
2373+
# ought to be invalid?
2374+
dict(
2375+
allow_threads=False,
2376+
allow_daemon_threads=True,
2377+
),
2378+
dict(
2379+
gil='own',
2380+
use_main_obmalloc=True,
2381+
),
2382+
]
2383+
invalid = [
2384+
dict(
2385+
use_main_obmalloc=False,
2386+
check_multi_interp_extensions=False
2387+
),
2388+
]
2389+
def match(config, override_cases):
2390+
ns = vars(config)
2391+
for overrides in override_cases:
2392+
if dict(ns, **overrides) == ns:
2393+
return True
2394+
return False
2395+
2396+
def check(config):
2397+
script = 'pass'
2398+
rc = _testinternalcapi.run_in_subinterp_with_config(script, config)
2399+
self.assertEqual(rc, 0)
2400+
2401+
for config in self.iter_all_configs():
2402+
if config.gil == 'default':
2403+
continue
2404+
if match(config, invalid):
2405+
with self.subTest(f'invalid: {config}'):
2406+
with self.assertRaises(RuntimeError):
2407+
check(config)
2408+
elif match(config, questionable):
2409+
with self.subTest(f'questionable: {config}'):
2410+
check(config)
2411+
else:
2412+
with self.subTest(f'valid: {config}'):
2413+
check(config)
2414+
2415+
@requires_subinterpreters
2416+
def test_get_config(self):
2417+
@contextlib.contextmanager
2418+
def new_interp(config):
2419+
interpid = _testinternalcapi.new_interpreter(config)
2420+
try:
2421+
yield interpid
2422+
finally:
2423+
try:
2424+
_interpreters.destroy(interpid)
2425+
except _interpreters.InterpreterNotFoundError:
2426+
pass
2427+
2428+
with self.subTest('main'):
2429+
expected = _testinternalcapi.new_interp_config('legacy')
2430+
expected.gil = 'own'
2431+
interpid = _interpreters.get_main()
2432+
config = _testinternalcapi.get_interp_config(interpid)
2433+
self.assert_ns_equal(config, expected)
2434+
2435+
with self.subTest('isolated'):
2436+
expected = _testinternalcapi.new_interp_config('isolated')
2437+
with new_interp('isolated') as interpid:
2438+
config = _testinternalcapi.get_interp_config(interpid)
2439+
self.assert_ns_equal(config, expected)
2440+
2441+
with self.subTest('legacy'):
2442+
expected = _testinternalcapi.new_interp_config('legacy')
2443+
with new_interp('legacy') as interpid:
2444+
config = _testinternalcapi.get_interp_config(interpid)
2445+
self.assert_ns_equal(config, expected)
2446+
2447+
with self.subTest('custom'):
2448+
orig = _testinternalcapi.new_interp_config(
2449+
'empty',
2450+
use_main_obmalloc=True,
2451+
gil='shared',
2452+
)
2453+
with new_interp(orig) as interpid:
2454+
config = _testinternalcapi.get_interp_config(interpid)
2455+
self.assert_ns_equal(config, orig)
2456+
2457+
22072458
@requires_subinterpreters
22082459
class InterpreterIDTests(unittest.TestCase):
22092460

‎Lib/test/test_import/__init__.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1823,15 +1823,19 @@ def check_compatible_fresh(self, name, *, strict=False, isolated=False):
18231823
**(self.ISOLATED if isolated else self.NOT_ISOLATED),
18241824
check_multi_interp_extensions=strict,
18251825
)
1826+
gil = kwargs['gil']
1827+
kwargs['gil'] = 'default' if gil == 0 else (
1828+
'shared' if gil == 1 else 'own' if gil == 2 else gil)
18261829
_, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f'''
18271830
import _testinternalcapi, sys
18281831
assert (
18291832
{name!r} in sys.builtin_module_names or
18301833
{name!r} not in sys.modules
18311834
), repr({name!r})
1835+
config = type(sys.implementation)(**{kwargs})
18321836
ret = _testinternalcapi.run_in_subinterp_with_config(
18331837
{self.import_script(name, "sys.stdout.fileno()")!r},
1834-
**{kwargs},
1838+
config,
18351839
)
18361840
assert ret == 0, ret
18371841
'''))
@@ -1847,12 +1851,16 @@ def check_incompatible_fresh(self, name, *, isolated=False):
18471851
**(self.ISOLATED if isolated else self.NOT_ISOLATED),
18481852
check_multi_interp_extensions=True,
18491853
)
1854+
gil = kwargs['gil']
1855+
kwargs['gil'] = 'default' if gil == 0 else (
1856+
'shared' if gil == 1 else 'own' if gil == 2 else gil)
18501857
_, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f'''
18511858
import _testinternalcapi, sys
18521859
assert {name!r} not in sys.modules, {name!r}
1860+
config = type(sys.implementation)(**{kwargs})
18531861
ret = _testinternalcapi.run_in_subinterp_with_config(
18541862
{self.import_script(name, "sys.stdout.fileno()")!r},
1855-
**{kwargs},
1863+
config,
18561864
)
18571865
assert ret == 0, ret
18581866
'''))

‎Makefile.pre.in

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -440,6 +440,7 @@ PYTHON_OBJS= \
440440
Python/import.o \
441441
Python/importdl.o \
442442
Python/initconfig.o \
443+
Python/interpconfig.o \
443444
Python/instrumentation.o \
444445
Python/intrinsics.o \
445446
Python/jit.o \
@@ -1687,6 +1688,10 @@ Modules/_xxinterpchannelsmodule.o: $(srcdir)/Modules/_xxinterpchannelsmodule.c $
16871688

16881689
Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h
16891690

1691+
Python/initconfig.o: $(srcdir)/Python/initconfig.c $(srcdir)/Python/config_common.h
1692+
1693+
Python/interpconfig.o: $(srcdir)/Python/interpconfig.c $(srcdir)/Python/config_common.h
1694+
16901695
Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile
16911696
$(CC) -c $(PY_CORE_CFLAGS) \
16921697
-DSOABI='"$(SOABI)"' \

0 commit comments

Comments
 (0)
0