8000 gh-101659: Isolate "obmalloc" State to Each Interpreter (gh-101660) · python/cpython@df3173d · GitHub
[go: up one dir, main page]
Skip to content

Commit df3173d

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-101659: Isolate "obmalloc" State to Each Interpreter (gh-101660)
This is strictly about moving the "obmalloc" runtime state from `_PyRuntimeState` to `PyInterpreterState`. Doing so improves isolation between interpreters, specifically most of the memory (incl. objects) allocated for each interpreter's use. This is important for a per-interpreter GIL, but such isolation is valuable even without it. FWIW, a per-interpreter obmalloc is the proverbial canary-in-the-coalmine when it comes to the isolation of objects between interpreters. Any object that leaks (unintentionally) to another interpreter is highly likely to cause a crash (on debug builds at least). That's a useful thing to know, relative to interpreter isolation.
1 parent 01be52e commit df3173d

20 files changed

+322
-73
lines changed

‎Include/cpython/initconfig.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,8 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config,
245245
/* --- PyInterpreterConfig ------------------------------------ */
246246

247247
typedef struct {
248+
// XXX "allow_object_sharing"? "own_objects"?
249+
int use_main_obmalloc;
248250
int allow_fork;
249251
int allow_exec;
250252
int allow_threads;
@@ -254,6 +256,7 @@ typedef struct {
254256

255257
#define _PyInterpreterConfig_INIT \
256258
{ \
259+
.use_main_obmalloc = 0, \
257260
.allow_fork = 0, \
258261
.allow_exec = 0, \
259262
.allow_threads = 1, \
@@ -263,6 +266,7 @@ typedef struct {
263266

264267
#define _PyInterpreterConfig_LEGACY_INIT \
265268
{ \
269+
.use_main_obmalloc = 1, \
266270
.allow_fork = 1, \
267271
.allow_exec = 1, \
268272
.allow_threads = 1, \

‎Include/cpython/pystate.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ is available in a given context. For example, forking the process
1111
might not be allowed in the current interpreter (i.e. os.fork() would fail).
1212
*/
1313

14+
/* Set if the interpreter share obmalloc runtime state
15+
with the main interpreter. */
16+
#define Py_RTFLAGS_USE_MAIN_OBMALLOC (1UL << 5)
17+
1418
/* Set if import should check a module for subinterpreter support. */
1519
#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8)
1620

‎Include/internal/pycore_interp.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,12 @@ extern "C" {
2323
#include "pycore_function.h" // FUNC_MAX_WATCHERS
2424
#include "pycore_genobject.h" // struct _Py_async_gen_state
2525
#include "pycore_gc.h" // struct _gc_runtime_state
26+
#include "pycore_global_objects.h" // struct _Py_interp_static_objects
2627
#include "pycore_import.h" // struct _import_state
2728
#include "pycore_instruments.h" // PY_MONITORING_EVENTS
2829
#include "pycore_list.h" // struct _Py_list_state
29-
#include "pycore_global_objects.h" // struct _Py_interp_static_objects
3030
#include "pycore_object_state.h" // struct _py_object_state
31+
#include "pycore_obmalloc.h" // struct obmalloc_state
3132
#include "pycore_tuple.h" // struct _Py_tuple_state
3233
#include "pycore_typeobject.h" // struct type_cache
3334
#include "pycore_unicodeobject.h" // struct _Py_unicode_state
@@ -82,6 +83,8 @@ struct _is {
8283
int _initialized;
8384
int finalizing;
8485

86+
struct _obmalloc_state obmalloc;
87+
8588
struct _ceval_state ceval;
8689
struct _gc_runtime_state gc;
8790

‎Include/internal/pycore_obmalloc.h

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -657,8 +657,12 @@ struct _obmalloc_usage {
657657
#endif /* WITH_PYMALLOC_RADIX_TREE */
658658

659659

660-
struct _obmalloc_state {
660+
struct _obmalloc_global_state {
661661
int dump_debug_stats;
662+
Py_ssize_t interpreter_leaks;
663+
};
664+
665+
struct _obmalloc_state {
662666
struct _obmalloc_pools pools;
663667
struct _obmalloc_mgmt mgmt;
664668
struct _obmalloc_usage usage;
@@ -675,7 +679,11 @@ void _PyObject_VirtualFree(void *, size_t size);
675679

676680

677681
/* This function returns the number of allocated memory blocks, regardless of size */
678-
PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
682+
extern Py_ssize_t _Py_GetGlobalAllocatedBlocks(void);
683+
#define _Py_GetAllocatedBlocks() \
684+
_Py_GetGlobalAllocatedBlocks()
685+
extern Py_ssize_t _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *);
686+
extern void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *);
679687

680688

681689
#ifdef WITH_PYMALLOC

‎Include/internal/pycore_obmalloc_init.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,13 @@ extern "C" {
5454
# error "NB_SMALL_SIZE_CLASSES should be less than 64"
5555
#endif
5656

57-
#define _obmalloc_state_INIT(obmalloc) \
57+
#define _obmalloc_global_state_INIT \
5858
{ \
5959
.dump_debug_stats = -1, \
60+
}
61+
62+
#define _obmalloc_state_INIT(obmalloc) \
63+
{ \
6064
.pools = { \
6165
.used = _obmalloc_pools_INIT(obmalloc.pools), \
6266
}, \

‎Include/internal/pycore_pylifecycle.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ extern void _PyAtExit_Fini(PyInterpreterState *interp);
6464
extern void _PyThread_FiniType(PyInterpreterState *interp);
6565
extern void _Py_Deepfreeze_Fini(void);
6666
extern void _PyArg_Fini(void);
67+
extern void _Py_FinalizeAllocatedBlocks(_PyRuntimeState *);
6768

6869
extern PyStatus _PyGILState_Init(PyInterpreterState *interp);
6970
extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate);

‎Include/internal/pycore_pystate.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ _Py_IsMainInterpreter(PyInterpreterState *interp)
3333
return (interp == _PyInterpreterState_Main());
3434
}
3535

36+
static inline int
37+
_Py_IsMainInterpreterFinalizing(PyInterpreterState *interp)
38+
{
39+
return (_PyRuntimeState_GetFinalizing(interp->runtime) != NULL &&
40+
interp == &interp->runtime->_main_interpreter);
41+
}
42+
3643

3744
static inline const PyConfig *
3845
_Py_GetMainConfig(void)

‎Include/internal/pycore_runtime.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ extern "C" {
2121
#include "pycore_pymem.h" // struct _pymem_allocators
2222
#include "pycore_pyhash.h" // struct pyhash_runtime_state
2323
#include "pycore_pythread.h" // struct _pythread_runtime_state
24-
#include "pycore_obmalloc.h" // struct obmalloc_state
2524
#include "pycore_signal.h" // struct _signals_runtime_state
2625
#include "pycore_time.h" // struct _time_runtime_state
2726
#include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state
@@ -88,7 +87,7 @@ typedef struct pyruntimestate {
8887
_Py_atomic_address _finalizing;
8988

9089
struct _pymem_allocators allocators;
91-
struct _obmalloc_state obmalloc;
90+
struct _obmalloc_global_state obmalloc;
9291
struct pyhash_runtime_state pyhash_state;
9392
struct _time_runtime_state time;
9493
struct _pythread_runtime_state threads;

‎Include/internal/pycore_runtime_init.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ extern PyTypeObject _PyExc_MemoryError;
2929
_pymem_allocators_debug_INIT, \
3030
_pymem_allocators_obj_arena_INIT, \
3131
}, \
32-
.obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \
32+
.obmalloc = _obmalloc_global_state_INIT, \
3333
.pyhash_state = pyhash_state_INIT, \
3434
.signals = _signals_RUNTIME_INIT, \
3535
.interpreters = { \
@@ -93,6 +93,7 @@ extern PyTypeObject _PyExc_MemoryError;
9393
{ \
9494
.id_refcount = -1, \
9595
.imports = IMPORTS_INIT, \
96+
.obmalloc = _obmalloc_state_INIT(INTERP.obmalloc), \
9697
.ceval = { \
9798
.recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
9899
}, \

‎Lib/test/test_capi/test_misc.py

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,20 +1211,25 @@ def test_configured_settings(self):
12111211
"""
12121212
import json
12131213

1214+
OBMALLOC = 1<<5
12141215
EXTENSIONS = 1<<8
12151216
THREADS = 1<<10
12161217
DAEMON_THREADS = 1<<11
12171218
FORK = 1<<15
12181219
EXEC = 1<<16
12191220

1220-
features = ['fork', 'exec', 'threads', 'daemon_threads', 'extensions']
1221+
features = ['obmalloc', 'fork', 'exec', 'threads', 'daemon_threads',
1222+
'extensions']
12211223
kwlist = [f'allow_{n}' for n in features]
1224+
kwlist[0] = 'use_main_obmalloc'
12221225
kwlist[-1] = 'check_multi_interp_extensions'
1226+
1227+
# expected to work
12231228
for config, expected in {
1224-
(True, True, True, True, True):
1225-
FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
1226-
(False, False, False, False, False): 0,
1227-
(False, False, True, False, True): THREADS | EXTENSIONS,
1229+
(True, True, True, True, True, True):
1230+
OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
1231+
(True, False, False, False, False, False): OBMALLOC,
1232+
(False, False, False, True, False, True): THREADS | EXTENSIONS,
12281233
}.items():
12291234
kwargs = dict(zip(kwlist, config))
12301235
expected = {
@@ -1246,6 +1251,20 @@ d 10000 ef test_configured_settings(self):
12461251

12471252
self.assertEqual(settings, expected)
12481253

1254+
# expected to fail
1255+
for config in [
1256+
(False, False, False, False, False, False),
1257+
]:
1258+
kwargs = dict(zip(kwlist, config))
1259+
with self.subTest(config):
1260+
script = textwrap.dedent(f'''
1261+
import _testinternalcapi
1262+
_testinternalcapi.get_interp_settings()
1263+
raise NotImplementedError('unreachable')
1264+
''')
1265+
with self.assertRaises(RuntimeError):
1266+
support.run_in_subinterp_with_config(script, **kwargs)
1267+
12491268
@unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module")
12501269
@unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
12511270
def test_overridden_setting_extensions_subinterp_check(self):
@@ -1257,13 +1276,15 @@ def test_overridden_setting_extensions_subinterp_check(self):
12571276
"""
12581277
import json
12591278

1279+
OBMALLOC = 1<<5
12601280
EXTENSIONS = 1<<8
12611281
THREADS = 1<<10
12621282
DAEMON_THREADS = 1<<11
12631283
FORK = 1<<15
12641284
EXEC = 1<<16
1265-
BASE_FLAGS = FORK | EXEC | THREADS | DAEMON_THREADS
1285+
BASE_FLAGS = OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS
12661286
base_kwargs = {
1287+
'use_main_obmalloc': True,
12671288
'allow_fork': True,
12681289
'allow_exec': True,
12691290
'allow_threads': True,

‎Lib/test/test_embed.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1656,6 +1656,7 @@ def test_init_use_frozen_modules(self):
16561656
api=API_PYTHON, env=env)
16571657

16581658
def test_init_main_interpreter_settings(self):
1659+
OBMALLOC = 1<<5
16591660
EXTENSIONS = 1<<8
16601661
THREADS = 1<<10
16611662
DAEMON_THREADS = 1<<11
@@ -1664,7 +1665,7 @@ def test_init_main_interpreter_settings(self):
16641665
expected = {
16651666
# All optional features should be enabled.
16661667
'feature_flags':
1667-
FORK | EXEC | THREADS | DAEMON_THREADS,
1668+
OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS,
16681669
}
16691670
out, err = self.run_embedded_interpreter(
16701671
'test_init_main_interpreter_settings',

‎Lib/test/test_import/__init__.py

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1636,7 +1636,12 @@ class SubinterpImportTests(unittest.TestCase):
16361636
allow_exec=False,
16371637
allow_threads=True,
16381638
allow_daemon_threads=False,
1639+
# Isolation-related config values aren't included here.
16391640
)
1641+
ISOLATED = dict(
1642+
use_main_obmalloc=False,
1643+
)
1644+
NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()}
16401645

16411646
@unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
16421647
def pipe(self):
@@ -1669,6 +1674,7 @@ def import_script(self, name, fd, check_override=None):
16691674
def run_here(self, name, *,
16701675
check_singlephase_setting=False,
16711676
check_singlephase_override=None,
1677+
isolated=False,
16721678
):
16731679
"""
16741680
Try importing the named module in a subinterpreter.
@@ -1689,6 +1695,7 @@ def run_here(self, name, *,
16891695

16901696
kwargs = dict(
16911697
**self.RUN_KWARGS,
1698+
**(self.ISOLATED if isolated else self.NOT_ISOLATED),
16921699
check_multi_interp_extensions=check_singlephase_setting,
16931700
)
16941701

@@ -1699,33 +1706,36 @@ def run_here(self, name, *,
16991706
self.assertEqual(ret, 0)
17001707
return os.read(r, 100)
17011708

1702-
def check_compatible_here(self, name, *, strict=False):
1709+
def check_compatible_here(self, name, *, strict=False, isolated=False):
17031710
# Verify that the named module may be imported in a subinterpreter.
17041711
# (See run_here() for more info.)
17051712
out = self.run_here(name,
17061713
check_singlephase_setting=strict,
1714+
isolated=isolated,
17071715
)
17081716
self.assertEqual(out, b'okay')
17091717

1710-
def check_incompatible_here(self, name):
1718+
def check_incompatible_here(self, name, *, isolated=False):
17111719
# Differences from check_compatible_here():
17121720
# * verify that import fails
17131721
# * "strict" is always True
17141722
out = self.run_here(name,
17151723
check_singlephase_setting=True,
1724+
isolated=isolated,
17161725
)
17171726
self.assertEqual(
17181727
out.decode('utf-8'),
17191728
f'ImportError: module {name} does not support loading in subinterpreters',
17201729
)
17211730

1722-
def check_compatible_fresh(self, name, *, strict=False):
1731+
def check_compatible_fresh(self, name, *, strict=False, isolated=False):
17231732
# Differences from check_compatible_here():
17241733
# * subinterpreter in a new process
17251734
# * module has never been imported before in that process
17261735
# * this tests importing the module for the first time
17271736
kwargs = dict(
17281737
**self.RUN_KWARGS,
1738+
**(self.ISOLATED if isolated else self.NOT_ISOLATED),
17291739
check_multi_interp_extensions=strict,
17301740
)
17311741
_, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f'''
@@ -1743,12 +1753,13 @@ def check_compatible_fresh(self, name, *, strict=False):
17431753
self.assertEqual(err, b'')
17441754
self.assertEqual(out, b'okay')
17451755

1746-
def check_incompatible_fresh(self, name):
1756+
def check_incompatible_fresh(self, name, *, isolated=False):
17471757
# Differences from check_compatible_fresh():
17481758
# * verify that import fails
17491759
# * "strict" is always True
17501760
kwargs = dict(
17511761
**self.RUN_KWARGS,
1762+
**(self.ISOLATED if isolated else self.NOT_ISOLATED),
17521763
check_multi_interp_extensions=True,
17531764
)
17541765
_, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f'''
@@ -1854,6 +1865,14 @@ def check_incompatible(setting, override):
18541865
with self.subTest('config: check disabled; override: disabled'):
18551866
check_compatible(False, -1)
18561867

1868+
def test_isolated_config(self):
1869+
module = 'threading'
1870+
require_pure_python(module)
1871+
with self.subTest(f'{module}: strict, not fresh'):
1872+
self.check_compatible_here(module, strict=True, isolated=True)
1873+
with self.subTest(f'{module}: strict, fresh'):
1874+
self.check_compatible_fresh(module, strict=True, isolated=True)
1875+
18571876

18581877
class TestSinglePhaseSnapshot(ModuleSnapshot):
18591878

‎Lib/test/test_threading.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1343,6 +1343,7 @@ def func():
13431343
import test.support
13441344
test.support.run_in_subinterp_with_config(
13451345
{subinterp_code!r},
1346+
use_main_obmalloc=True,
13461347
allow_fork=True,
13471348
allow_exec=True,
13481349
allow_threads={allowed},

‎Modules/_testcapimodule.c

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1482,6 +1482,7 @@ static PyObject *
14821482
run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
14831483
{
14841484
const char *code;
1485+
int use_main_obmalloc = -1;
14851486
int allow_fork = -1;
14861487
int allow_exec = -1;
14871488
int allow_threads = -1;
@@ -1493,19 +1494,25 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
14931494
PyCompilerFlags cflags = {0};
14941495

14951496
static char *kwlist[] = {"code",
1497+
"use_main_obmalloc",
14961498
"allow_fork",
14971499
"allow_exec",
14981500
"allow_threads",
14991501
"allow_daemon_threads",
15001502
"check_multi_interp_extensions",
15011503
NULL};
15021504
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
1503-
"s$ppppp:run_in_subinterp_with_config", kwlist,
1504-
&code, &allow_fork, &allow_exec,
1505+
"s$pppppp:run_in_subinterp_with_config", kwlist,
1506+
&code, &use_main_obmalloc,
1507+
&allow_fork, &allow_exec,
15051508
&allow_threads, &allow_daemon_threads,
15061509
&check_multi_interp_extensions)) {
15071510
return NULL;
15081511
}
1512+
if (use_main_obmalloc < 0) {
1513+
PyErr_SetString(PyExc_ValueError, "missing use_main_obmalloc");
1514+
return NULL;
1515+
}
15091516
if (allow_fork < 0) {
15101517
PyErr_SetString(PyExc_ValueError, "missing allow_fork");
15111518
return NULL;
@@ -1532,6 +1539,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
15321539
PyThreadState_Swap(NULL);
15331540

15341541
const _PyInterpreterConfig config = {
1542+
.use_main_obmalloc = use_main_obmalloc,
15351543
.allow_fork = allow_fork,
15361544
.allow_exec = allow_exec,
15371545
.allow_threads = allow_threads,

0 commit comments

Comments
 (0)
0