Add _PyInterpreterConfig.check_multi_interp_extensions and Py_RTFLAGS…

…_MULTI_INTERP_EXTENSIONS.
python · ericsnowcurrently · Feb 16, 2023 · Oct 24, 2022 · Oct 22, 2022 · Oct 24, 2022
commit 49b4895e6c18414ea0c4c7e43d0197d958928c1c
@@ -248,6 +248,7 @@ typedef struct {
     int allow_exec;
     int allow_threads;
     int allow_daemon_threads;
+    int check_multi_interp_extensions;
 } _PyInterpreterConfig;
 
 #define _PyInterpreterConfig_INIT \
@@ -256,6 +257,7 @@ typedef struct {
         .allow_exec = 0, \
         .allow_threads = 1, \
         .allow_daemon_threads = 0, \
+        .check_multi_interp_extensions = 1, \
     }
 
 #define _PyInterpreterConfig_LEGACY_INIT \
@@ -264,6 +266,7 @@ typedef struct {
         .allow_exec = 1, \
         .allow_threads = 1, \
         .allow_daemon_threads = 1, \
+        .check_multi_interp_extensions = 0, \
     }
 
 /* --- Helper functions --------------------------------------- */

@@ -11,6 +11,9 @@ is available in a given context.  For example, forking the process
 might not be allowed in the current interpreter (i.e. os.fork() would fail).
 */
 
+/* Set if import should check a module for subinterpreter support. */
+#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8)
+
 /* Set if threads are allowed. */
 #define Py_RTFLAGS_THREADS (1UL << 10)
 

@@ -1297,17 +1297,20 @@ def test_configured_settings(self):
         """
         import json
 
+        EXTENSIONS = 1<<8
         THREADS = 1<<10
         DAEMON_THREADS = 1<<11
         FORK = 1<<15
         EXEC = 1<<16
 
-        features = ['fork', 'exec', 'threads', 'daemon_threads']
+        features = ['fork', 'exec', 'threads', 'daemon_threads', 'extensions']
         kwlist = [f'allow_{n}' for n in features]
+        kwlist[-1] = 'check_multi_interp_extensions'
         for config, expected in {
-            (True, True, True, True): FORK | EXEC | THREADS | DAEMON_THREADS,
-            (False, False, False, False): 0,
-            (False, False, True, False): THREADS,
+            (True, True, True, True, True):
+                FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
+            (False, False, False, False, False): 0,
+            (False, False, True, False, True): THREADS | EXTENSIONS,
         }.items():
             kwargs = dict(zip(kwlist, config))
             expected = {

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
@@ -1652,13 +1652,15 @@ def test_init_use_frozen_modules(self):
                                        api=API_PYTHON, env=env)
 
     def test_init_main_interpreter_settings(self):
+        EXTENSIONS = 1<<8
         THREADS = 1<<10
         DAEMON_THREADS = 1<<11
         FORK = 1<<15
         EXEC = 1<<16
         expected = {
             # All optional features should be enabled.
-            'feature_flags': FORK | EXEC | THREADS | DAEMON_THREADS,
+            'feature_flags':
+                FORK | EXEC | THREADS | DAEMON_THREADS,
         }
         out, err = self.run_embedded_interpreter(
             'test_init_main_interpreter_settings',

@@ -1430,13 +1430,17 @@ def import_script(self, name, fd):
             os.write({fd}, text.encode('utf-8'))
             ''')
 
-    def check_compatible_shared(self, name):
+    def check_compatible_shared(self, name, *, strict=False):
         # Verify that the named module may be imported in a subinterpreter.
         #
         # The subinterpreter will be in the current process.
         # The module will have already been imported in the main interpreter.
         # Thus, for extension/builtin modules, the module definition will
         # have been loaded already and cached globally.
+        #
+        # "strict" determines whether or not the interpreter will be
+        # configured to check for modules that are not compatible
+        # with use in multiple interpreters.
 
         # This check should always pass for all modules if not strict.
 
@@ -1446,12 +1450,13 @@ def check_compatible_shared(self, name):
         ret = run_in_subinterp_with_config(
             self.import_script(name, w),
             **self.RUN_KWARGS,
+            check_multi_interp_extensions=strict,
         )
         self.assertEqual(ret, 0)
         out = os.read(r, 100)
         self.assertEqual(out, b'okay')
 
-    def check_compatible_isolated(self, name):
+    def check_compatible_isolated(self, name, *, strict=False):
         # Differences from check_compatible_shared():
         #  * subinterpreter in a new process
         #  * module has never been imported before in that process
@@ -1465,67 +1470,60 @@ def check_compatible_isolated(self, name):
             ret = _testcapi.run_in_subinterp_with_config(
                 {self.import_script(name, "sys.stdout.fileno()")!r},
                 **{self.RUN_KWARGS},
+                check_multi_interp_extensions={strict},
             )
             assert ret == 0, ret
             '''))
         self.assertEqual(err, b'')
         self.assertEqual(out, b'okay')
 
-    def check_incompatible_isolated(self, name):
-        # Differences from check_compatible_isolated():
-        #  * verify that import fails
-        _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f'''
-            import _testcapi, sys
-            assert {name!r} not in sys.modules, {name!r}
-            ret = _testcapi.run_in_subinterp_with_config(
-                {self.import_script(name, "sys.stdout.fileno()")!r},
-                **{self.RUN_KWARGS},
-            )
-            assert ret == 0, ret
-            '''))
-        self.assertEqual(err, b'')
-        self.assertEqual(
-            out.decode('utf-8'),
-            f'ImportError: module {name} does not support loading in subinterpreters',
-        )
-
     def test_builtin_compat(self):
         module = 'sys'
-        with self.subTest(f'{module}: shared'):
-            self.check_compatible_shared(module)
+        with self.subTest(f'{module}: not strict'):
+            self.check_compatible_shared(module, strict=False)
+        with self.subTest(f'{module}: strict, shared'):
+            self.check_compatible_shared(module, strict=True)
 
     @cpython_only
     def test_frozen_compat(self):
         module = '_frozen_importlib'
         if __import__(module).__spec__.origin != 'frozen':
             raise unittest.SkipTest(f'{module} is unexpectedly not frozen')
-        with self.subTest(f'{module}: shared'):
-            self.check_compatible_shared(module)
+        with self.subTest(f'{module}: not strict'):
+            self.check_compatible_shared(module, strict=False)
+        with self.subTest(f'{module}: strict, shared'):
+            self.check_compatible_shared(module, strict=True)
 
-    @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module")
+    @unittest.skipIf(_testsinglephase is None, "test requires _testsinglphase module")
     def test_single_init_extension_compat(self):
         module = '_testsinglephase'
-        with self.subTest(f'{module}: shared'):
+        with self.subTest(f'{module}: not strict'):
+            self.check_compatible_shared(module, strict=False)
+        with self.subTest(f'{module}: strict, shared'):
             self.check_compatible_shared(module)
-        with self.subTest(f'{module}: isolated'):
+        with self.subTest(f'{module}: strict, isolated'):
             self.check_compatible_isolated(module)
 
     @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module")
     def test_multi_init_extension_compat(self):
         module = '_testmultiphase'
-        with self.subTest(f'{module}: shared'):
-            self.check_compatible_shared(module)
-        with self.subTest(f'{module}: isolated'):
-            self.check_compatible_isolated(module)
+        with self.subTest(f'{module}: not strict'):
+            self.check_compatible_shared(module, strict=False)
+        with self.subTest(f'{module}: strict, shared'):
+            self.check_compatible_shared(module, strict=True)
+        with self.subTest(f'{module}: strict, isolated'):
+            self.check_compatible_isolated(module, strict=True)
 
     def test_python_compat(self):
         module = 'threading'
         if __import__(module).__spec__.origin == 'frozen':
             raise unittest.SkipTest(f'{module} is unexpectedly frozen')
-        with self.subTest(f'{module}: shared'):
-            self.check_compatible_shared(module)
-        with self.subTest(f'{module}: isolated'):
-            self.check_compatible_isolated(module)
+        with self.subTest(f'{module}: not strict'):
+            self.check_compatible_shared(module, strict=False)
+        with self.subTest(f'{module}: strict, shared'):
+            self.check_compatible_shared(module, strict=True)
+        with self.subTest(f'{module}: strict, isolated'):
+            self.check_compatible_isolated(module, strict=True)
 
 
 if __name__ == '__main__':

diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py
@@ -1347,6 +1347,7 @@ def func():
                 allow_exec=True,
                 allow_threads={allowed},
                 allow_daemon_threads={daemon_allowed},
+                check_multi_interp_extensions=False,
             )
             """)
         with test.support.SuppressCrashReport():

diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
@@ -1578,6 +1578,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
     int allow_exec = -1;
     int allow_threads = -1;
     int allow_daemon_threads = -1;
+    int check_multi_interp_extensions = -1;
     int r;
     PyThreadState *substate, *mainstate;
     /* only initialise 'cflags.cf_flags' to test backwards compatibility */
@@ -1588,11 +1589,13 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
                              "allow_exec",
                              "allow_threads",
                              "allow_daemon_threads",
+                             "check_multi_interp_extensions",
                              NULL};
     if (!PyArg_ParseTupleAndKeywords(args, kwargs,
-                    "s$pppp:run_in_subinterp_with_config", kwlist,
+                    "s$ppppp:run_in_subinterp_with_config", kwlist,
                     &code, &allow_fork, &allow_exec,
-                    &allow_threads, &allow_daemon_threads)) {
+                    &allow_threads, &allow_daemon_threads,
+                    &check_multi_interp_extensions)) {
         return NULL;
     }
     if (allow_fork < 0) {
@@ -1611,6 +1614,10 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
         PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads");
         return NULL;
     }
+    if (check_multi_interp_extensions < 0) {
+        PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions");
+        return NULL;
+    }
 
     mainstate = PyThreadState_Get();
 
@@ -1621,6 +1628,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
         .allow_exec = allow_exec,
         .allow_threads = allow_threads,
         .allow_daemon_threads = allow_daemon_threads,
+        .check_multi_interp_extensions = check_multi_interp_extensions,
     };
     substate = _Py_NewInterpreterFromConfig(&config);
     if (substate == NULL) {

@@ -642,6 +642,10 @@ init_interp_settings(PyInterpreterState *interp, const _PyInterpreterConfig *con
     if (config->allow_daemon_threads) {
         interp->feature_flags |= Py_RTFLAGS_DAEMON_THREADS;
     }
+
+    if (config->check_multi_interp_extensions) {
+        interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS;
+    }
 }