8000 gh-109853: Fix sys.path[0] For Subinterpreters (gh-109994) · python/cpython@a040a32 · GitHub
[go: up one dir, main page]
Skip to content

Commit a040a32

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-109853: Fix sys.path[0] For Subinterpreters (gh-109994)
This change makes sure sys.path[0] is set properly for subinterpreters. Before, it wasn't getting set at all. This PR does not address the broader concerns from gh-109853.
1 parent fc2cb86 commit a040a32

File tree

7 files changed

+214
-10
lines changed

7 files changed

+214
-10
lines changed

‎Include/cpython/initconfig.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,9 @@ typedef struct PyConfig {
204204
wchar_t *run_module;
205205
wchar_t *run_filename;
206206

207+
/* --- Set by Py_Main() -------------------------- */
208+
wchar_t *sys_path_0;
209+
207210
/* --- Private fields ---------------------------- */
208211

209212
// Install importlib? If equals to 0, importlib is not initialized at all.

‎Lib/test/test_embed.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -505,6 +505,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
505505
'run_command': None,
506506
'run_module': None,
507507
'run_filename': None,
508+
'sys_path_0': None,
508509

509510
'_install_importlib': 1,
510511
'check_hash_pycs_mode': 'default',
@@ -1132,6 +1133,7 @@ def test_init_run_main(self):
11321133
'program_name': './python3',
11331134
'run_command': code + '\n',
11341135
'parse_argv': 2,
1136+
'sys_path_0': '',
11351137
}
11361138
self.check_all_configs("test_init_run_main", config, api=API_PYTHON)
11371139

@@ -1147,6 +1149,7 @@ def test_init_main(self):
11471149
'run_command': code + '\n',
11481150
'parse_argv': 2,
11491151
'_init_main': 0,
1152+
'sys_path_0': '',
11501153
}
11511154
self.check_all_configs("test_init_main", config,
11521155
api=API_PYTHON,

‎Lib/test/test_interpreters.py

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import contextlib
2+
import json
23
import os
4+
import os.path
35
import sys
46
import threading
57
from textwrap import dedent
@@ -9,6 +11,7 @@
911
from test import support
1012
from test.support import import_helper
1113
from test.support import threading_helper
14+
from test.support import os_helper
1215
_interpreters = import_helper.import_module('_xxsubinterpreters')
1316
_channels = import_helper.import_module('_xxinterpchannels')
1417
from test.support import interpreters
@@ -488,6 +491,154 @@ def task():
488491
pass
489492

490493

494+
class StartupTests(TestBase):
495+
496+
# We want to ensure the initial state of subinterpreters
497+
# matches expectations.
498+
499+
_subtest_count = 0
500+
501+
@contextlib.contextmanager
502+
def subTest(self, *args):
503+
with super().subTest(*args) as ctx:
504+
self._subtest_count += 1
505+
try:
506+
yield ctx
507+
finally:
508+
if self._debugged_in_subtest:
509+
if self._subtest_count == 1:
510+
# The first subtest adds a leading newline, so we
511+
# compensate here by not printing a trailing newline.
512+
print('### end subtest debug ###', end='')
513+
else:
514+
print('### end subtest debug ###')
515+
self._debugged_in_subtest = False
516+
517+
def debug(self, msg, *, header=None):
518+
if header:
519+
self._debug(f'--- {header} ---')
520+
if msg:
521+
if msg.endswith(os.linesep):
522+
self._debug(msg[:-len(os.linesep)])
523+
else:
524+
self._debug(msg)
525+
self._debug('<no newline>')
526+
self._debug('------')
527+
else:
528+
self._debug(msg)
529+
530+
_debugged = False
531+
_debugged_in_subtest = False
532+
def _debug(self, msg):
533+
if not self._debugged:
534+
print()
535+
self._debugged = True
536+
if self._subtest is not None:
537+
if True:
538+
if not self._debugged_in_subtest:
539+
self._debugged_in_subtest = True
540+
print('### start subtest debug ###')
541+
print(msg)
542+
else:
543+
print(msg)
544+
545+
def create_temp_dir(self):
546+
import tempfile
547+
tmp = tempfile.mkdtemp(prefix='test_interpreters_')
548+
tmp = os.path.realpath(tmp)
549+
self.addCleanup(os_helper.rmtree, tmp)
550+
return tmp
551+
552+
def write_script(self, *path, text):
553+
filename = os.path.join(*path)
554+
dirname = os.path.dirname(filename)
555+
if dirname:
556+
os.makedirs(dirname, exist_ok=True)
557+
with open(filename, 'w', encoding='utf-8') as outfile:
558+
outfile.write(dedent(text))
559+
return filename
560+
561+
@support.requires_subprocess()
562+
def run_python(self, argv, *, cwd=None):
563+
# This method is inspired by
564+
# EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py.
565+
import shlex
566+
import subprocess
567+
if isinstance(argv, str):
568+
argv = shlex.split(argv)
569+
argv = [sys.executable, *argv]
570+
try:
571+
proc = subprocess.run(
572+
argv,
573+
cwd=cwd,
574+
capture_output=True,
575+
text=True,
576+
)
577+
except Exception as exc:
578+
self.debug(f'# cmd: {shlex.join(argv)}')
579+
if isinstance(exc, FileNotFoundError) and not exc.filename:
580+
if os.path.exists(argv[0]):
581+
exists = 'exists'
582+
else:
583+
exists = 'does not exist'
584+
self.debug(f'{argv[0]} {exists}')
585+
raise # re-raise
586+
assert proc.stderr == '' or proc.returncode != 0, proc.stderr
587+
if proc.returncode != 0 and support.verbose:
588+
self.debug(f'# python3 {shlex.join(argv[1:])} failed:')
589+
self.debug(proc.stdout, header='stdout')
590+
self.debug(proc.stderr, header='stderr')
591+
self.assertEqual(proc.returncode, 0)
592+
self.assertEqual(proc.stderr, '')
593+
return proc.stdout
594+
595+
def test_sys_path_0(self):
596+
# The main interpreter's sys.path[0] should be used by subinterpreters.
597+
script = '''
598+
import sys
599+
from test.support import interpreters
600+
601+
orig = sys.path[0]
602+
603+
interp = interpreters.create()
604+
interp.run(f"""if True:
605+
import json
606+
import sys
607+
print(json.dumps({{
608+
'main': {orig!r},
609+
'sub': sys.path[0],
610+
}}, indent=4), flush=True)
611+
""")
612+
'''
613+
# <tmp>/
614+
# pkg/
615+
# __init__.py
616+
# __main__.py
617+
# script.py
618+
# script.py
619+
cwd = self.create_temp_dir()
620+
self.write_script(cwd, 'pkg', '__init__.py', text='')
621+
self.write_script(cwd, 'pkg', '__main__.py', text=script)
622+
self.write_script(cwd, 'pkg', 'script.py', text=script)
623+
self.write_script(cwd, 'script.py', text=script)
624+
625+
cases = [
626+
('script.py', cwd),
627+
('-m script', cwd),
628+
('-m pkg', cwd),
629+
('-m pkg.script', cwd),
630+
('-c "import script"', ''),
631+
]
632+
for argv, expected in cases:
633+
with self.subTest(f'python3 {argv}'):
634+
out = self.run_python(argv, cwd=cwd)
635+
data = json.loads(out)
636+
sp0_main, sp0_sub = data['main'], data['sub']
637+
self.assertEqual(sp0_sub, sp0_main)
638+
self.assertEqual(sp0_sub, expected)
639+
# XXX Also check them all with the -P cmdline flag?
640+
641+
491642
class FinalizationTests(TestBase):
492643

493644
def test_gh_109793(self):

‎Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
``sys.path[0]`` is now set correctly for subinterpreters.

‎Modules/main.c

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -556,6 +556,11 @@ pymain_run_python(int *exitcode)
556556
goto error;
557557
}
558558

559+
// XXX Calculate config->sys_path_0 in getpath.py.
560+
// The tricky part is that we can't check the path importers yet
561+
// at that point.
562+
assert(config->sys_path_0 == NULL);
563+
559564
if (config->run_filename != NULL) {
560565
/* If filename is a package (ex: directory or ZIP file) which contains
561566
__main__.py, main_importer_path is set to filename and will be
@@ -571,24 +576,37 @@ pymain_run_python(int *exitcode)
571576
// import readline and rlcompleter before script dir is added to sys.path
572577
pymain_import_readline(config);
573578

579+
PyObject *path0 = NULL;
574580
if (main_importer_path != NULL) {
575-
if (pymain_sys_path_add_path0(interp, main_importer_path) < 0) {
576-
goto error;
577-
}
581+
path0 = Py_NewRef(main_importer_path);
578582
}
579583
else if (!config->safe_path) {
580-
PyObject *path0 = NULL;
581584
int res = _PyPathConfig_ComputeSysPath0(&config->argv, &path0);
582585
if (res < 0) {
583586
goto error;
584587
}
585-
586-
if (res > 0) {
587-
if (pymain_sys_path_add_path0(interp, path0) < 0) {
588-
Py_DECREF(path0);
589-
goto error;
590-
}
588+
else if (res == 0) {
589+
Py_CLEAR(path0);
590+
}
591+
}
592+
// XXX Apply config->sys_path_0 in init_interp_main(). We have
593+
// to be sure to get readline/rlcompleter imported at the correct time.
594+
if (path0 != NULL) {
595+
wchar_t *wstr = PyUnicode_AsWideCharString(path0, NULL);
596+
if (wstr == NULL) {
591597
Py_DECREF(path0);
598+
goto error;
599+
}
600+
config->sys_path_0 = _PyMem_RawWcsdup(wstr);
601+
PyMem_Free(wstr);
602+
if (config->sys_path_0 == NULL) {
603+
Py_DECREF(path0);
604+
goto error;
605+
}
606+
int res = pymain_sys_path_add_path0(interp, path0);
607+
Py_DECREF(path0);
608+
if (res < 0) {
609+
goto error;
592610
}
593611
}
594612

‎Python/initconfig.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = {
9797
SPEC(pythonpath_env, WSTR_OPT),
9898
SPEC(home, WSTR_OPT),
9999
SPEC(platlibdir, WSTR),
100+
SPEC(sys_path_0, WSTR_OPT),
100101
SPEC(module_search_paths_set, UINT),
101102
SPEC(module_search_paths, WSTR_LIST),
102103
SPEC(stdlib_dir, WSTR_OPT),
@@ -770,6 +771,7 @@ PyConfig_Clear(PyConfig *config)
770771
CLEAR(config->exec_prefix);
771772
CLEAR(config->base_exec_prefix);
772773
CLEAR(config->platlibdir);
774+
CLEAR(config->sys_path_0);
773775

774776
CLEAR(config->filesystem_encoding);
775777
CLEAR(config->filesystem_errors);
@@ -3051,6 +3053,7 @@ _Py_DumpPathConfig(PyThreadState *tstate)
30513053
PySys_WriteStderr(" import site = %i\n", config->site_import);
30523054
PySys_WriteStderr(" is in build tree = %i\n", config->_is_python_build);
30533055
DUMP_CONFIG("stdlib dir", stdlib_dir);
3056+
DUMP_CONFIG("sys.path[0]", sys_path_0);
30543057
#undef DUMP_CONFIG
30553058

30563059
#define DUMP_SYS(NAME) \

‎Python/pylifecycle.c

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1209,6 +1209,31 @@ init_interp_main(PyThreadState *tstate)
12091209
}
12101210
}
12111211

1212+
if (!is_main_interp) {
1213+
// The main interpreter is handled in Py_Main(), for now.
1214+
if (config->sys_path_0 != NULL) {
1215+
PyObject *path0 = PyUnicode_FromWideChar(config->sys_path_0, -1);
1216+
if (path0 == NULL) {
1217+
return _PyStatus_ERR("can't initialize sys.path[0]");
1218+
}
1219+
PyObject *sysdict = interp->sysdict;
1220+
if (sysdict == NULL) {
1221+
Py_DECREF(path0);
1222+
return _PyStatus_ERR("can't initialize sys.path[0]");
1223+
}
1224+
PyObject *sys_path = PyDict_GetItemWithError(sysdict, &_Py_ID(path));
1225+
if (sys_path == NULL) {
1226+
Py_DECREF(path0);
1227+
return _PyStatus_ERR("can't initialize sys.path[0]");
1228+
}
1229+
int res = PyList_Insert(sys_path, 0, path0);
1230+
Py_DECREF(path0);
1231+
if (res) {
1232+
return _PyStatus_ERR("can't initialize sys.path[0]");
1233+
}
1234+
}
1235+
}
1236+
12121237
assert(!_PyErr_Occurred(tstate));
12131238

12141239
return _PyStatus_OK();

0 commit comments

Comments
 (0)
0