8000 gh-115382: Remove foreign import paths from cross compiles · vfazio/cpython@601a820 · GitHub
[go: up one dir, main page]
Skip to content

Commit 601a820

Browse files
vfaziovfazio
committed
pythongh-115382: Remove foreign import paths from cross compiles
Previously, when a build was configured to use a host interpreter via --with-build-python, the PYTHON_FOR_BUILD config value included a path in PYTHONPATH that pointed to the target's built external modules. For "normal" foreign architecture cross compiles, when loading compiled external libraries, the target libraries were processed first due to their precedence in sys.path. These libraries are then ruled out because of a mismatch in the SOABI so the import mechanism continues searching in sys.path for modules until it finds the host's native modules. However, if the host interpreter and the target python are on the same version + SOABI combination, the host interpreter would attempt to load the target's external modules due precedence in sys.path. Despite the "match", the target build may be linked against a different libc or may include instructions that are not supported on the host, so loading/executing the target's external modules can lead to crashes. Now, the path to the target's external modules is no longer in PYTHONPATH to prevent accidentally loading these foreign modules. Some build scripts need to interrogate sysconfig via `get_config_var{s}` to determine what target modules were built as well as other target specific config values. This was previously done by specifying _PYTHON_SYSCONFIGDATA_NAME in the environment and leveraging the target's module path in PYTHONPATH so it could be imported in sysconfig._init_posix. These build scripts now check if the environment is configured to use a host interpreter and will now load the target's sysconfigdata module based on the information in the environment and query it as necessary. Signed-off-by: Vincent Fazio <vfazio@gmail.com>
1 parent 0656509 commit 601a820

File tree

4 files changed

+66
-6
lines changed

4 files changed

+66
-6
lines changed

‎Lib/ensurepip/__init__.py

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,32 @@
1616
# policies recommend against bundling dependencies. For example, Fedora
1717
# installs wheel packages in the /usr/share/python-wheels/ directory and don't
1818
# install the ensurepip._bundled package.
19-
if (_pkg_dir := sysconfig.get_config_var('WHEEL_PKG_DIR')) is not None:
20-
_WHEEL_PKG_DIR = Path(_pkg_dir).resolve()
19+
if '_PYTHON_HOST_PLATFORM' in os.environ:
20+
# When invoked during a cross compile, use the sysconfigdata file from the build directory.
21+
22+
from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES
23+
from importlib.util import module_from_spec
24+
25+
build_dir = os.environ.get("_PYTHON_PROJECT_BASE")
26+
pybuild = os.path.join(build_dir, "pybuilddir.txt")
27+
if os.path.exists(pybuild):
28+
with open(pybuild, encoding="utf-8") as f:
29+
builddir = f.read()
30+
target_lib_dir = os.path.join(build_dir, builddir)
31+
spec = FileFinder(target_lib_dir ,(SourceFileLoader, SOURCE_SUFFIXES)).find_spec(
32+
os.environ.get("_PYTHON_SYSCONFIGDATA_NAME")
33+
)
34+
if spec is not None:
35+
target_module = module_from_spec(spec)
36+
spec.loader.exec_module(target_module)
37+
38+
if (_pkg_dir := target_module.build_time_vars.get('WHEEL_PKG_DIR')) is not None:
39+
_WHEEL_PKG_DIR = Path(_pkg_dir).resolve()
40+
elif (_pkg_dir := sysconfig.get_config_var('WHEEL_PKG_DIR')) is not None:
41+
_WHEEL_PKG_DIR = Path(_pkg_dir).resolve()
2142
else:
2243
_WHEEL_PKG_DIR = None
2344

24-
2545
def _find_wheel_pkg_dir_pip():
2646
if _WHEEL_PKG_DIR is None:
2747
# NOTE: The compile-time `WHEEL_PKG_DIR` is unset so there is no place

‎Tools/build/check_extension_modules.py

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,41 @@ def __bool__(self):
124124

125125
ModuleInfo = collections.namedtuple("ModuleInfo", "name state")
126126

127+
class _SysConfigShim:
128+
"""
129+
A class to shim the sysconfig data from the cross compile build directory.
130+
131+
It's not safe to include foreign import directories in sys.path as the host
132+
compatible interpreter may attempt to load libraries it's incompatible with.
133+
134+
However, we need to interrogate the target build to check that modules were
135+
compiled correctly so we need to load data from sysconfigdata that resides
136+
in the build directory.
137+
"""
138+
def __init__(self, lib_path: pathlib.Path):
139+
from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES
140+
from importlib.util import module_from_spec
141+
142+
configdata = os.environ.get("_PYTHON_SYSCONFIGDATA_NAME")
143+
if configdata is None:
144+
raise RuntimeError(f'_PYTHON_SYSCONFIGDATA_NAME is required for cross compilation')
145+
spec = FileFinder(str(lib_path) ,(SourceFileLoader, SOURCE_SUFFIXES)).find_spec(configdata)
146+
if spec is None:
147+
raise RuntimeError(f'Could not find find sysconfig data for {configdata}')
148+
149+
self.target_module = module_from_spec(spec)
150+
spec.loader.exec_module(self.target_module)
151+
152+
def get_config_var(self, name: str):
153+
return self.get_config_vars().get(name)
154+
155+
def get_config_vars(self, *args):
156+
if args:
157+
vals = []
158+
for name in args:
159+
vals.append(self.target_module.build_time_vars.get(name))
160+
return vals
161+
return self.target_module.build_time_vars
127162

128163
class ModuleChecker:
129164
pybuilddir_txt = "pybuilddir.txt"
@@ -138,10 +173,15 @@ class ModuleChecker:
138173

139174
def __init__(self, cross_compiling: bool = False, strict: bool = False):
140175
self.cross_compiling = cross_compiling
176+
self.builddir = self.get_builddir()
177+
if self.cross_compiling:
178+
shim = _SysConfigShim(self.builddir)
179+
sysconfig.get_config_var = shim.get_config_var
180+
sysconfig.get_config_vars = shim.get_config_vars
181+
141182
self.strict_extensions_build = strict
142183
self.ext_suffix = sysconfig.get_config_var("EXT_SUFFIX")
143184
self.platform = sysconfig.get_platform()
144-
self.builddir = self.get_builddir()
145185
self.modules = self.get_modules()
146186

147187
self.builtin_ok = []

‎configure

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎configure.ac

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ AC_ARG_WITH([build-python],
164164
dnl Build Python interpreter is used for regeneration and freezing.
165165
ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python
166166
PYTHON_FOR_FREEZE="$with_build_python"
167-
PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python
167+
PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python
168168
AC_MSG_RESULT([$with_build_python])
169169
], [
170170
AS_VAR_IF([cross_compiling], [yes],

0 commit comments

Comments
 (0)
0