From efd5863ee09843074db0f6ea8ea2ef1017663adb Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Tue, 20 Feb 2024 21:17:19 +0100 Subject: [PATCH] First draft of supporting fat builds on macOS with the experimental JIT --- Tools/jit/_targets.py | 9 ++++-- Tools/jit/build.py | 33 ++++++++++++++++--- configure | 75 ++++++++++++++++++++++++------------------- configure.ac | 51 +++++++++++++++++------------ 4 files changed, 106 insertions(+), 62 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 51b091eb246413..37b3e596bd73a4 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -25,6 +25,8 @@ PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" +ASYNCIO_RUNNER=asyncio.Runner() + _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) _R = typing.TypeVar( @@ -106,6 +108,7 @@ async def _compile( o = tempdir / f"{opname}.o" args = [ f"--target={self.triple}", + "-isysroot", "/Users/ronald/Applications/Xcode-beta.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk", "-DPy_BUILD_CORE", "-D_DEBUG" if self.debug else "-DNDEBUG", f"-D_JIT_OPCODE={opname}", @@ -152,17 +155,17 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: tasks.append(group.create_task(coro, name=opname)) return {task.get_name(): task.result() for task in tasks} - def build(self, out: pathlib.Path, *, comment: str = "") -> None: + def build(self, out: pathlib.Path, *, comment: str = "", stencils_h: str = "jit_stencils.h") -> None: """Build jit_stencils.h in the given directory.""" digest = f"// {self._compute_digest(out)}\n" - jit_stencils = out / "jit_stencils.h" + jit_stencils = out / stencils_h if ( not self.force and jit_stencils.exists() and jit_stencils.read_text().startswith(digest) ): return - stencil_groups = asyncio.run(self._build_stencils()) + stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) with jit_stencils.open("w") as file: file.write(digest) if comment: diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 4d4ace14ebf26c..800d8e31b0f464 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -10,7 +10,7 @@ comment = f"$ {shlex.join([sys.executable] + sys.argv)}" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "target", type=_targets.get_target, help="a PEP 11 target triple to compile for" + "target", nargs="+", type=_targets.get_target, help="a PEP 11 target triple to compile for" ) parser.add_argument( "-d", "--debug", action="store_true", help="compile for a debug build of Python" @@ -22,7 +22,30 @@ "-v", "--verbose", action="store_true", help="echo commands as they are run" ) args = parser.parse_args() - args.target.debug = args.debug - args.target.force = args.force - args.target.verbose = args.verbose - args.target.build(pathlib.Path.cwd(), comment=comment) + + if len(args.target) == -1: + args.target.debug = args.debug + args.target.force = args.force + args.target.verbose = args.verbose + args.target.build(pathlib.Path.cwd(), comment=comment) + + else: + # Multiple triples specified, assume this is a macOS multiarchitecture build + # - Generate multiple stencil headers + # - Generate a helper header that include sthe stencils for the current + # architecture. + for target in args.target: + target.debug = args.debug + target.force = args.force + target.verbose = args.verbose + target.build(pathlib.Path.cwd(), comment=comment, stencils_h=f"jit_stencils-{target.triple}.h") + + with open("jit_stencils.h", "w") as fp: + for idx, target in enumerate(args.target): + cpu, _, _ = target.triple.partition("-") + fp.write(f"#{'if' if idx == 0 else 'elif'} defined(__{cpu}__)\n") + fp.write(f'# include "jit_stencils-{target.triple}.h"\n') + + fp.write("#else\n") + fp.write('# error "unexpected cpu type"\n') + fp.write("#endif\n") diff --git a/configure b/configure index ba2d49df7c65fe..d47ae755643d12 100755 --- a/configure +++ b/configure @@ -893,6 +893,8 @@ LDSHARED SHLIB_SUFFIX DSYMUTIL_PATH DSYMUTIL +JIT_STENCILS_H +REGEN_JIT_COMMAND UNIVERSAL_ARCH_FLAGS WASM_STDLIB WASM_ASSETS_DIR @@ -920,8 +922,6 @@ LLVM_AR PROFILE_TASK DEF_MAKE_RULE DEF_MAKE_ALL_RULE -JIT_STENCILS_H -REGEN_JIT_COMMAND ABIFLAGS LN MKDIR_P @@ -1079,11 +1079,11 @@ with_pydebug with_trace_refs enable_pystats with_assertions -enable_experimental_jit enable_optimizations with_lto enable_bolt with_strict_overflow +enable_experimental_jit with_dsymutil with_address_sanitizer with_memory_sanitizer @@ -1807,13 +1807,13 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) - --enable-experimental-jit - build the experimental just-in-time compiler - (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer (default is no) + --enable-experimental-jit + build the experimental just-in-time compiler + (default is no) --enable-loadable-sqlite-extensions support loadable extensions in the sqlite3 module, see Doc/library/sqlite3.rst (default is no) @@ -8036,33 +8036,6 @@ else printf "%s\n" "no" >&6; } fi -# Check for --enable-experimental-jit: -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5 -printf %s "checking for --enable-experimental-jit... " >&6; } -# Check whether --enable-experimental-jit was given. -if test ${enable_experimental_jit+y} -then : - enableval=$enable_experimental_jit; -else $as_nop - enable_experimental_jit=no -fi - -if test "x$enable_experimental_jit" = xno -then : - -else $as_nop - as_fn_append CFLAGS_NODIST " -D_Py_JIT" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" - JIT_STENCILS_H="jit_stencils.h" - if test "x$Py_DEBUG" = xtrue -then : - as_fn_append REGEN_JIT_COMMAND " --debug" -fi -fi - - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5 -printf "%s\n" "$enable_experimental_jit" >&6; } # Enable optimization flags @@ -9948,42 +9921,50 @@ printf "%s\n" "$CC" >&6; } UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPPLES=`echo {ppc,i386}-apple-darwin` ;; 64-bit) UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=`echo {ppc64,x86_64}-apple-darwin` ;; all) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin` ;; universal2) UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" LIPO_32BIT_FLAGS="" LIPO_INTEL64_FLAGS="-extract x86_64" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=`echo {aarch64,x86_64}-apple-darwin` ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386" + ARCH_TRIPPLES=`echo {i386,x86_64}-apple-darwin` ;; intel-32) UNIVERSAL_ARCH_FLAGS="-arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPPLES=i386-apple-darwin ;; intel-64) UNIVERSAL_ARCH_FLAGS="-arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=x86_64-apple-darwin ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPPLES=`echo {i386,ppc,x86_64}-apple-darwin` ;; *) as_fn_error $? "proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way" "$LINENO" 5 @@ -10098,6 +10079,34 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac +# Check for --enable-experimental-jit: +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5 +printf %s "checking for --enable-experimental-jit... " >&6; } +# Check whether --enable-experimental-jit was given. +if test ${enable_experimental_jit+y} +then : + enableval=$enable_experimental_jit; +else $as_nop + enable_experimental_jit=no +fi + +if test "x$enable_experimental_jit" = xno +then : + +else $as_nop + as_fn_append CFLAGS_NODIST " -D_Py_JIT" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPPLES:-$host}" + JIT_STENCILS_H="jit_stencils.h" + if test "x$Py_DEBUG" = xtrue +then : + as_fn_append REGEN_JIT_COMMAND " --debug" +fi +fi + + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5 +printf "%s\n" "$enable_experimental_jit" >&6; } + case "$CC_BASENAME" in *mpicc*) CFLAGS_NODIST="$CFLAGS_NODIST" diff --git a/configure.ac b/configure.ac index b39af7422c4c7c..97d8c830a126c6 100644 --- a/configure.ac +++ b/configure.ac @@ -1621,27 +1621,6 @@ else AC_MSG_RESULT([no]) fi -# Check for --enable-experimental-jit: -AC_MSG_CHECKING([for --enable-experimental-jit]) -AC_ARG_ENABLE([experimental-jit], - [AS_HELP_STRING([--enable-experimental-jit], - [build the experimental just-in-time compiler (default is no)])], - [], - [enable_experimental_jit=no]) -AS_VAR_IF([enable_experimental_jit], - [no], - [], - [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"]) - AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) - AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) - AS_VAR_IF([Py_DEBUG], - [true], - [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], - [])]) -AC_SUBST([REGEN_JIT_COMMAND]) -AC_SUBST([JIT_STENCILS_H]) -AC_MSG_RESULT([$enable_experimental_jit]) # Enable optimization flags AC_SUBST([DEF_MAKE_ALL_RULE]) @@ -2436,42 +2415,50 @@ yes) UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPPLES=`echo {ppc,i386}-apple-darwin` ;; 64-bit) UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=`echo {ppc64,x86_64}-apple-darwin` ;; all) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin` ;; universal2) UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" LIPO_32BIT_FLAGS="" LIPO_INTEL64_FLAGS="-extract x86_64" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=`echo {aarch64,x86_64}-apple-darwin` ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386" + ARCH_TRIPPLES=`echo {i386,x86_64}-apple-darwin` ;; intel-32) UNIVERSAL_ARCH_FLAGS="-arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPPLES=i386-apple-darwin ;; intel-64) UNIVERSAL_ARCH_FLAGS="-arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=x86_64-apple-darwin ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPPLES=`echo {i386,ppc,x86_64}-apple-darwin` ;; *) AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way]) @@ -2565,6 +2552,28 @@ yes) ;; esac +# Check for --enable-experimental-jit: +AC_MSG_CHECKING([for --enable-experimental-jit]) +AC_ARG_ENABLE([experimental-jit], + [AS_HELP_STRING([--enable-experimental-jit], + [build the experimental just-in-time compiler (default is no)])], + [], + [enable_experimental_jit=no]) +AS_VAR_IF([enable_experimental_jit], + [no], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"]) + AS_VAR_SET([REGEN_JIT_COMMAND], + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPPLES:-$host}"]) + AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) + AS_VAR_IF([Py_DEBUG], + [true], + [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], + [])]) +AC_SUBST([REGEN_JIT_COMMAND]) +AC_SUBST([JIT_STENCILS_H]) +AC_MSG_RESULT([$enable_experimental_jit]) + case "$CC_BASENAME" in *mpicc*) CFLAGS_NODIST="$CFLAGS_NODIST"