8000 gh-114809: Support fat builds on macOS with the experimental JIT by ronaldoussoren · Pull Request #115742 · python/cpython · GitHub
[go: up one dir, main page]
Skip to content

gh-114809: Support fat builds on macOS with the experimental JIT #115742

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

gh-114809: Support fat builds on macOS with the experimental JIT #115742

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions Tools/jit/_targets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h"
TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c"

ASYNCIO_RUNNER=asyncio.Runner()


_S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection)
_R = typing.TypeVar(
Expand Down Expand Up @@ -106,6 +108,7 @@ async def _compile(
o = tempdir / f"{opname}.o"
args = [
f"--target={self.triple}",
"-isysroot", "/Users/ronald/Applications/Xcode-beta.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk",
Copy link
Contributor Author
10000

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will be removed in the final version, and likely needs a different issue and patch: I installed llvm using a download on llvm's GitHub, and use Xcode to build python (not Command Line Tools). Without this addition "clang-16" cannot find the SDK to use.

"-DPy_BUILD_CORE",
"-D_DEBUG" if self.debug else "-DNDEBUG",
f"-D_JIT_OPCODE={opname}",
Expand Down Expand Up @@ -152,17 +155,17 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]:
tasks.append(group.create_task(coro, name=opname))
return {task.get_name(): task.result() for task in tasks}

def build(self, out: pathlib.Path, *, comment: str = "") -> None:
def build(self, out: pathlib.Path, *, comment: str = "", stencils_h: str = "jit_stencils.h") -> None:
"""Build jit_stencils.h in the given directory."""
digest = f"// {self._compute_digest(out)}\n"
jit_stencils = out / "jit_stencils.h"
jit_stencils = out / stencils_h
if (
not self.force
and jit_stencils.exists()
and jit_stencils.read_text().startswith(digest)
):
return
stencil_groups = asyncio.run(self._build_stencils())
stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils())
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if this is the right way to enable running multiple builds, I'm a very light asyncio user. Without this change the second run fails due to running with a different runloop.

with jit_stencils.open("w") as file:
file.write(digest)
if comment:
Expand Down
33 changes: 28 additions & 5 deletions Tools/jit/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
comment = f"$ {shlex.join([sys.executable] + sys.argv)}"
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"target", type=_targets.get_target, help="a PEP 11 target triple to compile for"
"target", nargs="+", type=_targets.get_target, help="a PEP 11 target triple to compile for"
)
parser.add_argument(
"-d", "--debug", action="store_true", help="compile for a debug build of Python"
Expand All @@ -22,7 +22,30 @@
"-v", "--verbose", action="store_true", help="echo commands as they are run"
)
args = parser.parse_args()
args.target.debug = args.debug
args.target.force = args.force
args.target.verbose = args.verbose
args.target.build(pathlib.Path.cwd(), comment=comment)

if len(args.target) == -1:
args.target.debug = args.debug
args.target.force = args.force
args.target.verbose = args.verbose
args.target.build(pathlib.Path.cwd(), comment=comment)

else:
# Multiple triples specified, assume this is a macOS multiarchitecture build
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This bit is not ideal, multiple files are created and that likely breaks dependency handling in the Makefile.

It should be possible to publish everything in a single header file, but that requires more significant changes to the builder.

# - Generate multiple stencil headers
# - Generate a helper header that include sthe stencils for the current
# architecture.
for target in args.target:
target.debug = args.debug
target.force = args.force
target.verbose = args.verbose
target.build(pathlib.Path.cwd(), comment=comment, stencils_h=f"jit_stencils-{target.triple}.h")

with open("jit_stencils.h", "w") as fp:
for idx, target in enumerate(args.target):
cpu, _, _ = target.triple.partition("-")
fp.write(f"#{'if' if idx == 0 else 'elif'} defined(__{cpu}__)\n")
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This works for x86_64 and arm64, but might break for ancient systems running PPC and PPC64. I don't care for those.

fp.write(f'# include "jit_stencils-{target.triple}.h"\n')

fp.write("#else\n")
fp.write('# error "unexpected cpu type"\n')
fp.write("#endif\n")
75 changes: 42 additions & 33 deletions configure
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

51 changes: 30 additions & 21 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1621,27 +1621,6 @@ else
AC_MSG_RESULT([no])
fi

# Check for --enable-experimental-jit:
AC_MSG_CHECKING([for --enable-experimental-jit])
AC_ARG_ENABLE([experimental-jit],
[AS_HELP_STRING([--enable-experimental-jit],
[build the experimental just-in-time compiler (default is no)])],
[],
[enable_experimental_jit=no])
AS_VAR_IF([enable_experimental_jit],
[no],
[],
[AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"])
AS_VAR_SET([REGEN_JIT_COMMAND],
["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"])
AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"])
AS_VAR_IF([Py_DEBUG],
[true],
[AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])],
[])])
AC_SUBST([REGEN_JIT_COMMAND])
AC_SUBST([JIT_STENCILS_H])
AC_MSG_RESULT([$enable_experimental_jit])

# Enable optimization flags
AC_SUBST([DEF_MAKE_ALL_RULE])
Expand Down Expand Up @@ -2436,42 +2415,50 @@ yes)
UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT=""
ARCH_TRIPPLES=`echo {ppc,i386}-apple-darwin`
;;
64-bit)
UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT="true"
ARCH_TRIPPLES=`echo {ppc64,x86_64}-apple-darwin`
;;
all)
UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64"
LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386"
ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc"
ARCH_TRIPPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin`
;;
universal2)
UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64"
LIPO_32BIT_FLAGS=""
LIPO_INTEL64_FLAGS="-extract x86_64"
ARCH_RUN_32BIT="true"
ARCH_TRIPPLES=`echo {aarch64,x86_64}-apple-darwin`
;;
intel)
UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64"
LIPO_32BIT_FLAGS="-extract i386"
ARCH_RUN_32BIT="/usr/bin/arch -i386"
ARCH_TRIPPLES=`echo {i386,x86_64}-apple-darwin`
;;
intel-32)
UNIVERSAL_ARCH_FLAGS="-arch i386"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT=""
ARCH_TRIPPLES=i386-apple-darwin
;;
intel-64)
UNIVERSAL_ARCH_FLAGS="-arch x86_64"
LIPO_32BIT_FLAGS=""
ARCH_RUN_32BIT="true"
ARCH_TRIPPLES=x86_64-apple-darwin
;;
3-way)
UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64"
LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386"
ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc"
ARCH_TRIPPLES=`echo {i386,ppc,x86_64}-apple-darwin`
;;
*)
AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way])
Expand Down Expand Up @@ -2565,6 +2552,28 @@ yes)
;;
esac

# Check for --enable-experimental-jit:
AC_MSG_CHECKING([for --enable-experimental-jit])
AC_ARG_ENABLE([experimental-jit],
[AS_HELP_STRING([--enable-experimental-jit],
[build the experimental just-in-time compiler (default is no)])],
[],
[enable_experimental_jit=no])
AS_VAR_IF([enable_experimental_jit],
[no],
[],
[AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"])
AS_VAR_SET([REGEN_JIT_COMMAND],
["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPPLES:-$host}"])
AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"])
AS_VAR_IF([Py_DEBUG],
[true],
[AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])],
[])])
AC_SUBST([REGEN_JIT_COMMAND])
AC_SUBST([JIT_STENCILS_H])
AC_MSG_RESULT([$enable_experimental_jit])

case "$CC_BASENAME" in
*mpicc*)
CFLAGS_NODIST="$CFLAGS_NODIST"
Expand Down
0