From d8cfc0a5f477239ddaec2aab2cddab4f43ce7ff1 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 1 May 2025 17:20:12 -0700 Subject: [PATCH 1/2] Get LLVM from cpython-bin-deps, and clean things up a bit --- .github/workflows/jit.yml | 2 +- PCbuild/build.bat | 1 + PCbuild/get_externals.bat | 4 +++- PCbuild/regen.targets | 7 +++++++ Tools/jit/README.md | 4 +++- Tools/jit/_llvm.py | 8 ++++++++ Tools/jit/_targets.py | 2 ++ Tools/jit/build.py | 26 ++++++++++++++++---------- 8 files changed, 41 insertions(+), 13 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index af460f4264932c..6c4c5928bde8f2 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -95,10 +95,10 @@ jobs: with: python-version: '3.11' + # PCbuild downloads LLVM automatically: - name: Windows if: runner.os == 'Windows' run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/PCbuild/build.bat b/PCbuild/build.bat index db67ae72981345..26cf7ee526d2e3 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -111,6 +111,7 @@ if "%IncludeExternals%"=="" set IncludeExternals=true if "%IncludeCTypes%"=="" set IncludeCTypes=true if "%IncludeSSL%"=="" set IncludeSSL=true if "%IncludeTkinter%"=="" set IncludeTkinter=true +if "%UseJIT%" NEQ "true" set IncludeLLVM=false if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index b28ce9caeb3f13..49ace616793d8e 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -15,6 +15,7 @@ set IncludeSSLSrc=false if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts if "%~1"=="--no-libffi" (set IncludeLibffi=false) & shift & goto CheckOpts +if "%~1"=="--no-llvm" (set IncludeLLVM=false) & shift & goto CheckOpts if "%~1"=="--tkinter-src" (set IncludeTkinterSrc=true) & shift & goto CheckOpts if "%~1"=="--openssl-src" (set IncludeSSLSrc=true) & shift & goto CheckOpts if "%~1"=="--libffi-src" (set IncludeLibffiSrc=true) & shift & goto CheckOpts @@ -80,6 +81,7 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.16.2 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 +if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-19.1.7.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -98,7 +100,7 @@ goto end :usage echo.Valid options: -c, --clean, --clean-only, --organization, --python, -echo.--no-tkinter, --no-openssl +echo.--no-tkinter, --no-openssl, --no-llvm echo. echo.Pull all sources and binaries necessary for compiling optional extension echo.modules that rely on external libraries. diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 416241d9d0df10..e7822a126c6304 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -30,7 +30,11 @@ <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" /> <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> + <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> + <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> + <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> + <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> @@ -124,6 +128,9 @@ + + + str | None: return path # Versioned executables: path = f"{tool}-{_LLVM_VERSION}" + if await _check_tool_version(path, echo=echo): + return path + # PCbuild externals: + externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) + path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) if await _check_tool_version(path, echo=echo): return path # Homebrew-installed executables: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index e0ac735a20f691..6ceb4404e74ce7 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -23,8 +23,10 @@ TOOLS_JIT = TOOLS_JIT_BUILD.parent TOOLS = TOOLS_JIT.parent CPYTHON = TOOLS.parent +EXTERNALS = CPYTHON / "externals" PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" + ASYNCIO_RUNNER = asyncio.Runner() _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 4d1e484b6838eb..49b08f477dbed7 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -8,6 +8,7 @@ import _targets if __name__ == "__main__": + out = pathlib.Path.cwd().resolve() comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -31,17 +32,22 @@ target.force = args.force target.verbose = args.verbose target.build( - pathlib.Path.cwd(), + out, comment=comment, stencils_h=f"jit_stencils-{target.triple}.h", force=args.force, ) - - with open("jit_stencils.h", "w") as fp: - for idx, target in enumerate(args.target): - fp.write(f"#{'if' if idx == 0 else 'elif'} {target.condition}\n") - fp.write(f'#include "jit_stencils-{target.triple}.h"\n') - - fp.write("#else\n") - fp.write('#error "unexpected target"\n') - fp.write("#endif\n") + jit_stencils_h = out / "jit_stencils.h" + lines = [f"// {comment}\n"] + guard = "#if" + for target in args.target: + lines.append(f"{guard} {target.condition}\n") + lines.append(f'#include "jit_stencils-{target.triple}.h"\n') + guard = "#elif" + lines.append("#else\n") + lines.append('#error "unexpected target"\n') + lines.append("#endif\n") + body = "".join(lines) + # Don't touch the file if it hasn't changed (so we don't trigger a rebuild): + if not jit_stencils_h.is_file() or jit_stencils_h.read_text() != body: + jit_stencils_h.write_text(body) From b57fe8b52792f18d3f21c00cdcdb21e38583f95c Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 1 May 2025 17:27:13 -0700 Subject: [PATCH 2/2] blurb add --- .../next/Build/2025-05-01-17-27-06.gh-issue-113464.vjE5X4.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2025-05-01-17-27-06.gh-issue-113464.vjE5X4.rst diff --git a/Misc/NEWS.d/next/Build/2025-05-01-17-27-06.gh-issue-113464.vjE5X4.rst b/Misc/NEWS.d/next/Build/2025-05-01-17-27-06.gh-issue-113464.vjE5X4.rst new file mode 100644 index 00000000000000..d150b19d5f1687 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-05-01-17-27-06.gh-issue-113464.vjE5X4.rst @@ -0,0 +1,3 @@ +Use the cpython-bin-deps "externals" repository for Windows LLVM dependency +management. Installing LLVM manually is no longer necessary for Windows JIT +builds.