From 5fc189d2ff70463a258438ce210871435164ad94 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 13 Jun 2018 03:00:31 -0700 Subject: [PATCH 1/7] Patches to build on clang-cl This was mostly: - forward declare struct timeval in pytime (it should have been anyway) - moving struct packing in the correct place - Using a more portable, standard garunteed stringize macro - defining compiler names --- Include/pytime.h | 6 ++++++ .../2018-06-13-03-08-13.bpo-33351.fn4g9Z.rst | 3 +++ Modules/_tracemalloc.c | 15 +++++++++++---- PC/pyconfig.h | 14 ++++++-------- 4 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2018-06-13-03-08-13.bpo-33351.fn4g9Z.rst diff --git a/Include/pytime.h b/Include/pytime.h index bdda1da2e6b8f2..43eba304c2b89d 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -13,6 +13,12 @@ functions and constants extern "C" { #endif +#if defined(_MSC_VER) + /* Forward declare struct timeval so that clang-cl doesn't complain about it + being a local declaration later on in _PyTime_AsTimeval.*/ + struct timeval; +#endif /* _MSC_VER */ + /* _PyTime_t: Python timestamp with subsecond precision. It can be used to store a duration, and so indirectly a date (related to another date, like UNIX epoch). */ diff --git a/Misc/NEWS.d/next/Build/2018-06-13-03-08-13.bpo-33351.fn4g9Z.rst b/Misc/NEWS.d/next/Build/2018-06-13-03-08-13.bpo-33351.fn4g9Z.rst new file mode 100644 index 00000000000000..0d02f3371e06b4 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2018-06-13-03-08-13.bpo-33351.fn4g9Z.rst @@ -0,0 +1,3 @@ +Port CPython to build with clang-cl on Windows. + +Patch by Ethan Smith diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index ddf6ef4e11dd4f..10466b22092b9e 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -48,7 +48,10 @@ static PyThread_type_lock tables_lock; #define DEFAULT_DOMAIN 0 -/* Pack the frame_t structure to reduce the memory footprint. */ +/* Pack the pointer_t structure to reduce the memory footprint. */ +#if defined(_MSC_VER) +#pragma pack(push, 4) +#endif typedef struct #ifdef __GNUC__ __attribute__((packed)) @@ -57,14 +60,18 @@ __attribute__((packed)) uintptr_t ptr; unsigned int domain; } pointer_t; +#ifdef _MSC_VER +#pragma pack(pop) +#endif /* Pack the frame_t structure to reduce the memory footprint on 64-bit - architectures: 12 bytes instead of 16. */ +architectures: 12 bytes instead of 16. */ +#if defined(_MSC_VER) +#pragma pack(push, 4) +#endif typedef struct #ifdef __GNUC__ __attribute__((packed)) -#elif defined(_MSC_VER) -#pragma pack(push, 4) #endif { /* filename cannot be NULL: "" is used if the Python frame diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 424c5be370927f..0e07f878891574 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -94,15 +94,9 @@ WIN32 is still required for the locale module. /* e.g., this produces, after compile-time string catenation, * ("[MSC v.1200 32 bit (Intel)]") * - * _Py_STRINGIZE(_MSC_VER) expands to - * _Py_STRINGIZE1((_MSC_VER)) expands to - * _Py_STRINGIZE2(_MSC_VER) but as this call is the result of token-pasting - * it's scanned again for macros and so further expands to (under MSVC 6) - * _Py_STRINGIZE2(1200) which then expands to - * "1200" + * The double-stringize hack, a method to get the string version of _MSC_VER */ -#define _Py_STRINGIZE(X) _Py_STRINGIZE1((X)) -#define _Py_STRINGIZE1(X) _Py_STRINGIZE2 ## X +#define _Py_STRINGIZE(X) _Py_STRINGIZE2(X) #define _Py_STRINGIZE2(X) #X /* MSVC defines _WINxx to differentiate the windows platform types @@ -122,6 +116,8 @@ WIN32 is still required for the locale module. #if defined(_M_X64) || defined(_M_AMD64) #if defined(__INTEL_COMPILER) #define COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 64 bit (amd64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#elif defined(__clang__) +#define COMPILER ("[clang v." _Py_STRINGIZE(__clang_version__) " 64 bit (amd64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #else #define COMPILER _Py_PASTE_VERSION("64 bit (AMD64)") #endif /* __INTEL_COMPILER */ @@ -175,6 +171,8 @@ typedef _W64 int ssize_t; #if defined(_M_IX86) #if defined(__INTEL_COMPILER) #define COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#elif defined(__clang__) +#define COMPILER ("[clang v." _Py_STRINGIZE(__clang_version__) "32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #else #define COMPILER _Py_PASTE_VERSION("32 bit (Intel)") #endif /* __INTEL_COMPILER */ From 402596b25eba2832892be9838bd40f9abbb6088c Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Wed, 5 Feb 2020 23:02:00 -0600 Subject: [PATCH 2/7] Add clang-cl jobs to azure --- .azure-pipelines/ci.yml | 14 ++++++++++++++ .azure-pipelines/windows-steps.yml | 12 ++++++++++++ 2 files changed, 26 insertions(+) diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 4c2f115cd9b600..5c58b0a89aa19f 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -138,13 +138,27 @@ jobs: win32: arch: win32 buildOpt: + installClang: no testRunTitle: '$(Build.SourceBranchName)-win32' testRunPlatform: win32 win64: arch: amd64 buildOpt: '-p x64' + installClang: no testRunTitle: '$(Build.SourceBranchName)-win64' testRunPlatform: win64 + win32_clang_cl: + arch: win32 + buildOpt: '"/p:CLToolExe=clang-cl.exe" "/p:CLToolPath=C:\Program Files (x86)\LLVM\bin"' + installClang: yes + testRunTitle: '$(Build.SourceBranchName)-clang_cl-win32' + testRunPlatform: win32 + win64_clang_cl: + arch: amd64 + buildOpt: '-p x64 "/p:CLToolExe=clang-cl.exe" "/p:CLToolPath=C:\Program Files\LLVM\bin"' + installClang: yes + testRunTitle: '$(Build.SourceBranchName)-clang_cl-win64' + testRunPlatform: win64 maxParallel: 4 steps: diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml index f502c40637c310..612591c928ff41 100644 --- a/.azure-pipelines/windows-steps.yml +++ b/.azure-pipelines/windows-steps.yml @@ -12,6 +12,18 @@ steps: Write-Host '##vso[task.setvariable variable=EXTERNALS_DIR]$(Build.BinariesDirectory)\externals' displayName: Update build locations +- powershell: | + (New-Object System.Net.WebClient).DownloadFile("https://github.com/llvm/llvm-project/releases/download/llvmorg-9.0.1/LLVM-9.0.1-win32.exe", "LLVM-win.exe") + Start-Process .\LLVM-win.exe -ArgumentList '/S' -Wait + displayName: Install clang on win32 + condition: and(eq('yes', variables['installClang']), eq('win32', variables['arch'])) + +- powershell: | + (New-Object System.Net.WebClient).DownloadFile("https://github.com/llvm/llvm-project/releases/download/llvmorg-9.0.1/LLVM-9.0.1-win64.exe", "LLVM-win.exe") + Start-Process .\LLVM-win.exe -ArgumentList '/S' -Wait + displayName: Install clang on win64 + condition: and(eq('yes', variables['installClang']), eq('amd64', variables['arch'])) + - script: PCbuild\build.bat -e $(buildOpt) displayName: 'Build CPython' env: From e97d807c2e3efdec0a8a69552258e25b1aba5bb4 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Thu, 6 Feb 2020 01:16:43 -0600 Subject: [PATCH 3/7] Add clang-cl support to distutils Thanks to Zufu Liu (@zufuliu) for the code --- Lib/distutils/_msvccompiler.py | 33 +++++++++++++++++++++++++++++++-- Lib/distutils/ccompiler.py | 2 ++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py index e8e4b717b9736f..52c9a4f9cc37e6 100644 --- a/Lib/distutils/_msvccompiler.py +++ b/Lib/distutils/_msvccompiler.py @@ -194,6 +194,14 @@ def _find_exe(exe, paths=None): 'win-arm64' : 'x86_arm64' } +# A map keyed by get_platform() return values to value accepted by +# clang as the triple. +PLAT_TO_LLVM_TARGETS = { + 'win32': 'i686', + 'win-amd64': 'x86_64', + 'win-arm64': 'aarch64', +} + # A set containing the DLLs that are guaranteed to be available for # all micro versions of this Python version. Known extension # dependencies that are not in this set will be copied to the output @@ -231,11 +239,12 @@ class MSVCCompiler(CCompiler) : exe_extension = '.exe' - def __init__(self, verbose=0, dry_run=0, force=0): + def __init__(self, verbose=0, dry_run=0, force=0, use_clang_cl=False): CCompiler.__init__ (self, verbose, dry_run, force) # target platform (.plat_name is consistent with 'bdist') self.plat_name = None self.initialized = False + self.use_clang_cl = use_clang_cl def initialize(self, plat_name=None): # multi-init means we would need to check platform same each time... @@ -257,7 +266,10 @@ def initialize(self, plat_name=None): self._paths = vc_env.get('path', '') paths = self._paths.split(os.pathsep) - self.cc = _find_exe("cl.exe", paths) + if self.use_clang_cl: + self.cc = _find_exe("clang-cl.exe") + else: + self.cc = _find_exe("cl.exe", paths) self.linker = _find_exe("link.exe", paths) self.lib = _find_exe("lib.exe", paths) self.rc = _find_exe("rc.exe", paths) # resource compiler @@ -295,6 +307,16 @@ def initialize(self, plat_name=None): ldflags_debug = [ '/nologo', '/INCREMENTAL:NO', '/LTCG', '/DEBUG:FULL' ] + if self.use_clang_cl: + # Add target for clang + target_flag = "{}-pc-windows-msvc".format(PLAT_TO_LLVM_TARGETS[plat_name]) + self.compile_options.append(target_flag) + self.compile_options_debug.append(target_flag) + # Remove whole program optimization flags to avoid warnings about + # unrecognized options + self.compile_options.remove('/GL') + ldflags.remove('/LTCG') + ldflags_debug.remove('/LTCG') self.ldflags_exe = [*ldflags, '/MANIFEST:EMBED,ID=1'] self.ldflags_exe_debug = [*ldflags_debug, '/MANIFEST:EMBED,ID=1'] @@ -587,3 +609,10 @@ def find_library_file(self, dirs, lib, debug=0): else: # Oops, didn't find it in *any* of 'dirs' return None + + +class ClangMSVCCompiler(MSVCCompiler): + compiler_type = 'clang-cl' + + def __init__(self, verbose=0, dry_run=0, force=0): + MSVCCompiler.__init__(self, verbose, dry_run, force, True) diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py index 4cfc6c7065e922..d5143b5c646779 100644 --- a/Lib/distutils/ccompiler.py +++ b/Lib/distutils/ccompiler.py @@ -968,6 +968,8 @@ def get_default_compiler(osname=None, platform=None): "Mingw32 port of GNU C Compiler for Win32"), 'bcpp': ('bcppcompiler', 'BCPPCompiler', "Borland C++ Compiler"), + 'clang-cl':('_msvccompiler', 'ClangMSVCCompiler', + "clang-cl for Microsoft Visual C++"), } def show_compilers(): From 7b8fb96552ef3c350011364cd095fe3d282f44b4 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Thu, 6 Feb 2020 02:12:37 -0600 Subject: [PATCH 4/7] Remove win32 clang-cl Azure jobs for now --- .azure-pipelines/ci.yml | 6 ------ .azure-pipelines/pr | 0 .azure-pipelines/pr.yml | 9 +++++++++ 3 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 .azure-pipelines/pr diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 5c58b0a89aa19f..d81613cebd4504 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -147,12 +147,6 @@ jobs: installClang: no testRunTitle: '$(Build.SourceBranchName)-win64' testRunPlatform: win64 - win32_clang_cl: - arch: win32 - buildOpt: '"/p:CLToolExe=clang-cl.exe" "/p:CLToolPath=C:\Program Files (x86)\LLVM\bin"' - installClang: yes - testRunTitle: '$(Build.SourceBranchName)-clang_cl-win32' - testRunPlatform: win32 win64_clang_cl: arch: amd64 buildOpt: '-p x64 "/p:CLToolExe=clang-cl.exe" "/p:CLToolPath=C:\Program Files\LLVM\bin"' diff --git a/.azure-pipelines/pr b/.azure-pipelines/pr new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 73d4f55b864500..07820e5d673525 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -138,16 +138,25 @@ jobs: win32: arch: win32 buildOpt: + installClang: no testRunTitle: '$(System.PullRequest.TargetBranch)-win32' testRunPlatform: win32 win64: arch: amd64 buildOpt: '-p x64' + installClang: no testRunTitle: '$(System.PullRequest.TargetBranch)-win64' testRunPlatform: win64 winarm64: arch: arm64 buildOpt: '-p arm64' + installClang: no + win64_clang_cl: + arch: amd64 + buildOpt: '-p x64 "/p:CLToolExe=clang-cl.exe" "/p:CLToolPath=C:\Program Files\LLVM\bin"' + installClang: yes + testRunTitle: '$(Build.SourceBranchName)-clang_cl-win64' + testRunPlatform: win64 maxParallel: 4 steps: From 15edc23a8307498d07525c4439e6e524cf13dba9 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 25 Feb 2020 10:09:39 -0600 Subject: [PATCH 5/7] Add arm LLVM target --- Lib/distutils/_msvccompiler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py index 52c9a4f9cc37e6..b65b1f33e1841e 100644 --- a/Lib/distutils/_msvccompiler.py +++ b/Lib/distutils/_msvccompiler.py @@ -199,6 +199,7 @@ def _find_exe(exe, paths=None): PLAT_TO_LLVM_TARGETS = { 'win32': 'i686', 'win-amd64': 'x86_64', + 'win-arm32': 'arm', 'win-arm64': 'aarch64', } From 79fe165efe95f7fa87a058a504773ee8397794d6 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Thu, 27 Feb 2020 17:43:32 -0600 Subject: [PATCH 6/7] Fix target flag Also use the full target triple in the map --- Lib/distutils/_msvccompiler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py index b65b1f33e1841e..a9fbbf9824f092 100644 --- a/Lib/distutils/_msvccompiler.py +++ b/Lib/distutils/_msvccompiler.py @@ -197,10 +197,10 @@ def _find_exe(exe, paths=None): # A map keyed by get_platform() return values to value accepted by # clang as the triple. PLAT_TO_LLVM_TARGETS = { - 'win32': 'i686', - 'win-amd64': 'x86_64', - 'win-arm32': 'arm', - 'win-arm64': 'aarch64', + 'win32': 'i686-pc-windows-msvc', + 'win-amd64': 'x86_64-pc-windows-msvc', + 'win-arm32': 'arm-pc-windows-msvc', + 'win-arm64': 'aarch64-pc-windows-msvc', } # A set containing the DLLs that are guaranteed to be available for @@ -310,7 +310,7 @@ def initialize(self, plat_name=None): ] if self.use_clang_cl: # Add target for clang - target_flag = "{}-pc-windows-msvc".format(PLAT_TO_LLVM_TARGETS[plat_name]) + target_flag = "--target=" + PLAT_TO_LLVM_TARGETS[plat_name] self.compile_options.append(target_flag) self.compile_options_debug.append(target_flag) # Remove whole program optimization flags to avoid warnings about From 0ce40aceff57640a482bedbec2c85d1361cd3a43 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Sat, 4 Jul 2020 15:26:13 -0500 Subject: [PATCH 7/7] Fix formatting --- Modules/_tracemalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index b07e5434b37deb..d2784b1aa88f8e 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -51,7 +51,7 @@ static PyThread_type_lock tables_lock; #define DEFAULT_DOMAIN 0 /* Pack the frame_t structure to reduce the memory footprint on 64-bit -architectures: 12 bytes instead of 16. */ + architectures: 12 bytes instead of 16. */ #if defined(_MSC_VER) #pragma pack(push, 4) #endif