[windows][toolchain] Build sanitizers and builtins standalone for all SDKs (#78861)

Unified builds of compiler-rt together with LLVM failed for the Android SDKs. It got too complicated to redirect the way LLVM would configure the nested build-trees. Standalone builds slightly increase build time, but they turned out much simpler and we end up with less duplication of definitions.

Author: weliveindetail

cmake/caches/Windows-aarch64.cmake

@@ -26,22 +26,6 @@ set(LLVM_DEFAULT_TARGET_TRIPLE aarch64-unknown-windows-msvc CACHE STRING "")
 set(LLVM_APPEND_VC_REV NO CACHE BOOL "")
 set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR YES CACHE BOOL "")
 set(LLVM_ENABLE_PYTHON YES CACHE BOOL "")
-set(LLVM_RUNTIME_TARGETS
-      aarch64-unknown-windows-msvc
-    CACHE STRING "")
-foreach(target ${LLVM_RUNTIME_TARGETS})
-  set(RUNTIMES_${target}_LLVM_ENABLE_RUNTIMES
-        compiler-rt
-      CACHE STRING "")
-  set(RUNTIMES_${target}_CMAKE_MT mt CACHE STRING "")
-  set(RUNTIMES_${target}_CMAKE_SYSTEM_NAME Windows CACHE STRING "")
-  set(RUNTIMES_${target}_CMAKE_BUILD_TYPE Release CACHE STRING "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_CRT NO CACHE BOOL "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_LIBFUZZER NO CACHE BOOL "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_PROFILE YES CACHE BOOL "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_SANITIZERS NO CACHE BOOL "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_XRAY NO CACHE BOOL "")
-endforeach()
 
 set(LLVM_TARGETS_TO_BUILD AArch64 ARM WebAssembly X86 CACHE STRING "")
 
@@ -167,7 +151,6 @@ set(LLVM_DISTRIBUTION_COMPONENTS
       libclang
       libclang-headers
       LTO
-      runtimes
       ${LLVM_TOOLCHAIN_TOOLS}
       ${CLANG_TOOLS}
       ${LLD_TOOLS}

cmake/caches/Windows-x86_64.cmake

@@ -27,62 +27,6 @@ set(LLVM_APPEND_VC_REV NO CACHE BOOL "")
 set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR YES CACHE BOOL "")
 set(LLVM_ENABLE_PYTHON YES CACHE BOOL "")
 
-set(DEFAULT_BUILTIN_TARGETS
-      x86_64-unknown-windows-msvc
-      aarch64-unknown-windows-msvc)
-# Build the android builtins if NDK path is provided.
-if(NOT "$ENV{NDKPATH}" STREQUAL "")
-  list(APPEND DEFAULT_BUILTIN_TARGETS
-       aarch64-unknown-linux-android
-       x86_64-unknown-linux-android)
-endif()
-
-# The builtin targets are used to build the compiler-rt builtins.
-set(LLVM_BUILTIN_TARGETS ${DEFAULT_BUILTIN_TARGETS} CACHE STRING "")
-
-# The runtime targets are used to build the compiler-rt profile library.
-set(LLVM_RUNTIME_TARGETS
-      x86_64-unknown-windows-msvc
-      aarch64-unknown-windows-msvc
-    CACHE STRING "")
-
-foreach(target ${LLVM_RUNTIME_TARGETS})
-  set(RUNTIMES_${target}_LLVM_ENABLE_RUNTIMES
-        compiler-rt
-      CACHE STRING "")
-  set(RUNTIMES_${target}_CMAKE_MT mt CACHE STRING "")
-  set(RUNTIMES_${target}_CMAKE_SYSTEM_NAME Windows CACHE STRING "")
-  set(RUNTIMES_${target}_CMAKE_BUILD_TYPE Release CACHE STRING "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_BUILTINS NO CACHE BOOL "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_CRT NO CACHE BOOL "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_LIBFUZZER NO CACHE BOOL "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_ORC NO CACHE BOOL "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_PROFILE YES CACHE BOOL "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_SANITIZERS NO CACHE BOOL "")
-  set(RUNTIMES_${target}_COMPILER_RT_BUILD_XRAY NO CACHE BOOL "")
-endforeach()
-
-foreach(target ${LLVM_BUILTIN_TARGETS})
-  set(BUILTINS_${target}_CMAKE_MT mt CACHE STRING "")
-  if(${target} MATCHES windows-msvc)
-    set(BUILTINS_${target}_CMAKE_SYSTEM_NAME Windows CACHE STRING "")
-  elseif(${target} MATCHES linux-android)
-    # Use a single 'linux' directory and arch-based lib names on Android.
-    set(BUILTINS_${target}_LLVM_ENABLE_PER_TARGET_RUNTIME_DIR NO CACHE BOOL "")
-    set(BUILTINS_${target}_CMAKE_SYSTEM_NAME Android CACHE STRING "")
-    if(${target} MATCHES aarch64)
-      set(BUILTINS_${target}_CMAKE_ANDROID_ARCH_ABI arm64-v8a CACHE STRING "")
-    else()
-      set(BUILTINS_${target}_CMAKE_ANDROID_ARCH_ABI x86_64 CACHE STRING "")
-    endif()
-    set(BUILTINS_${target}_CMAKE_ANDROID_NDK $ENV{NDKPATH} CACHE PATH "")
-    set(BUILTINS_${target}_CMAKE_ANDROID_API 21 CACHE STRING "")
-    set(BUILTINS_${target}_CMAKE_C_COMPILER_TARGET "${target}21" CACHE STRING "")
-    set(BUILTINS_${target}_CMAKE_CXX_COMPILER_TARGET "${target}21" CACHE STRING "")
-  endif()
-  set(BUILTINS_${target}_CMAKE_BUILD_TYPE Release CACHE STRING "")
-endforeach()
-
 set(LLVM_TARGETS_TO_BUILD AArch64 ARM WebAssembly X86 CACHE STRING "")
 
 # Disable certain targets to reduce the configure time or to avoid configuration
@@ -207,8 +151,6 @@ set(LLVM_DISTRIBUTION_COMPONENTS
       libclang
       libclang-headers
       LTO
-      builtins
-      runtimes
       ${LLVM_TOOLCHAIN_TOOLS}
       ${CLANG_TOOLS}
       ${LLD_TOOLS}

test/Driver/sanitize_coverage.swift

@@ -1,3 +1,5 @@
+// XFAIL: OS=windows-msvc
+
 // Different sanitizer coverage types
 // RUN: %swiftc_driver -driver-print-jobs -sanitize-coverage=func -sanitize=address %s | %FileCheck -check-prefix=SANCOV_FUNC %s
 // RUN: %swiftc_driver -driver-print-jobs -sanitize-coverage=bb -sanitize=address %s | %FileCheck -check-prefix=SANCOV_BB %s

test/IRGen/address_sanitizer_use_odr_indicator.swift

@@ -1,3 +1,4 @@
+// XFAIL: OS=windows-msvc
 // REQUIRES: asan_runtime
 
 // Default instrumentation that does not use ODR indicators

test/Interpreter/indirect_enum.swift

@@ -1,3 +1,5 @@
+// XFAIL: OS=windows-msvc
+
 // RUN: %target-swiftc_driver %s -g -sanitize=address -o %t_asan-binary
 // RUN: %target-codesign %t_asan-binary
 // RUN: env ASAN_OPTIONS=detect_leaks=0 %target-run %t_asan-binary

test/Reflection/typeref_decoding_asan.swift

@@ -1,4 +1,5 @@
 // UNSUPPORTED: OS=linux-gnu && CPU=aarch64
+// XFAIL: OS=windows-msvc
 
 // rdar://100805115
 // UNSUPPORTED: CPU=arm64e

test/Sanitizers/asan/asan.swift

@@ -1,3 +1,5 @@
+// XFAIL: OS=windows-msvc
+
 // RUN: %target-swiftc_driver %s -g -sanitize=address -o %t_asan-binary
 // RUN: %target-codesign %t_asan-binary
 // RUN: env %env-ASAN_OPTIONS=abort_on_error=0 not %target-run %t_asan-binary 2>&1 | %FileCheck %s

test/Sanitizers/sanitizer_coverage.swift

@@ -9,6 +9,7 @@
 // For now restrict this test to platforms where we know this test will pass
 // REQUIRES: CPU=x86_64
 // UNSUPPORTED: remote_run
+// XFAIL: OS=windows-msvc
 
 func sayHello() {
   print("Hello")

utils/build.ps1

@@ -423,6 +423,8 @@ enum TargetComponent {
   Foundation
   XCTest
   Testing
+  ClangBuiltins
+  ClangRuntime
 }
 
 function Get-TargetProjectBinaryCache($Arch, [TargetComponent]$Project) {
@@ -1690,13 +1692,61 @@ function Build-LLVM([Platform]$Platform, $Arch) {
     -Bin (Get-TargetProjectBinaryCache $Arch LLVM) `
     -Arch $Arch `
     -Platform $Platform `
-    -UseMSVCCompilers C,CXX `
+    -UseBuiltCompilers C,CXX `
     -Defines @{
       CMAKE_SYSTEM_NAME = $Platform.ToString();
       LLVM_HOST_TRIPLE = $Arch.LLVMTarget;
     }
 }
 
+function Build-Sanitizers([Platform]$Platform, $Arch) {
+  $LLVMTargetCache = $(Get-TargetProjectBinaryCache $Arch LLVM)
+  $LITVersionStr = $(Invoke-Program $(Get-PythonExecutable) "$LLVMTargetCache\bin\llvm-lit.py" --version)
+  if (-not ($LITVersionStr -match "lit (\d+)\.\d+\.\d+.*")) {
+    throw "Unexpected version string output from llvm-lit.py"
+  }
+  $LLVMVersionMajor = $Matches.1
+  $InstallTo = "$($HostArch.ToolchainInstallRoot)\usr\lib\clang\$LLVMVersionMajor"
+  Write-Host "Sanitizers SDK directory: $InstallTo"
+
+  Build-CMakeProject `
+    -Src $SourceCache\llvm-project\compiler-rt\lib\builtins `
+    -Bin "$(Get-TargetProjectBinaryCache $Arch ClangBuiltins)" `
+    -InstallTo $InstallTo `
+    -Arch $Arch `
+    -Platform $Platform `
+    -UseBuiltCompilers ASM,C,CXX `
+    -BuildTargets "install-compiler-rt" `
+    -Defines (@{
+      CMAKE_SYSTEM_NAME = $Platform.ToString();
+      LLVM_DIR = "$LLVMTargetCache\lib\cmake\llvm";
+      LLVM_ENABLE_PER_TARGET_RUNTIME_DIR = "YES";
+      COMPILER_RT_DEFAULT_TARGET_ONLY = "YES";
+    })
+  
+  Build-CMakeProject `
+    -Src $SourceCache\llvm-project\compiler-rt `
+    -Bin "$(Get-TargetProjectBinaryCache $Arch ClangRuntime)" `
+    -InstallTo $InstallTo `
+    -Arch $Arch `
+    -Platform $Platform `
+    -UseBuiltCompilers ASM,C,CXX `
+    -BuildTargets "install-compiler-rt" `
+    -Defines (@{
+      CMAKE_SYSTEM_NAME = $Platform.ToString();
+      LLVM_DIR = "$LLVMTargetCache\lib\cmake\llvm";
+      LLVM_ENABLE_PER_TARGET_RUNTIME_DIR = "YES";
+      COMPILER_RT_DEFAULT_TARGET_ONLY = "YES";
+      COMPILER_RT_BUILD_BUILTINS = "NO";
+      COMPILER_RT_BUILD_CRT = "NO";
+      COMPILER_RT_BUILD_LIBFUZZER = "NO";
+      COMPILER_RT_BUILD_ORC = "NO";
+      COMPILER_RT_BUILD_XRAY = "NO";
+      COMPILER_RT_BUILD_PROFILE = "YES";
+      COMPILER_RT_BUILD_SANITIZERS = "YES";
+    })
+}
+
 function Build-ZLib([Platform]$Platform, $Arch) {
   $ArchName = $Arch.LLVMName
 
@@ -2919,6 +2969,7 @@ if (-not $SkipBuild) {
     Invoke-BuildStep Build-FoundationMacros -Build Windows $BuildArch
     Invoke-BuildStep Build-TestingMacros -Build Windows $BuildArch
     Invoke-BuildStep Build-Foundation Windows $Arch
+    Invoke-BuildStep Build-Sanitizers Windows $Arch
     Invoke-BuildStep Build-XCTest Windows $Arch
     Invoke-BuildStep Build-Testing Windows $Arch
     Invoke-BuildStep Write-SDKSettingsPlist Windows $Arch
@@ -2938,6 +2989,7 @@ if (-not $SkipBuild) {
     Invoke-BuildStep Build-Runtime Android $Arch
     Invoke-BuildStep Build-Dispatch Android $Arch
     Invoke-BuildStep Build-Foundation Android $Arch
+    Invoke-BuildStep Build-Sanitizers Android $Arch
     Invoke-BuildStep Build-XCTest Android $Arch
     Invoke-BuildStep Build-Testing Android $Arch
     Invoke-BuildStep Write-SDKSettingsPlist Android $Arch