Merge pull request #2339 from swiftwasm/katei/build-only-stdlib

Build only stdlib

Author: kateinoigakukun

.github/workflows/main.yml

@@ -139,7 +139,7 @@ jobs:
       - name: Pack test results
         working-directory: ${{ github.workspace }}/../
         run: |
-          tar cJf ./swift/swift-test-results.tar.gz target-build/*/swift-macosx-x86_64/swift-test-results
+          tar cJf ./swift/swift-test-results.tar.gz target-build/swift-stdlib-wasi-wasm32/swift-test-results
       - name: Upload test results
         uses: actions/upload-artifact@v1
         with:

cmake/caches/Runtime-WASI-wasm32.cmake

@@ -0,0 +1,33 @@
+set(SWIFT_PRIMARY_VARIANT_SDK WASI CACHE STRING "")
+set(SWIFT_PRIMARY_VARIANT_ARCH wasm32 CACHE STRING "")
+set(SWIFT_HOST_VARIANT_SDK NONE CACHE STRING "")
+set(SWIFT_HOST_VARIANT_ARCH NONE CACHE STRING "")
+
+set(SWIFT_SDKS WASI CACHE STRING "")
+
+set(SWIFT_INCLUDE_TOOLS NO CACHE BOOL "")
+set(SWIFT_INCLUDE_TESTS TRUE CACHE BOOL "")
+set(SWIFT_INCLUDE_DOCS NO CACHE BOOL "")
+
+set(SWIFT_BUILD_SYNTAXPARSERLIB NO CACHE BOOL "")
+set(SWIFT_BUILD_SOURCEKIT NO CACHE BOOL "")
+set(SWIFT_ENABLE_SOURCEKIT_TESTS NO CACHE BOOL "")
+
+set(SWIFTWASM_DISABLE_REFLECTION_TEST YES CACHE BOOL "")
+
+# stdlib configurations
+set(SWIFT_BUILD_STATIC_STDLIB YES CACHE BOOL "")
+set(SWIFT_BUILD_DYNAMIC_STDLIB NO CACHE BOOL "")
+set(SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY NO CACHE BOOL "")
+# TODO(katei): This should get turned off, as this is not an ABI stable platform.
+# But current CMake build system doesn't support SWIFT_STDLIB_STABLE_ABI=NO
+set(SWIFT_STDLIB_STABLE_ABI YES CACHE BOOL "")
+
+# build with the host compiler
+set(SWIFT_BUILD_RUNTIME_WITH_HOST_COMPILER YES CACHE BOOL "")
+
+set(WASI_ICU_URL "https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz" CACHE STRING "")
+set(WASI_ICU_MD5 "25943864ebbfff15cf5aee8d9d5cc4d7" CACHE STRING "")
+
+
+set(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME YES CACHE BOOL "")

utils/build-script-impl

@@ -1702,15 +1702,6 @@ for host in "${ALL_HOSTS[@]}"; do
                     fi
                 # fi
 
-                # wasm: SWIFT_WASI_SDK_PATH is used to determine wasi-sysroot
-                # to build compiler-rt for wasm32-wasi
-                if [[ ! "${SKIP_BUILD_WASM}" ]]; then
-                    cmake_options=(
-                        "${cmake_options[@]}"
-                        -DCOMPILER_RT_SWIFT_WASI_SDK_PATH:STRING="${WASI_SDK}"
-                    )
-                fi
-
                 cmake_options+=(
                     -DLLVM_ENABLE_PROJECTS="$(join ";" ${llvm_enable_projects[@]})"
                 )

utils/webassembly/build-foundation.sh

@@ -3,7 +3,7 @@ set -ex
 DESTINATION_TOOLCHAIN=$1
 SOURCE_PATH="$(cd "$(dirname $0)/../../.." && pwd)"
 
-FOUNDATION_BUILD="$SOURCE_PATH/target-build/Ninja-Release/foundation-wasi-wasm32"
+FOUNDATION_BUILD="$SOURCE_PATH/target-build/foundation-wasi-wasm32"
 
 mkdir -p $FOUNDATION_BUILD
 cd $FOUNDATION_BUILD

utils/webassembly/build-presets.ini

@@ -66,71 +66,3 @@ skip-test-libicu
 skip-test-foundation
 skip-test-libdispatch
 skip-test-xctest
-
-[preset: webassembly-target]
-
-mixin-preset=webassembly
-wasm
-build-stdlib-deployment-targets=wasi-wasm32
-build-swift-dynamic-sdk-overlay=false
-build-swift-dynamic-stdlib=false
-build-swift-static-sdk-overlay
-build-swift-static-stdlib
-stdlib-deployment-targets=wasi-wasm32
-wasi-sdk=%(SOURCE_PATH)s/wasi-sdk
-
-
-[preset: webassembly-target-install]
-mixin-preset=
-    webassembly-target
-    webassembly-install
-# Install lld, compiler-rt and clang
-install-llvm
-
-
-[preset: webassembly-linux-target]
-
-mixin-preset=webassembly-target
-extra-cmake-options=
-    -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
-    -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
-    -DSWIFT_PRIMARY_VARIANT_SDK:STRING=WASI
-    -DSWIFT_PRIMARY_VARIANT_ARCH:STRING=wasm32
-    -DSWIFT_SDKS='WASI;LINUX'
-    -DSWIFT_STDLIB_SINGLE_THREADED_RUNTIME=TRUE
-    -DSWIFT_BUILD_SOURCEKIT=FALSE
-    -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
-    -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
-    -DCMAKE_AR="%(LLVM_BIN_DIR)s/llvm-ar"
-    -DCMAKE_RANLIB="%(LLVM_BIN_DIR)s/llvm-ranlib"
-    -DCLANG_COMPILER_RT_CMAKE_ARGS='-DCMAKE_TOOLCHAIN_FILE=%(SOURCE_PATH)s/swift/utils/webassembly/compiler-rt-cache.cmake'
-    -DSWIFTWASM_DISABLE_REFLECTION_TEST=TRUE
-
-[preset: webassembly-macos-target]
-
-mixin-preset=webassembly-target
-extra-cmake-options=
-    -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
-    -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
-    -DSWIFT_PRIMARY_VARIANT_SDK:STRING=WASI
-    -DSWIFT_PRIMARY_VARIANT_ARCH:STRING=wasm32
-    -DSWIFT_SDKS='WASI'
-    -DSWIFT_STDLIB_SINGLE_THREADED_RUNTIME=TRUE
-    -DSWIFT_OSX_x86_64_ICU_STATICLIB=TRUE
-    -DSWIFT_BUILD_SOURCEKIT=FALSE
-    -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
-    -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
-    -DCMAKE_AR='%(LLVM_BIN_DIR)s/llvm-ar'
-    -DCMAKE_RANLIB='%(LLVM_BIN_DIR)s/llvm-ranlib'
-    -DCLANG_COMPILER_RT_CMAKE_ARGS='-DCMAKE_TOOLCHAIN_FILE=%(SOURCE_PATH)s/swift/utils/webassembly/compiler-rt-cache.cmake'
-    -DSWIFTWASM_DISABLE_REFLECTION_TEST=TRUE
-
-[preset: webassembly-linux-target-install]
-mixin-preset=
-    webassembly-target-install
-    webassembly-linux-target
-
-[preset: webassembly-macos-target-install]
-mixin-preset=
-    webassembly-target-install
-    webassembly-macos-target

utils/webassembly/build-toolchain.sh

@@ -10,7 +10,6 @@ case $(uname -s) in
   Darwin)
     OS_SUFFIX=macos_x86_64
     HOST_PRESET=webassembly-host-install
-    TARGET_PRESET=webassembly-macos-target-install
     HOST_SUFFIX=macosx-x86_64
   ;;
   Linux)
@@ -23,7 +22,6 @@ case $(uname -s) in
       exit 1
     fi
     HOST_PRESET=webassembly-linux-host-install
-    TARGET_PRESET=webassembly-linux-target-install
     HOST_SUFFIX=linux-x86_64
   ;;
   *)
@@ -35,28 +33,18 @@ esac
 YEAR=$(date +"%Y")
 MONTH=$(date +"%m")
 DAY=$(date +"%d")
-TOOLCHAIN_VERSION="${YEAR}${MONTH}${DAY}"
 TOOLCHAIN_NAME="swift-wasm-DEVELOPMENT-SNAPSHOT-${YEAR}-${MONTH}-${DAY}-a"
 
 PACKAGE_ARTIFACT="$SOURCE_PATH/swift-wasm-DEVELOPMENT-SNAPSHOT-${OS_SUFFIX}.tar.gz"
 
-BUNDLE_IDENTIFIER="swiftwasm.${YEAR}${MONTH}${DAY}"
-DISPLAY_NAME_SHORT="Swift for WebAssembly Development Snapshot"
-DISPLAY_NAME="${DISPLAY_NAME_SHORT} ${YEAR}-${MONTH}-${DAY}"
-
-DIST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/dist-toolchain-sdk
 HOST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/host-toolchain-sdk
-TARGET_TOOLCHAIN_DESTDIR=$SOURCE_PATH/target-toolchain-sdk
-
+DIST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/dist-toolchain-sdk
 DIST_TOOLCHAIN_SDK=$DIST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
-HOST_TOOLCHAIN_SDK=$HOST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
-TARGET_TOOLCHAIN_SDK=$TARGET_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
 
 
 HOST_BUILD_ROOT=$SOURCE_PATH/host-build
 TARGET_BUILD_ROOT=$SOURCE_PATH/target-build
 HOST_BUILD_DIR=$HOST_BUILD_ROOT/Ninja-Release
-TARGET_BUILD_DIR=$TARGET_BUILD_ROOT/Ninja-Release
 
 build_host_toolchain() {
   # Build the host toolchain and SDK first.
@@ -71,49 +59,65 @@ build_host_toolchain() {
 }
 
 build_target_toolchain() {
-  mkdir -p "$HOST_BUILD_DIR/"
-  # Copy the host build dir to reuse it.
-  if [[ ! -e "$HOST_BUILD_DIR/llvm-$HOST_SUFFIX" ]]; then
-    cp -r "$HOST_BUILD_DIR/llvm-$HOST_SUFFIX" "$TARGET_BUILD_DIR/llvm-$HOST_SUFFIX"
-    # Clean up compiler-rt dir to cross compile it for host and wasm32
-    (cd "$TARGET_BUILD_DIR/llvm-$HOST_SUFFIX" && ninja compiler-rt-clear)
-  fi
-
-  # build the cross-compilled toolchain
-  env SWIFT_BUILD_ROOT="$TARGET_BUILD_ROOT" \
-    "$SOURCE_PATH/swift/utils/build-script" \
-    --preset-file="$UTILS_PATH/build-presets.ini" \
-    --preset=$TARGET_PRESET \
-    --build-dir="$TARGET_BUILD_DIR" \
-    INSTALL_DESTDIR="$TARGET_TOOLCHAIN_DESTDIR" \
-    SOURCE_PATH="$SOURCE_PATH" \
-    BUNDLE_IDENTIFIER="${BUNDLE_IDENTIFIER}" \
-    DISPLAY_NAME="${DISPLAY_NAME}" \
-    DISPLAY_NAME_SHORT="${DISPLAY_NAME_SHORT}" \
-    TOOLCHAIN_NAME="${TOOLCHAIN_NAME}" \
-    TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" \
-    LLVM_BIN_DIR="${HOST_TOOLCHAIN_SDK}/usr/bin" \
-    C_CXX_LAUNCHER="$(which sccache)"
+  rm -rf "$DIST_TOOLCHAIN_DESTDIR"
+  cp -r "$HOST_TOOLCHAIN_DESTDIR" "$DIST_TOOLCHAIN_DESTDIR"
 
-  "$UTILS_PATH/build-foundation.sh" "$TARGET_TOOLCHAIN_SDK"
-  "$UTILS_PATH/build-xctest.sh" "$TARGET_TOOLCHAIN_SDK"
+  COMPILER_RT_BUILD_DIR="$TARGET_BUILD_ROOT/compiler-rt-wasi-wasm32"
+  cmake -B "$COMPILER_RT_BUILD_DIR" \
+    -D CMAKE_TOOLCHAIN_FILE="$SOURCE_PATH/swift/utils/webassembly/compiler-rt-cache.cmake" \
+    -D CMAKE_BUILD_TYPE=Release \
+    -D CMAKE_C_COMPILER="$HOST_BUILD_DIR/llvm-$HOST_SUFFIX/bin/clang" \
+    -D CMAKE_CXX_COMPILER="$HOST_BUILD_DIR/llvm-$HOST_SUFFIX/bin/clang++" \
+    -D CMAKE_RANLIB="$HOST_BUILD_DIR/llvm-$HOST_SUFFIX/bin/llvm-ranlib" \
+    -D CMAKE_AR="$HOST_BUILD_DIR/llvm-$HOST_SUFFIX/bin/llvm-ar" \
+    -D CMAKE_C_COMPILER_LAUNCHER="$(which sccache)" \
+    -D CMAKE_CXX_COMPILER_LAUNCHER="$(which sccache)" \
+    -D CMAKE_INSTALL_PREFIX="$DIST_TOOLCHAIN_SDK/usr/lib/clang/10.0.0/" \
+    -D COMPILER_RT_SWIFT_WASI_SDK_PATH="$WASI_SDK_PATH" \
+    -G Ninja \
+    -S "$SOURCE_PATH/llvm-project/compiler-rt"
+
+  ninja install -C "$COMPILER_RT_BUILD_DIR"
+
+  SWIFT_STDLIB_BUILD_DIR="$TARGET_BUILD_ROOT/swift-stdlib-wasi-wasm32"
+  cmake -B "$TARGET_BUILD_ROOT/swift-stdlib-wasi-wasm32" \
+    -C "$SOURCE_PATH/swift/cmake/caches/Runtime-WASI-wasm32.cmake" \
+    -D CMAKE_BUILD_TYPE=Release \
+    -D CMAKE_C_COMPILER="$HOST_BUILD_DIR/llvm-$HOST_SUFFIX/bin/clang" \
+    -D CMAKE_CXX_COMPILER="$HOST_BUILD_DIR/llvm-$HOST_SUFFIX/bin/clang++" \
+    -D CMAKE_RANLIB="$HOST_BUILD_DIR/llvm-$HOST_SUFFIX/bin/llvm-ranlib" \
+    -D CMAKE_AR="$HOST_BUILD_DIR/llvm-$HOST_SUFFIX/bin/llvm-ar" \
+    -D CMAKE_C_COMPILER_LAUNCHER="$(which sccache)" \
+    -D CMAKE_CXX_COMPILER_LAUNCHER="$(which sccache)" \
+    -D CMAKE_INSTALL_PREFIX="$DIST_TOOLCHAIN_SDK/usr" \
+    -D LLVM_DIR="$HOST_BUILD_DIR/llvm-$HOST_SUFFIX/lib/cmake/llvm/" \
+    -D SWIFT_NATIVE_SWIFT_TOOLS_PATH="$HOST_BUILD_DIR/swift-$HOST_SUFFIX/bin" \
+    -D SWIFT_WASI_SDK_PATH="$WASI_SDK_PATH" \
+    -G Ninja \
+    -S "$SOURCE_PATH/swift"
+
+  ninja install -C "$SWIFT_STDLIB_BUILD_DIR"
+
+  # Copy tool binaries in target build dir to test stdlib
+  rsync -a "$HOST_BUILD_DIR/llvm-$HOST_SUFFIX/bin/" "$SWIFT_STDLIB_BUILD_DIR/bin/"
+  rsync -a "$HOST_BUILD_DIR/swift-$HOST_SUFFIX/bin/" "$SWIFT_STDLIB_BUILD_DIR/bin/"
+
+  # Link compiler-rt libs to stdlib build dir
+  mkdir -p "$SWIFT_STDLIB_BUILD_DIR/lib/clang/10.0.0/"
+  ln -fs "$COMPILER_RT_BUILD_DIR/lib" "$SWIFT_STDLIB_BUILD_DIR/lib/clang/10.0.0/lib"
+
+  # Remove host CoreFoundation module directory to avoid module conflict
+  # while building Foundation
+  rm -rf "$DIST_TOOLCHAIN_SDK/usr/lib/swift_static/CoreFoundation"
+  "$UTILS_PATH/build-foundation.sh" "$DIST_TOOLCHAIN_SDK"
+  "$UTILS_PATH/build-xctest.sh" "$DIST_TOOLCHAIN_SDK"
 
 }
 
-merge_toolchains() {
-  rm -rf "$DIST_TOOLCHAIN_DESTDIR"
-  # Copy the base host toolchain
-  cp -r "$HOST_TOOLCHAIN_DESTDIR" "$DIST_TOOLCHAIN_DESTDIR"
-
+embed_wasi_sysroot() {
   # Merge wasi-sdk and the toolchain
   cp -r "$WASI_SDK_PATH/share/wasi-sysroot" "$DIST_TOOLCHAIN_SDK/usr/share"
 
-  # Copy the target environment stdlib into the toolchain
-  # Avoid copying usr/lib/swift/clang because our toolchain's one is a directory
-  # but nightly's one is symbolic link. A simple copy fails to merge them.
-  rsync -v -a "$TARGET_TOOLCHAIN_SDK/usr/lib/" "$DIST_TOOLCHAIN_SDK/usr/lib/" --exclude 'swift/clang'
-  rsync -v -a "$TARGET_TOOLCHAIN_SDK/usr/bin/" "$DIST_TOOLCHAIN_SDK/usr/bin/"
-
   # Replace absolute sysroot path with relative path
   sed -i.bak -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" "$DIST_TOOLCHAIN_SDK/usr/lib/swift/wasi/wasm32/wasi.modulemap"
   rm "$DIST_TOOLCHAIN_SDK/usr/lib/swift/wasi/wasm32/wasi.modulemap.bak"
@@ -164,7 +168,7 @@ create_darwin_info_plist() {
 build_host_toolchain
 build_target_toolchain
 
-merge_toolchains
+embed_wasi_sysroot
 
 if [[ "$(uname)" == "Darwin" ]]; then
   create_darwin_info_plist

utils/webassembly/build-xctest.sh

@@ -3,7 +3,7 @@ set -ex
 DESTINATION_TOOLCHAIN=$1
 SOURCE_PATH="$(cd "$(dirname $0)/../../.." && pwd)"
 
-BUILD_DIR="$SOURCE_PATH/target-build/Ninja-Release/xctest-wasi-wasm32"
+BUILD_DIR="$SOURCE_PATH/target-build/xctest-wasi-wasm32"
 
 mkdir -p $BUILD_DIR
 cd $BUILD_DIR

utils/webassembly/ci.sh

@@ -1,4 +1,4 @@
-#/bin/bash
+#!/bin/bash
 
 set -ex
 
@@ -23,7 +23,7 @@ esac
 
 BUILD_SCRIPT=$UTILS_PATH/build-toolchain.sh
 RUN_TEST_BIN=$SWIFT_PATH/utils/run-test
-TARGET_BUILD_DIR=$SOURCE_PATH/target-build/Ninja-Release
+TARGET_STDLIB_BUILD_DIR=$SOURCE_PATH/target-build/swift-stdlib-wasi-wasm32
 
 $DEPENDENCIES_SCRIPT
 
@@ -36,28 +36,20 @@ $BUILD_SCRIPT
 
 echo "Build script completed, will attempt to run test suites..."
 
-if [[ "$(uname)" == "Darwin" ]]; then
-  # workaround: host target test directory is necessary to use run-test
-  mkdir -p $TARGET_BUILD_DIR/swift-macosx-x86_64/test-macosx-x86_64
-  HOST_PLATFORM=macosx
-else
-  HOST_PLATFORM=linux
-fi
+# workaround: host target test directory is necessary to use run-test
+mkdir -p "$TARGET_STDLIB_BUILD_DIR/test-$HOST_SUFFIX"
+
+# Run tests
+$RUN_TEST_BIN --build-dir "$TARGET_STDLIB_BUILD_DIR" --target wasi-wasm32 \
+  "$TARGET_STDLIB_BUILD_DIR/test-wasi-wasm32/stdlib"
+$RUN_TEST_BIN --build-dir "$TARGET_STDLIB_BUILD_DIR" --target wasi-wasm32 \
+  "$TARGET_STDLIB_BUILD_DIR/test-wasi-wasm32/LTO"
 
 if [[ "$(uname)" == "Linux" ]]; then
-  $RUN_TEST_BIN --build-dir $TARGET_BUILD_DIR --target wasi-wasm32 \
-    $TARGET_BUILD_DIR/swift-${HOST_PLATFORM}-x86_64/test-wasi-wasm32/stdlib
-  $RUN_TEST_BIN --build-dir $TARGET_BUILD_DIR --target wasi-wasm32 \
-    $TARGET_BUILD_DIR/swift-${HOST_PLATFORM}-x86_64/test-wasi-wasm32/LTO
-  echo "Skip running test suites for Linux"
+  echo "Skip running all test suites for Linux"
 else
-  $RUN_TEST_BIN --build-dir $TARGET_BUILD_DIR --target wasi-wasm32 \
- 	$TARGET_BUILD_DIR/swift-${HOST_PLATFORM}-x86_64/test-wasi-wasm32/stdlib
-  $RUN_TEST_BIN --build-dir $TARGET_BUILD_DIR --target wasi-wasm32 \
-	$TARGET_BUILD_DIR/swift-${HOST_PLATFORM}-x86_64/test-wasi-wasm32/LTO
-
-  # Run test but ignore failure temporarily
-  ninja check-swift-wasi-wasm32 -C $TARGET_BUILD_DIR/swift-$HOST_SUFFIX || true
+  # Run all tests but ignore failure temporarily
+  ninja check-swift-wasi-wasm32 -C "$TARGET_STDLIB_BUILD_DIR" || true
 fi
 
 echo "The test suite has finished"