[WASM] Refactoring build script for debuggability

kateinoigakukun · kateinoigakukun · commit a0db7cbbdc21 · 2020-09-20T11:34:46.000+09:00
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -45,8 +45,6 @@ jobs:
       - name: Build Linux installable archive
         run: |
           ./utils/webassembly/ci.sh
-          echo "Cleanup build directory to free disk space"
-          rm -rf ../build
       - name: Upload Linux installable archive
         uses: actions/upload-artifact@v1
         with:
@@ -90,7 +88,7 @@ jobs:
           name: macos-installable
           path: ../swift-wasm-DEVELOPMENT-SNAPSHOT-osx.tar.gz
       - name: Pack test results
-        run: tar cJf swift-test-results.tar.gz ../build/*/swift-macosx-x86_64/swift-test-results
+        run: tar cJf swift-test-results.tar.gz ../target-build/*/swift-macosx-x86_64/swift-test-results
       - name: Upload test results
         uses: actions/upload-artifact@v1
         with:
diff --git a/utils/webassembly/build-foundation.sh b/utils/webassembly/build-foundation.sh
@@ -3,11 +3,7 @@ set -ex
 DESTINATION_TOOLCHAIN=$1
 SOURCE_PATH="$(cd "$(dirname $0)/../../.." && pwd)"
 
-# Remove host CoreFoundation (which can be different from the target) headers
-# to avoid shadowing the wasm32 target CoreFoundation
-rm -rf $DESTINATION_TOOLCHAIN/usr/lib/swift/CoreFoundation
-
-FOUNDATION_BUILD="$SOURCE_PATH/build/Ninja-ReleaseAssert/foundation-wasi-wasm32"
+FOUNDATION_BUILD="$SOURCE_PATH/target-build/Ninja-ReleaseAssert/foundation-wasi-wasm32"
 
 mkdir -p $FOUNDATION_BUILD
 cd $FOUNDATION_BUILD
@@ -19,22 +15,12 @@ cmake -G Ninja \
   -DWASI_SDK_PATH="$SOURCE_PATH/wasi-sdk" \
   -DICU_ROOT="$SOURCE_PATH/icu_out" \
   -DBUILD_SHARED_LIBS=OFF \
+  -DCMAKE_Swift_COMPILER_FORCED=ON \
   "${SOURCE_PATH}/swift-corelibs-foundation"
   
 ninja -v
 ninja -v install
 
-# On macOS the target CoreFoundation shadows the CoreFoundation suppplied by Xcode.
-# On Linux though there's no system CoreFoundation, its headers have to be shipped
-# in the installable archive and serve for both host and target.
-if [[ "$(uname)" == "Darwin" ]]; then
-  mv $DESTINATION_TOOLCHAIN/usr/lib/swift_static/CoreFoundation \
-    $DESTINATION_TOOLCHAIN/usr/lib/swift/wasi/wasm32/CoreFoundation
-else
-  mv $DESTINATION_TOOLCHAIN/usr/lib/swift_static/CoreFoundation \
-    $DESTINATION_TOOLCHAIN/usr/lib/swift/CoreFoundation
-fi
-
 # .swiftdoc and .swiftmodule files should live in `swift`, not in `swift_static`
 mv $DESTINATION_TOOLCHAIN/usr/lib/swift_static/wasi/wasm32/Foundation.swift* \
-  $DESTINATION_TOOLCHAIN/usr/lib/swift/wasi/wasm32
+  $DESTINATION_TOOLCHAIN/usr/lib/swift/wasi/wasm32
diff --git a/utils/webassembly/build-toolchain.sh b/utils/webassembly/build-toolchain.sh
@@ -11,11 +11,13 @@ case $(uname -s) in
     OS_SUFFIX=osx
     HOST_PRESET=webassembly-host
     TARGET_PRESET=webassembly-macos-target
+    HOST_SUFFIX=macosx-x86_64
   ;;
   Linux)
     OS_SUFFIX=linux
     HOST_PRESET=webassembly-linux-host
     TARGET_PRESET=webassembly-linux-target
+    HOST_SUFFIX=linux-x86_64
   ;;
   *)
     echo "Unrecognised platform $(uname -s)"
@@ -29,69 +31,92 @@ DAY=$(date +"%d")
 TOOLCHAIN_VERSION="${YEAR}${MONTH}${DAY}"
 TOOLCHAIN_NAME="swift-wasm-DEVELOPMENT-SNAPSHOT-${YEAR}-${MONTH}-${DAY}-a"
 ARCHIVE="${TOOLCHAIN_NAME}-${OS_SUFFIX}.tar.gz"
-INSTALLABLE_PACKAGE=$SOURCE_PATH/$ARCHIVE
 
 PACKAGE_ARTIFACT="$SOURCE_PATH/swift-wasm-DEVELOPMENT-SNAPSHOT-${OS_SUFFIX}.tar.gz"
 
 BUNDLE_IDENTIFIER="swiftwasm.${YEAR}${MONTH}${DAY}"
 DISPLAY_NAME_SHORT="Swift for WebAssembly Development Snapshot"
 DISPLAY_NAME="${DISPLAY_NAME_SHORT} ${YEAR}-${MONTH}-${DAY}"
 
+DIST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/dist-toolchain-sdk
 HOST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/host-toolchain-sdk
+TARGET_TOOLCHAIN_DESTDIR=$SOURCE_PATH/target-toolchain-sdk
+
+DIST_TOOLCHAIN_SDK=$DIST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
 HOST_TOOLCHAIN_SDK=$HOST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
+TARGET_TOOLCHAIN_SDK=$TARGET_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
+
 
-BUILD_DIR=$SOURCE_PATH/build/Ninja-ReleaseAssert
+HOST_BUILD_ROOT=$SOURCE_PATH/host-build
+TARGET_BUILD_ROOT=$SOURCE_PATH/target-build
+HOST_BUILD_DIR=$HOST_BUILD_ROOT/Ninja-ReleaseAssert
+TARGET_BUILD_DIR=$TARGET_BUILD_ROOT/Ninja-ReleaseAssert
 
 # Avoid clang headers symlink issues
 mkdir -p $HOST_TOOLCHAIN_SDK/usr/lib/clang/10.0.0
 
-# Build the host toolchain and SDK first.
-$SOURCE_PATH/swift/utils/build-script \
-  --preset=$HOST_PRESET \
-  INSTALL_DESTDIR="$HOST_TOOLCHAIN_DESTDIR" \
-  TOOLCHAIN_NAME="$TOOLCHAIN_NAME" \
-  C_CXX_LAUNCHER="$(which sccache)"
-
-# Clean up the host toolchain build directory so that the next
-# `build-script` invocation doesn't pick up wrong CMake config files.
-# For some reason passing `--reconfigure` to `build-script` won't do this.
-rm -rf $BUILD_DIR/swift-*
-# Clean up compiler-rt dir to cross compile it for host and wasm32
-(cd $BUILD_DIR/llvm-* && ninja compiler-rt-clear)
-
-# build the cross-compilled toolchain
-$SOURCE_PATH/swift/utils/build-script \
-  --preset=$TARGET_PRESET --reconfigure \
-  INSTALL_DESTDIR="$SOURCE_PATH/install" \
-  SOURCE_PATH="$SOURCE_PATH" \
-  BUNDLE_IDENTIFIER="${BUNDLE_IDENTIFIER}" \
-  DISPLAY_NAME="${DISPLAY_NAME}" \
-  DISPLAY_NAME_SHORT="${DISPLAY_NAME_SHORT}" \
-  TOOLCHAIN_NAME="${TOOLCHAIN_NAME}" \
-  TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" \
-  TOOLS_BIN_DIR="${HOST_TOOLCHAIN_SDK}/usr/bin" \
-  C_CXX_LAUNCHER="$(which sccache)"
-
-# Merge wasi-sdk and the toolchain
-cp -r $WASI_SDK_PATH/share/wasi-sysroot $HOST_TOOLCHAIN_SDK/usr/share
-
-# Replace absolute sysroot path with relative path
-sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/glibc.modulemap
-
-# Copy the target environment stdlib into the toolchain
-
-# Avoid copying usr/lib/swift/clang because our toolchain's one is a directory
-# but nightly's one is symbolic link. A simple copy fails to merge them.
-rsync -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/ $HOST_TOOLCHAIN_SDK/usr/lib/ --exclude 'swift/clang'
-rsync -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/bin/ $HOST_TOOLCHAIN_SDK/usr/bin/
-
-$UTILS_PATH/build-foundation.sh $HOST_TOOLCHAIN_SDK
-$UTILS_PATH/build-xctest.sh $HOST_TOOLCHAIN_SDK
-
-# Cleanup build directory on Linux CI
-if [[ -n "${CI}" && "$(uname)" == "Linux" ]]; then
-  rm -rf $BUILD_DIR/foundation-* $BUILD_DIR/swiftpm-*
-fi
-
-cd $HOST_TOOLCHAIN_DESTDIR
+build_host_toolchain() {
+  # Build the host toolchain and SDK first.
+  env SWIFT_BUILD_ROOT="$HOST_BUILD_ROOT" \
+    $SOURCE_PATH/swift/utils/build-script \
+    --preset=$HOST_PRESET \
+    --build-dir="$HOST_BUILD_DIR" \
+    INSTALL_DESTDIR="$HOST_TOOLCHAIN_DESTDIR" \
+    TOOLCHAIN_NAME="$TOOLCHAIN_NAME" \
+    C_CXX_LAUNCHER="$(which sccache)"
+}
+
+build_target_toolchain() {
+  mkdir -p $HOST_BUILD_DIR/
+  # Copy the host build dir to reuse it.
+  if [[ ! -e "$HOST_BUILD_DIR/llvm-$HOST_SUFFIX" ]]; then
+    cp -r "$HOST_BUILD_DIR/llvm-$HOST_SUFFIX" "$TARGET_BUILD_DIR/llvm-$HOST_SUFFIX"
+    # Clean up compiler-rt dir to cross compile it for host and wasm32
+    (cd "$TARGET_BUILD_DIR/llvm-$HOST_SUFFIX" && ninja compiler-rt-clear)
+  fi
+
+  # build the cross-compilled toolchain
+  env SWIFT_BUILD_ROOT="$TARGET_BUILD_ROOT" \
+    $SOURCE_PATH/swift/utils/build-script \
+    --preset=$TARGET_PRESET --reconfigure \
+    --build-dir="$TARGET_BUILD_DIR" \
+    INSTALL_DESTDIR="$TARGET_TOOLCHAIN_DESTDIR" \
+    SOURCE_PATH="$SOURCE_PATH" \
+    BUNDLE_IDENTIFIER="${BUNDLE_IDENTIFIER}" \
+    DISPLAY_NAME="${DISPLAY_NAME}" \
+    DISPLAY_NAME_SHORT="${DISPLAY_NAME_SHORT}" \
+    TOOLCHAIN_NAME="${TOOLCHAIN_NAME}" \
+    TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" \
+    TOOLS_BIN_DIR="${HOST_TOOLCHAIN_SDK}/usr/bin" \
+    C_CXX_LAUNCHER="$(which sccache)"
+
+  $UTILS_PATH/build-foundation.sh $TARGET_TOOLCHAIN_SDK
+  $UTILS_PATH/build-xctest.sh $TARGET_TOOLCHAIN_SDK
+
+}
+
+merge_toolchains() {
+  rm -rf "$DIST_TOOLCHAIN_DESTDIR"
+  # Copy the base host toolchain
+  cp -r "$HOST_TOOLCHAIN_DESTDIR" "$DIST_TOOLCHAIN_DESTDIR"
+
+  # Merge wasi-sdk and the toolchain
+  cp -r $WASI_SDK_PATH/share/wasi-sysroot $DIST_TOOLCHAIN_SDK/usr/share
+
+  # Copy the target environment stdlib into the toolchain
+  # Avoid copying usr/lib/swift/clang because our toolchain's one is a directory
+  # but nightly's one is symbolic link. A simple copy fails to merge them.
+  rsync -v -a $TARGET_TOOLCHAIN_SDK/usr/lib/ $DIST_TOOLCHAIN_SDK/usr/lib/ --exclude 'swift/clang'
+  rsync -v -a $TARGET_TOOLCHAIN_SDK/usr/bin/ $DIST_TOOLCHAIN_SDK/usr/bin/
+
+  # Replace absolute sysroot path with relative path
+  sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $DIST_TOOLCHAIN_SDK/usr/lib/swift/wasi/wasm32/glibc.modulemap
+}
+
+build_host_toolchain
+build_target_toolchain
+
+merge_toolchains
+
+cd $DIST_TOOLCHAIN_DESTDIR
 tar cfz $PACKAGE_ARTIFACT $TOOLCHAIN_NAME
diff --git a/utils/webassembly/build-xctest.sh b/utils/webassembly/build-xctest.sh
@@ -3,7 +3,7 @@ set -ex
 DESTINATION_TOOLCHAIN=$1
 SOURCE_PATH="$(cd "$(dirname $0)/../../.." && pwd)"
 
-BUILD_DIR="$SOURCE_PATH/build/Ninja-ReleaseAssert/xctest-wasi-wasm32"
+BUILD_DIR="$SOURCE_PATH/target-build/Ninja-ReleaseAssert/xctest-wasi-wasm32"
 
 mkdir -p $BUILD_DIR
 cd $BUILD_DIR
@@ -14,6 +14,7 @@ cmake -G Ninja \
   -DCMAKE_TOOLCHAIN_FILE="$SOURCE_PATH/swift/utils/webassembly/toolchain-wasi.cmake" \
   -DWASI_SDK_PATH="$SOURCE_PATH/wasi-sdk" \
   -DBUILD_SHARED_LIBS=OFF \
+  -DCMAKE_Swift_COMPILER_FORCED=ON \
   -DSWIFT_FOUNDATION_PATH=$DESTINATION_TOOLCHAIN/usr/lib/swift/wasi/wasm32 \
   "${SOURCE_PATH}/swift-corelibs-xctest"
   
diff --git a/utils/webassembly/ci.sh b/utils/webassembly/ci.sh
@@ -13,7 +13,7 @@ fi
 
 BUILD_SCRIPT=$UTILS_PATH/build-toolchain.sh
 RUN_TEST_BIN=$SWIFT_PATH/utils/run-test
-BUILD_DIR=$SOURCE_PATH/build/Ninja-ReleaseAssert
+TARGET_BUILD_DIR=$SOURCE_PATH/target-build/Ninja-ReleaseAssert
 
 $DEPENDENCIES_SCRIPT
 
@@ -26,15 +26,15 @@ $BUILD_SCRIPT
 
 if [[ "$(uname)" == "Darwin" ]]; then
   # workaround: host target test directory is necessary to use run-test
-  mkdir -p $BUILD_DIR/swift-macosx-x86_64/test-macosx-x86_64
+  mkdir -p $TARGET_BUILD_DIR/swift-macosx-x86_64/test-macosx-x86_64
 fi
 
 if [[ "$(uname)" == "Linux" ]]; then
-  $RUN_TEST_BIN --build-dir $BUILD_DIR --target wasi-wasm32 test/stdlib/ || true
+  $RUN_TEST_BIN --build-dir $TARGET_BUILD_DIR --target wasi-wasm32 test/stdlib/ || true
   echo "Skip running test suites for Linux"
 else
 
-  $RUN_TEST_BIN --build-dir $BUILD_DIR --target wasi-wasm32 test/stdlib/
+  $RUN_TEST_BIN --build-dir $TARGET_BUILD_DIR --target wasi-wasm32 test/stdlib/
 
   # Run test but ignore failure temporarily
   $BUILD_SCRIPT -t || true
