Upgrade Emscripten targets to use upstream LLVM backend

 - Compatible with Emscripten 1.38.46-upstream or later upstream.
 - Refactors the Emscripten target spec to share code with other wasm
   targets.
 - Replaces the old incorrect wasm32 C call ABI with the correct one,
   preserving the old one as wasm32_bindgen_compat for wasm-bindgen
   compatibility.
 - Updates the varargs ABI used by Emscripten and deletes the old one.
 - Removes the obsolete wasm32-experimental-emscripten target.
 - Uses EMCC_CFLAGS on CI to avoid the timeout problems with rust-lang#63649.

Author: tlively

config.toml.example

@@ -377,6 +377,7 @@
 # but you can also optionally enable the "emscripten" backend for asm.js or
 # make this an empty array (but that probably won't get too far in the
 # bootstrap)
+# FIXME: remove the obsolete emscripten backend option.
 #codegen-backends = ["llvm"]
 
 # This is the name of the directory in which codegen backends will get installed

src/bootstrap/builder.rs

@@ -980,6 +980,7 @@ impl<'a> Builder<'a> {
                 Some("-Wl,-rpath,@loader_path/../lib")
             } else if !target.contains("windows") &&
                       !target.contains("wasm32") &&
+                      !target.contains("emscripten") &&
                       !target.contains("fuchsia") {
                 Some("-Wl,-rpath,$ORIGIN/../lib")
             } else {

src/bootstrap/test.rs

@@ -1047,10 +1047,11 @@ impl Step for Compiletest {
         // Also provide `rust_test_helpers` for the host.
         builder.ensure(native::TestHelpers { target: compiler.host });
 
-        // wasm32 can't build the test helpers
-        if !target.contains("wasm32") {
+        // As well as the target, except for plain wasm32, which can't build it
+        if !target.contains("wasm32") || target.contains("emscripten") {
             builder.ensure(native::TestHelpers { target });
         }
+
         builder.ensure(RemoteCopyLibs { compiler, target });
 
         let mut cmd = builder.tool_cmd(Tool::Compiletest);

src/ci/docker/asmjs/Dockerfile

@@ -11,7 +11,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   sudo \
   gdb \
-  xz-utils
+  xz-utils \
+  bzip2
 
 COPY scripts/emscripten.sh /scripts/
 RUN bash /scripts/emscripten.sh
@@ -20,28 +21,21 @@ COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
 ENV PATH=$PATH:/emsdk-portable
-ENV PATH=$PATH:/emsdk-portable/clang/e1.38.15_64bit/
-ENV PATH=$PATH:/emsdk-portable/emscripten/1.38.15/
-ENV PATH=$PATH:/emsdk-portable/node/8.9.1_64bit/bin/
-ENV EMSCRIPTEN=/emsdk-portable/emscripten/1.38.15/
-ENV BINARYEN_ROOT=/emsdk-portable/clang/e1.38.15_64bit/binaryen/
+ENV PATH=$PATH:/emsdk-portable/upstream/emscripten/
+ENV PATH=$PATH:/emsdk-portable/node/12.9.1_64bit/bin/
+ENV BINARYEN_ROOT=/emsdk-portable/upstream/
 ENV EM_CONFIG=/emsdk-portable/.emscripten
 
 ENV TARGETS=asmjs-unknown-emscripten
 
-ENV RUST_CONFIGURE_ARGS --enable-emscripten --disable-optimize-tests
+# Use -O1 optimizations in the link step to reduce time spent optimizing JS.
+# FIXME: Switch to testing wasm32-unknown-emscripten and -O1 to save more time
+# once the CI picks up https://github.com/rust-lang/cargo/pull/7476.
+ENV EMCC_CFLAGS=-O1
 
-ENV SCRIPT python2.7 ../x.py test --target $TARGETS \
-  src/test/ui \
-  src/test/run-fail \
-  src/libstd \
-  src/liballoc \
-  src/libcore
+ENV SCRIPT python2.7 ../x.py test --target $TARGETS
 
-# Debug assertions in rustc are largely covered by other builders, and LLVM
-# assertions cause this builder to slow down by quite a large amount and don't
-# buy us a huge amount over other builders (not sure if we've ever seen an
-# asmjs-specific backend assertion trip), so disable assertions for these
-# tests.
-ENV NO_LLVM_ASSERTIONS=1
+# This is almost identical to the wasm32-unknown-emscripten target, so
+# running with assertions again is not useful
 ENV NO_DEBUG_ASSERTIONS=1
+ENV NO_LLVM_ASSERTIONS=1

src/ci/docker/disabled/wasm32-exp/Dockerfile

@@ -11,22 +11,29 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   sudo \
   gdb \
-  xz-utils
+  xz-utils \
+  bzip2
 
-# emscripten
 COPY scripts/emscripten.sh /scripts/
 RUN bash /scripts/emscripten.sh
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
 ENV PATH=$PATH:/emsdk-portable
-ENV PATH=$PATH:/emsdk-portable/clang/e1.38.15_64bit/
-ENV PATH=$PATH:/emsdk-portable/emscripten/1.38.15/
-ENV PATH=$PATH:/emsdk-portable/node/8.9.1_64bit/bin/
-ENV EMSCRIPTEN=/emsdk-portable/emscripten/1.38.15/
-ENV BINARYEN_ROOT=/emsdk-portable/clang/e1.38.15_64bit/binaryen/
+ENV PATH=$PATH:/emsdk-portable/upstream/emscripten/
+ENV PATH=$PATH:/emsdk-portable/node/12.9.1_64bit/bin/
+ENV BINARYEN_ROOT=/emsdk-portable/upstream/
 ENV EM_CONFIG=/emsdk-portable/.emscripten
 
 ENV TARGETS=wasm32-unknown-emscripten
-ENV SCRIPT python2.7 ../x.py test --target $TARGETS
+
+# FIXME: Re-enable these tests once https://github.com/rust-lang/cargo/pull/7476
+# is picked up by CI
+ENV SCRIPT python2.7 ../x.py test --target $TARGETS \
+    --exclude src/libcore \
+    --exclude src/liballoc \
+    --exclude src/libproc_macro \
+    --exclude src/libstd \
+    --exclude src/libterm \
+    --exclude src/libtest

src/ci/docker/disabled/wasm32-exp/node.sh

@@ -17,20 +17,15 @@ exit 1
   set -x
 }
 
-cd /
-curl -fL https://mozilla-games.s3.amazonaws.com/emscripten/releases/emsdk-portable.tar.gz | \
-    tar -xz
-
+git clone https://github.com/emscripten-core/emsdk.git /emsdk-portable
 cd /emsdk-portable
-./emsdk update
-hide_output ./emsdk install sdk-1.38.15-64bit
-./emsdk activate sdk-1.38.15-64bit
+hide_output ./emsdk install 1.38.46-upstream
+./emsdk activate 1.38.46-upstream
 
 # Compile and cache libc
 source ./emsdk_env.sh
 echo "main(){}" > a.c
 HOME=/emsdk-portable/ emcc a.c
-HOME=/emsdk-portable/ emcc -s BINARYEN=1 a.c
 rm -f a.*
 
 # Make emsdk usable by any user

src/ci/docker/disabled/wasm32/Dockerfile

@@ -1,10 +1,5 @@
-use std::cmp;
 use std::collections::BinaryHeap;
 use std::collections::binary_heap::{Drain, PeekMut};
-use std::panic::{self, AssertUnwindSafe};
-use std::sync::atomic::{AtomicUsize, Ordering};
-
-use rand::{thread_rng, seq::SliceRandom};
 
 #[test]
 fn test_iterator() {
@@ -281,9 +276,15 @@ fn assert_covariance() {
 // even if the order may not be correct.
 //
 // Destructors must be called exactly once per element.
+// FIXME: re-enable emscripten once it can unwind again
 #[test]
-#[cfg(not(miri))] // Miri does not support catching panics
+#[cfg(not(any(miri, target_os = "emscripten")))] // Miri does not support catching panics
 fn panic_safe() {
+    use std::cmp;
+    use std::panic::{self, AssertUnwindSafe};
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use rand::{thread_rng, seq::SliceRandom};
+
     static DROP_COUNTER: AtomicUsize = AtomicUsize::new(0);
 
     #[derive(Eq, PartialEq, Ord, Clone, Debug)]