Auto merge of #42784 - tlively:wasm-bot, r=alexcrichton

Make wasm32 buildbot test LLVM backend

This adds the experimental targets option to configure so it can be used
by the builders and changes the wasm32 Dockerfile accordingly. Instead
of using LLVM from the emsdk, the builder's emscripten tools now uses
the Rust in-tree LLVM, since this is the one built with wasm support.

configure

@@ -490,6 +490,7 @@ valopt musl-root-armhf "" "arm-unknown-linux-musleabihf install directory"
 valopt musl-root-armv7 "" "armv7-unknown-linux-musleabihf install directory"
 valopt extra-filename "" "Additional data that is hashed and passed to the -C extra-filename flag"
 valopt qemu-armhf-rootfs "" "rootfs in qemu testing, you probably don't want to use this"
+valopt experimental-targets "" "experimental LLVM targets to build"
 
 if [ -e ${CFG_SRC_DIR}.git ]
 then

src/bootstrap/config.rs

@@ -497,6 +497,9 @@ impl Config {
                 "CFG_TARGET" if value.len() > 0 => {
                     self.target.extend(value.split(" ").map(|s| s.to_string()));
                 }
+                "CFG_EXPERIMENTAL_TARGETS" if value.len() > 0 => {
+                    self.llvm_experimental_targets = Some(value.to_string());
+                }
                 "CFG_MUSL_ROOT" if value.len() > 0 => {
                     self.musl_root = Some(parse_configure_path(value));
                 }

src/ci/docker/disabled/wasm32/Dockerfile

@@ -11,16 +11,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   sudo \
   gdb \
-  xz-utils
+  xz-utils \
+  jq \
+  bzip2
 
 # dumb-init
 COPY scripts/dumb-init.sh /scripts/
 RUN sh /scripts/dumb-init.sh
 
 # emscripten
-COPY scripts/emscripten.sh /scripts/
-RUN bash /scripts/emscripten.sh
-COPY wasm32/node.sh /usr/local/bin/node
+COPY scripts/emscripten-wasm.sh /scripts/
+RUN bash /scripts/emscripten-wasm.sh
+COPY disabled/wasm32/node.sh /usr/local/bin/node
+
+# cache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
 
 # env
 ENV PATH=$PATH:/emsdk-portable
@@ -30,15 +36,11 @@ ENV EMSCRIPTEN=/emsdk-portable/emscripten/1.37.13/
 ENV BINARYEN_ROOT=/emsdk-portable/clang/e1.37.13_64bit/binaryen/
 ENV EM_CONFIG=/emsdk-portable/.emscripten
 
-ENV TARGETS=wasm32-unknown-emscripten
+ENV TARGETS=wasm32-unknown-emscripten,wasm32-experimental-emscripten
 
-ENV RUST_CONFIGURE_ARGS --target=$TARGETS
+ENV RUST_CONFIGURE_ARGS --target=$TARGETS --experimental-targets=WebAssembly
 
 ENV SCRIPT python2.7 ../x.py test --target $TARGETS
 
-# cache
-COPY scripts/sccache.sh /scripts/
-RUN sh /scripts/sccache.sh
-
 # init
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]

src/ci/docker/scripts/emscripten-wasm.sh

@@ -0,0 +1,64 @@
+# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+hide_output() {
+  set +x
+  on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/build.log
+exit 1
+"
+  trap "$on_err" ERR
+  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+  PING_LOOP_PID=$!
+  $@ &> /tmp/build.log
+  trap - ERR
+  kill $PING_LOOP_PID
+  rm -f /tmp/build.log
+  set -x
+}
+
+# Download emsdk
+cd /
+curl -L https://s3.amazonaws.com/mozilla-games/emscripten/releases/emsdk-portable.tar.gz | \
+    tar -xz
+
+# Download last known good emscripten from WebAssembly waterfall
+BUILD=$(curl -L https://storage.googleapis.com/wasm-llvm/builds/linux/lkgr.json | \
+    jq '.build | tonumber')
+curl -L https://storage.googleapis.com/wasm-llvm/builds/linux/$BUILD/wasm-binaries.tbz2 | \
+    hide_output tar xvkj
+
+# node 8 is required to run wasm
+cd /
+curl -L https://nodejs.org/dist/v8.0.0/node-v8.0.0-linux-x64.tar.xz | \
+    tar -xJ
+
+cd /emsdk-portable
+./emsdk update
+hide_output ./emsdk install sdk-1.37.13-64bit
+./emsdk activate sdk-1.37.13-64bit
+
+# Make emscripten use wasm-ready node and LLVM tools
+echo "NODE_JS='/node-v8.0.0-linux-x64/bin/node'" >> /root/.emscripten
+echo "LLVM_ROOT='/wasm-install/bin'" >> /root/.emscripten
+
+# Make emsdk usable by any user
+cp /root/.emscripten /emsdk-portable
+chmod a+rxw -R /emsdk-portable
+
+# Compile and cache libc
+source ./emsdk_env.sh
+echo "main(){}" > a.c
+HOME=/emsdk-portable/ emcc a.c
+HOME=/emsdk-portable/ emcc -s WASM=1 a.c
+rm -f a.*

src/ci/docker/scripts/emscripten.sh

@@ -50,4 +50,4 @@ chmod a+rxw -R /emsdk-portable
 # node 8 is required to run wasm
 cd /
 curl -L https://nodejs.org/dist/v8.0.0/node-v8.0.0-linux-x64.tar.xz | \
-    tar -xJ
+  tar -xJ

src/librustc_back/target/mod.rs

@@ -282,6 +282,9 @@ pub struct TargetOptions {
     /// user-defined libraries.
     pub post_link_args: LinkArgs,
 
+    /// Environment variables to be set before invoking the linker.
+    pub link_env: Vec<(String, String)>,
+
     /// Extra arguments to pass to the external assembler (when used)
     pub asm_args: Vec<String>,
 
@@ -451,6 +454,7 @@ impl Default for TargetOptions {
             pre_link_objects_dll: Vec::new(),
             post_link_objects: Vec::new(),
             late_link_args: LinkArgs::new(),
+            link_env: Vec::new(),
             archive_format: "gnu".to_string(),
             custom_unwind_resume: false,
             lib_allocation_crate: "alloc_system".to_string(),
@@ -620,6 +624,21 @@ impl Target {
                     base.options.$key_name = args;
                 }
             } );
+            ($key_name:ident, env) => ( {
+                let name = (stringify!($key_name)).replace("_", "-");
+                if let Some(a) = obj.find(&name[..]).and_then(|o| o.as_array()) {
+                    for o in a {
+                        if let Some(s) = o.as_string() {
+                            let p = s.split('=').collect::<Vec<_>>();
+                            if p.len() == 2 {
+                                let k = p[0].to_string();
+                                let v = p[1].to_string();
+                                base.options.$key_name.push((k, v));
+                            }
+                        }
+                    }
+                }
+            } );
         }
 
         key!(is_builtin, bool);
@@ -631,6 +650,7 @@ impl Target {
         key!(late_link_args, link_args);
         key!(post_link_objects, list);
         key!(post_link_args, link_args);
+        key!(link_env, env);
         key!(asm_args, list);
         key!(cpu);
         key!(features);
@@ -785,6 +805,17 @@ impl ToJson for Target {
                     d.insert(name.to_string(), obj.to_json());
                 }
             } );
+            (env - $attr:ident) => ( {
+                let name = (stringify!($attr)).replace("_", "-");
+                if default.$attr != self.options.$attr {
+                    let obj = self.options.$attr
+                        .iter()
+                        .map(|&(ref k, ref v)| k.clone() + "=" + &v)
+                        .collect::<Vec<_>>();
+                    d.insert(name.to_string(), obj.to_json());
+                }
+            } );
+
         }
 
         target_val!(llvm_target);
@@ -806,6 +837,7 @@ impl ToJson for Target {
         target_option_val!(link_args - late_link_args);
         target_option_val!(post_link_objects);
         target_option_val!(link_args - post_link_args);
+        target_option_val!(env - link_env);
         target_option_val!(asm_args);
         target_option_val!(cpu);
         target_option_val!(features);

src/librustc_back/target/wasm32_experimental_emscripten.rs

@@ -30,6 +30,7 @@ pub fn target() -> Result<Target, String> {
         // possibly interpret the wasm, and a .wasm file
         exe_suffix: ".js".to_string(),
         linker_is_gnu: true,
+        link_env: vec![("EMCC_WASM_BACKEND".to_string(), "1".to_string())],
         allow_asm: false,
         obj_is_bitcode: true,
         is_like_emscripten: true,

src/librustc_trans/back/link.rs

@@ -789,6 +789,9 @@ fn link_natively(sess: &Session,
     if let Some(args) = sess.target.target.options.post_link_args.get(&flavor) {
         cmd.args(args);
     }
+    for &(ref k, ref v) in &sess.target.target.options.link_env {
+        cmd.env(k, v);
+    }
 
     if sess.opts.debugging_opts.print_link_args {
         println!("{:?}", &cmd);

src/tools/compiletest/src/runtest.rs

@@ -1338,7 +1338,6 @@ actual:\n\
                              input)
     }
 
-
     fn compose_and_run(&self,
                        ProcArgs{ args, prog }: ProcArgs,
                        procenv: Vec<(String, String)> ,