Auto merge of #36339 - brson:emscripten-new, r=alexcrichton

Working asmjs and wasm targets

This patch set results in a working standard library for the asmjs-unknown-emscripten and wasm32-unknown-emscripten targets. It is based on the work of @badboy and @rschulman.

It does a few things:

- Updates LLVM with the emscripten [fastcomp](rust-lang/llvm#50) patches, which include the pnacl IR legalizer and the asm.js backend. This patch is thought not to have any significant effect on existing targets.
- Teaches rustbuild to correctly link C code with emscripten
- Updates gcc-rs to work correctly with emscripten
- Teaches rustbuild to run crate tests for emscripten with node
- Modifies Thread::new to return an error on emscripten, to facilitate debugging a common failure mode
- Modifies libtest to run in single-threaded mode for emscripten
- Ignores a host of tests that don't work yet, mostly dealing with threads and I/O
- Updates libc with wasm32 definitions (presently the same as asmjs)
- Adds a wasm32-unknown-emscripten target that feeds the output of LLVM's asmjs backend through emcc to generate wasm

Notes and caveats:

- This is only known to work with `--enable-rustbuild`.
- The wasm32 target can't be tested correctly yet because of issues in compiletest and limitations in node emscripten-core/emscripten#4542, but hello.rs does seem to work when run on node via the binaryen interpreter
- This requires an up to date installation of the emscripten sdk from its incoming branch
- Unwinding is very broken
- When enabling the emscripten targets jemalloc is disabled for all targets, which results in test failures for the host

Next steps are to fix the jemalloc issue, start building the two emscripten targets on the auto builders, then start producing nightlies.

#36317 tracks work on this.

Fixes #36515
Fixes #36515
Fixes #36356

Author: bors

mk/cfg/wasm32-unknown-emscripten.mk

@@ -0,0 +1,24 @@
+# wasm32-unknown-emscripten configuration
+CC_wasm32-unknown-emscripten=emcc
+CXX_wasm32-unknown-emscripten=em++
+CPP_wasm32-unknown-emscripten=$(CPP)
+AR_wasm32-unknown-emscripten=emar
+CFG_LIB_NAME_wasm32-unknown-emscripten=lib$(1).so
+CFG_STATIC_LIB_NAME_wasm32-unknown-emscripten=lib$(1).a
+CFG_LIB_GLOB_wasm32-unknown-emscripten=lib$(1)-*.so
+CFG_LIB_DSYM_GLOB_wasm32-unknown-emscripten=lib$(1)-*.dylib.dSYM
+CFG_JEMALLOC_CFLAGS_wasm32-unknown-emscripten := -m32 $(CFLAGS)
+CFG_GCCISH_CFLAGS_wasm32-unknown-emscripten :=  -g -fPIC -m32 -s BINARYEN=1 $(CFLAGS)
+CFG_GCCISH_CXXFLAGS_wasm32-unknown-emscripten := -fno-rtti -s BINARYEN=1 $(CXXFLAGS)
+CFG_GCCISH_LINK_FLAGS_wasm32-unknown-emscripten := -shared -fPIC -ldl -pthread  -lrt -g -m32 -s BINARYEN=1
+CFG_GCCISH_DEF_FLAG_wasm32-unknown-emscripten := -Wl,--export-dynamic,--dynamic-list=
+CFG_LLC_FLAGS_wasm32-unknown-emscripten :=
+CFG_INSTALL_NAME_wasm32-unknown-emscripten =
+CFG_EXE_SUFFIX_wasm32-unknown-emscripten =
+CFG_WINDOWSY_wasm32-unknown-emscripten :=
+CFG_UNIXY_wasm32-unknown-emscripten := 1
+CFG_LDPATH_wasm32-unknown-emscripten :=
+CFG_RUN_wasm32-unknown-emscripten=$(2)
+CFG_RUN_TARG_wasm32-unknown-emscripten=$(call CFG_RUN_wasm32-unknown-emscripten,,$(2))
+CFG_GNU_TRIPLE_wasm32-unknown-emscripten := wasm32-unknown-emscripten
+CFG_DISABLE_JEMALLOC_wasm32-unknown-emscripten := 1

mk/main.mk

@@ -300,7 +300,7 @@ endif
 # LLVM macros
 ######################################################################
 
-LLVM_OPTIONAL_COMPONENTS=x86 arm aarch64 mips powerpc pnacl systemz
+LLVM_OPTIONAL_COMPONENTS=x86 arm aarch64 mips powerpc pnacl systemz jsbackend
 LLVM_REQUIRED_COMPONENTS=ipo bitreader bitwriter linker asmparser mcjit \
                 interpreter instrumentation
 

src/bootstrap/Cargo.lock

@@ -108,6 +108,10 @@ pub fn compiletest(build: &Build,
     cmd.arg("--host").arg(compiler.host);
     cmd.arg("--llvm-filecheck").arg(build.llvm_filecheck(&build.config.build));
 
+    if let Some(nodejs) = build.config.nodejs.as_ref() {
+        cmd.arg("--nodejs").arg(nodejs);
+    }
+
     let mut flags = vec!["-Crpath".to_string()];
     if build.config.rust_optimize_tests {
         flags.push("-O".to_string());
@@ -323,6 +327,9 @@ pub fn krate(build: &Build,
     if target.contains("android") {
         build.run(cargo.arg("--no-run"));
         krate_android(build, compiler, target, mode);
+    } else if target.contains("emscripten") {
+        build.run(cargo.arg("--no-run"));
+        krate_emscripten(build, compiler, target, mode);
     } else {
         cargo.args(&build.flags.args);
         build.run(&mut cargo);
@@ -371,6 +378,35 @@ fn krate_android(build: &Build,
     }
 }
 
+fn krate_emscripten(build: &Build,
+                    compiler: &Compiler,
+                    target: &str,
+                    mode: Mode) {
+     let mut tests = Vec::new();
+     let out_dir = build.cargo_out(compiler, mode, target);
+     find_tests(&out_dir, target, &mut tests);
+     find_tests(&out_dir.join("deps"), target, &mut tests);
+
+     for test in tests {
+         let test_file_name = test.to_string_lossy().into_owned();
+         println!("running {}", test_file_name);
+         let nodejs = build.config.nodejs.as_ref().expect("nodejs not configured");
+         let status = Command::new(nodejs)
+             .arg(&test_file_name)
+             .stderr(::std::process::Stdio::inherit())
+             .status();
+         match status {
+             Ok(status) => {
+                 if !status.success() {
+                     panic!("some tests failed");
+                 }
+             }
+             Err(e) => panic!(format!("failed to execute command: {}", e)),
+         };
+     }
+ }
+
+
 fn find_tests(dir: &Path,
               target: &str,
               dst: &mut Vec<PathBuf>) {
@@ -381,7 +417,8 @@ fn find_tests(dir: &Path,
         }
         let filename = e.file_name().into_string().unwrap();
         if (target.contains("windows") && filename.ends_with(".exe")) ||
-           (!target.contains("windows") && !filename.contains(".")) {
+           (!target.contains("windows") && !filename.contains(".")) ||
+           (target.contains("emscripten") && filename.contains(".js")){
             dst.push(e.path());
         }
     }

src/bootstrap/check.rs

@@ -975,7 +975,8 @@ impl Build {
         // than an entry here.
 
         let mut base = Vec::new();
-        if target != self.config.build && !target.contains("msvc") {
+        if target != self.config.build && !target.contains("msvc") &&
+            !target.contains("emscripten") {
             base.push(format!("-Clinker={}", self.cc(target).display()));
         }
         return base

src/bootstrap/lib.rs

@@ -65,7 +65,7 @@ pub fn llvm(build: &Build, target: &str) {
        .out_dir(&dst)
        .profile(if build.config.llvm_optimize {"Release"} else {"Debug"})
        .define("LLVM_ENABLE_ASSERTIONS", assertions)
-       .define("LLVM_TARGETS_TO_BUILD", "X86;ARM;AArch64;Mips;PowerPC;SystemZ")
+       .define("LLVM_TARGETS_TO_BUILD", "X86;ARM;AArch64;Mips;PowerPC;SystemZ;JSBackend")
        .define("LLVM_INCLUDE_EXAMPLES", "OFF")
        .define("LLVM_INCLUDE_TESTS", "OFF")
        .define("LLVM_INCLUDE_DOCS", "OFF")

src/bootstrap/native.rs

@@ -95,6 +95,13 @@ pub fn check(build: &mut Build) {
     // We're gonna build some custom C code here and there, host triples
     // also build some C++ shims for LLVM so we need a C++ compiler.
     for target in build.config.target.iter() {
+        // On emscripten we don't actually need the C compiler to just
+        // build the target artifacts, only for testing. For the sake
+        // of easier bot configuration, just skip detection.
+        if target.contains("emscripten") {
+            continue;
+        }
+
         need_cmd(build.cc(target).as_ref());
         if let Some(ar) = build.ar(target) {
             need_cmd(ar.as_ref());
@@ -104,22 +111,21 @@ pub fn check(build: &mut Build) {
         need_cmd(build.cxx(host).as_ref());
     }
 
+    // The msvc hosts don't use jemalloc, turn it off globally to
+    // avoid packaging the dummy liballoc_jemalloc on that platform.
+    for host in build.config.host.iter() {
+        if host.contains("msvc") {
+            build.config.use_jemalloc = false;
+        }
+    }
+
     // Externally configured LLVM requires FileCheck to exist
     let filecheck = build.llvm_filecheck(&build.config.build);
     if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests {
         panic!("filecheck executable {:?} does not exist", filecheck);
     }
 
     for target in build.config.target.iter() {
-        // Either can't build or don't want to run jemalloc on these targets
-        if target.contains("rumprun") ||
-           target.contains("bitrig") ||
-           target.contains("openbsd") ||
-           target.contains("msvc") ||
-           target.contains("emscripten") {
-            build.config.use_jemalloc = false;
-        }
-
         // Can't compile for iOS unless we're on OSX
         if target.contains("apple-ios") &&
            !build.config.build.contains("apple-darwin") {

src/bootstrap/sanity.rs

@@ -418,7 +418,6 @@ impl<'a> Step<'a> {
                     self.check_crate_std(compiler),
                     self.check_crate_test(compiler),
                     self.check_debuginfo(compiler),
-                    self.dist(stage),
                 ];
 
                 // If we're testing the build triple, then we know we can
@@ -463,6 +462,9 @@ impl<'a> Step<'a> {
                         // misc
                         self.check_linkcheck(stage),
                         self.check_tidy(stage),
+
+                        // can we make the distributables?
+                        self.dist(stage),
                     ]);
                 }
                 return base

src/bootstrap/step.rs

@@ -1000,6 +1000,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(target_os = "emscripten", ignore)]
     fn manually_share_arc() {
         let v = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
         let arc_v = Arc::new(v);

src/liballoc/arc.rs

@@ -27,6 +27,24 @@ fn main() {
     let build_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
     let src_dir = env::current_dir().unwrap();
 
+    // FIXME: This is a hack to support building targets that don't
+    // support jemalloc alongside hosts that do. The jemalloc build is
+    // controlled by a feature of the std crate, and if that feature
+    // changes between targets, it invalidates the fingerprint of
+    // std's build script (this is a cargo bug); so we must ensure
+    // that the feature set used by std is the same across all
+    // targets, which means we have to build the alloc_jemalloc crate
+    // for targets like emscripten, even if we don't use it.
+    if target.contains("rumprun") ||
+        target.contains("bitrig") ||
+        target.contains("openbsd") ||
+        target.contains("msvc") ||
+        target.contains("emscripten")
+    {
+        println!("cargo:rustc-cfg=dummy_jemalloc");
+        return;
+    }
+
     if let Some(jemalloc) = env::var_os("JEMALLOC_OVERRIDE") {
         let jemalloc = PathBuf::from(jemalloc);
         println!("cargo:rustc-link-search=native={}",