Auto merge of #58575 - mati865:musl_toolchain, r=<try>

WIP Musl host toolchain

Based on #55163 and #57359
Depends on #55566

CC #57439

### How it works

Tested compiler made by `dist` on glibc and musl based distributions and verified binaries it produces:
* Ubuntu (glibc) - installed it as a target for host toolchain and observed no regressions for static (default) linking, dynamic linking apparently requires musl build libgcc so I didn't test it.
* Alpine (musl) - installed as the host toolchain, by default it links statically (executables are portable and work on glibc distributions) but with `-C target-feature=-crt-static` Rust flag it links dynamically (executables require musl built libraries).

### What's debatable

It should be decided whether this toolchain should link dynamically or statically when using it on musl distribution. I believe the distributions would prefer dynamic linking but it'd be misleading because `$ARCH-unknown-linux-musl` target links statically on the other hosts.
Another problem is using `RUSTFLAGS='-C target-feature=-crt-static'` for dynamic builds which is really uncomfortable.

To address both issues I suggest leaving `$ARCH-unknown-linux-musl` static for both host and cross target and introducing "alias triple" `$ARCH-unknown-linux-dynmusl`. It'd be the same as `$ARCH-unknown-linux-musl` (and use the same libraries to avoid duplication) but it'd link dynamically.

<del>
### Why it's still WIP (help wanted)

I'm having a hard time getting all tests to pass and I'd appreciate help.

Non-verbose error:
<details>

```
Testing proc_macro stage1 (x86_64-unknown-linux-musl -> x86_64-unknown-linux-musl)
   Compiling proc_macro v0.0.0 (/checkout/src/libproc_macro)
error[E0463]: can't find crate for `std`

error[E0463]: can't find crate for `std`

error: aborting due to previous error

For more information about this error, try `rustc --explain E0463`.
error: aborting due to previous error

For more information about this error, try `rustc --explain E0463`.
[RUSTC-TIMING] proc_macro test:true 0.529
[RUSTC-TIMING] proc_macro test:false 0.530
error: Could not compile `proc_macro`.
warning: build failed, waiting for other jobs to finish...
error: Could not compile `proc_macro`.

To learn more, run the command again with --verbose.

command did not execute successfully: "/checkout/obj/build/x86_64-unknown-linux-gnu/stage0/bin/cargo" "test" "--target" "x86_64-unknown-linux-musl" "-j" "16" "--release" "--locked" "--manifest-path" "/checkout/src/libtest/Cargo.toml" "-p" "proc_macro" "--"
expected success, got: exit code: 101

failed to run: /checkout/obj/build/bootstrap/debug/bootstrap test --host x86_64-unknown-linux-musl --target x86_64-unknown-linux-musl
```
</details>

Verbose error:
<details>

```
Testing proc_macro stage1 (x86_64-unknown-linux-musl -> x86_64-unknown-linux-musl)
   Compiling proc_macro v0.0.0 (/checkout/src/libproc_macro)
     Running `/checkout/obj/build/bootstrap/debug/rustc --edition=2018 --crate-name proc_macro src/libproc_macro/lib.rs --color never --crate-type lib --emit=dep-info,link -C opt-level=2 -C metadata=09ddd3ecc930ab63 -C extra-filename=-09ddd3ecc930ab63 --out-dir /checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/x86_64-unknown-linux-musl/release/deps --target x86_64-unknown-linux-musl -L dependency=/checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/x86_64-unknown-linux-musl/release/deps -L dependency=/checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/release/deps -C target-feature=-crt-static`
     Running `/checkout/obj/build/bootstrap/debug/rustc --edition=2018 --crate-name proc_macro src/libproc_macro/lib.rs --color never --emit=dep-info,link -C opt-level=2 --test -C metadata=a564d363930469c8 -C extra-filename=-a564d363930469c8 --out-dir /checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/x86_64-unknown-linux-musl/release/deps --target x86_64-unknown-linux-musl -L dependency=/checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/x86_64-unknown-linux-musl/release/deps -L dependency=/checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/release/deps -C target-feature=-crt-static`
error[E0463]: can't find crate for `std`
error[E0463]: can't find crate for `std`

error: aborting due to previous error

For more information about this error, try `rustc --explain E0463`.
error: aborting due to previous error

For more information about this error, try `rustc --explain E0463`.
[RUSTC-TIMING] proc_macro test:false 0.248
error: Could not compile `proc_macro`.

Caused by:
  process didn't exit successfully: `/checkout/obj/build/bootstrap/debug/rustc --edition=2018 --crate-name proc_macro src/libproc_macro/lib.rs --color never --crate-type lib --emit=dep-info,link -C opt-level=2 -C metadata=09ddd3ecc930ab63 -C extra-filename=-09ddd3ecc930ab63 --out-dir /checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/x86_64-unknown-linux-musl/release/deps --target x86_64-unknown-linux-musl -L dependency=/checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/x86_64-unknown-linux-musl/release/deps -L dependency=/checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/release/deps -C target-feature=-crt-static` (exit code: 1)
warning: build failed, waiting for other jobs to finish...
[RUSTC-TIMING] proc_macro test:true 0.248
error: Could not compile `proc_macro`.

Caused by:
  process didn't exit successfully: `/checkout/obj/build/bootstrap/debug/rustc --edition=2018 --crate-name proc_macro src/libproc_macro/lib.rs --color never --emit=dep-info,link -C opt-level=2 --test -C metadata=a564d363930469c8 -C extra-filename=-a564d363930469c8 --out-dir /checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/x86_64-unknown-linux-musl/release/deps --target x86_64-unknown-linux-musl -L dependency=/checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/x86_64-unknown-linux-musl/release/deps -L dependency=/checkout/obj/build/x86_64-unknown-linux-musl/stage1-test/release/deps -C target-feature=-crt-static` (exit code: 1)

command did not execute successfully: "/checkout/obj/build/x86_64-unknown-linux-gnu/stage0/bin/cargo" "test" "--target" "x86_64-unknown-linux-musl" "-j" "16" "--release" "--locked" "--manifest-path" "/checkout/src/libtest/Cargo.toml" "--verbose" "-p" "proc_macro" "--"
expected success, got: exit code: 101

failed to run: /checkout/obj/build/bootstrap/debug/bootstrap test --host x86_64-unknown-linux-musl --target x86_64-unknown-linux-musl
```

</details>

Whole tests non-verbose output: [rust-tests.log](https://github.com/rust-lang/rust/files/2879945/rust-tests.log)

I think the error is because build system (correctly?) tries to use `obj/build/x86_64-unknown-linux-musl/stage1-test/x86_64-unknown-linux-musl/release/deps` which is empty but `obj/build/x86_64-unknown-linux-gnu/stage1-test/x86_64-unknown-linux-musl/release/deps` contains required libs.
</del>

.travis.yml

@@ -17,7 +17,7 @@ matrix:
  include:
  # Images used in testing PR and try-build should be run first.
  - env: IMAGE=x86_64-gnu-llvm-6.0 RUST_BACKTRACE=1
- if: type = pull_request OR branch = auto
+ if: branch = auto
 
  - env: IMAGE=dist-x86_64-linux DEPLOY=1
  if: branch = try OR branch = auto
@@ -159,7 +159,7 @@ matrix:
  - env: IMAGE=dist-x86_64-freebsd DEPLOY=1
  if: branch = auto
  - env: IMAGE=dist-x86_64-musl DEPLOY=1
- if: branch = auto
+# if: branch = auto
  - env: IMAGE=dist-x86_64-netbsd DEPLOY=1
  if: branch = auto
  - env: IMAGE=asmjs
@@ -185,7 +185,7 @@ matrix:
  - env: IMAGE=x86_64-gnu-distcheck
  if: branch = auto
  - env: IMAGE=mingw-check
- if: type = pull_request OR branch = auto
+ if: branch = auto
 
  - stage: publish toolstate
  if: branch = master AND type = push

src/bootstrap/test.rs

@@ -1636,8 +1636,9 @@ impl Step for Crate {
  // libstd, then what we're actually testing is the libstd produced in
  // stage1. Reflect that here by updating the compiler that we're working
  // with automatically.
+ // FIXME(mati865): do similar for the other branch if this is correct
  let compiler = if builder.force_use_stage1(compiler, target) {
- builder.compiler(1, compiler.host)
+ builder.compiler(1, builder.config.build)
  } else {
  compiler.clone()
  };
@@ -1802,6 +1803,10 @@ impl Step for CrateRustdoc {
  cargo.arg("--");
  cargo.args(&builder.config.cmd.test_args());
 
+ if target.contains("musl") {
+ cargo.arg("'-Ctarget-feature=-crt-static'");
+ }
+
  if !builder.config.verbose_tests {
  cargo.arg("--quiet");
  }

src/ci/docker/dist-x86_64-musl/Dockerfile

@@ -4,6 +4,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  g++ \
  make \
  file \
+ wget \
  curl \
  ca-certificates \
  python2.7 \
@@ -18,19 +19,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 WORKDIR /build/
 
-COPY scripts/musl.sh /build/
+COPY scripts/musl-toolchain.sh /build/
 # We need to mitigate rust-lang/rust#34978 when compiling musl itself as well
-RUN CC=gcc \
- CFLAGS="-Wa,-mrelax-relocations=no" \
- CXX=g++ \
- CXXFLAGS="-Wa,-mrelax-relocations=no" \
- bash musl.sh x86_64 && rm -rf /build
+# TODO: Check what this issue is and if we can ignore it
+
+RUN bash musl-toolchain.sh x86_64-linux-musl && rm -rf build
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
 ENV RUST_CONFIGURE_ARGS \
- --musl-root-x86_64=/musl-x86_64 \
+ --musl-root-x86_64=/usr/local/x86_64-linux-musl \
  --enable-extended \
  --disable-docs
 
@@ -41,6 +40,15 @@ ENV RUST_CONFIGURE_ARGS \
 # See: https://github.com/rust-lang/rust/issues/34978
 ENV CFLAGS_x86_64_unknown_linux_musl=-Wa,-mrelax-relocations=no
 
+ENV HOSTS=x86_64-unknown-linux-musl \
+ CC_x86_64_unknown_linux_musl=x86_64-linux-musl-gcc \
+ CXX_x86_64_unknown_linux_musl=x86_64-linux-musl-g++
+
+# CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABIHF_LINKER=musl-gcc \
+# CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABIHF_RUNNER="qemu-arm -L /musl-arm"
+
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+
 ENV SCRIPT \
- python2.7 ../x.py test --target x86_64-unknown-linux-musl && \
- python2.7 ../x.py dist --target x86_64-unknown-linux-musl
+ python2.7 ../x.py test --host $HOSTS --target $HOSTS && \
+ python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/scripts/musl-toolchain.sh

@@ -0,0 +1,66 @@
+set -ex
+
+hide_output() {
+ set +x
+ on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/build.log
+exit 1
+"
+ trap "$on_err" ERR
+ bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+ PING_LOOP_PID=$!
+ $@ &> /tmp/build.log
+ trap - ERR
+ kill $PING_LOOP_PID
+ rm /tmp/build.log
+ set -x
+}
+
+TARGET=$1
+#ARCH=$1
+#TARGET=linux-musl-$ARCH
+ARCH=x86_64
+
+OUTPUT=/usr/local
+shift
+
+git clone https://github.com/richfelker/musl-cross-make -b v0.9.7
+cd musl-cross-make
+
+hide_output make -j$(nproc) TARGET=$TARGET
+hide_output make install TARGET=$TARGET OUTPUT=$OUTPUT
+
+cd -
+
+# Make musl binaries executable
+
+ln -s $OUTPUT/$TARGET/lib/libc.so /lib/ld-musl-$ARCH.so.1
+echo $OUTPUT/$TARGET/lib >> /etc/ld-musl-$ARCH.path
+
+
+export CC=$TARGET-gcc
+export CXX=$TARGET-g++
+
+LLVM=70
+
+# may have been downloaded in a previous run
+if [ ! -d libunwind-release_$LLVM ]; then
+ curl -L https://github.com/llvm-mirror/llvm/archive/release_$LLVM.tar.gz | tar xzf -
+ curl -L https://github.com/llvm-mirror/libunwind/archive/release_$LLVM.tar.gz | tar xzf -
+fi
+
+mkdir libunwind-build
+cd libunwind-build
+cmake ../libunwind-release_$LLVM \
+ -DLLVM_PATH=/build/llvm-release_$LLVM \
+ -DLIBUNWIND_ENABLE_SHARED=0 \
+ -DCMAKE_C_COMPILER=$CC \
+ -DCMAKE_CXX_COMPILER=$CXX \
+ -DCMAKE_C_FLAGS="$CFLAGS" \
+ -DCMAKE_CXX_FLAGS="$CXXFLAGS"
+
+hide_output make -j$(nproc)
+cp lib/libunwind.a $OUTPUT/$TARGET/lib
+cd - && rm -rf libunwind-build
+

src/test/run-make-fulldeps/link-cfg/Makefile

@@ -2,7 +2,8 @@
 
 all: $(call DYLIB,return1) $(call DYLIB,return2) $(call NATIVE_STATICLIB,return3)
  ls $(TMPDIR)
- $(RUSTC) --print cfg --target x86_64-unknown-linux-musl | $(CGREP) crt-static
+
+ $(BARE_RUSTC) --print cfg --target x86_64-unknown-linux-musl | $(CGREP) crt-static
 
  $(RUSTC) no-deps.rs --cfg foo
  $(call RUN,no-deps)

src/test/run-make-fulldeps/linker-output-non-utf8/Makefile

@@ -20,4 +20,4 @@ all:
  $(RUSTC) library.rs
  mkdir $(bad_dir)
  mv $(TMPDIR)/liblibrary.a $(bad_dir)
- LIBRARY_PATH=$(bad_dir) $(RUSTC) exec.rs 2>&1 | $(CGREP) this_symbol_not_defined
+ $(RUSTC) -L $(bad_dir) exec.rs 2>&1 | $(CGREP) this_symbol_not_defined

src/test/run-make-fulldeps/reproducible-build/Makefile

@@ -1,4 +1,11 @@
 -include ../tools.mk
+
+# ignore-musl
+#
+# diff:
+# -/checkout/obj/build/x86_64-unknown-linux-musl/test/run-make-fulldeps/reproducible-build/reproducible-build/rustcORhwur/libunwind-cfdf9e23c318976b.rlib: 6220632559893696134
+# +/checkout/obj/build/x86_64-unknown-linux-musl/test/run-make-fulldeps/reproducible-build/reproducible-build/rustcMaw0kI/libunwind-cfdf9e23c318976b.rlib: 6220632559893696134
+
 all: \
  smoke \
  debug \

src/test/run-make/rustc-macro-dep-files/Makefile

@@ -3,6 +3,10 @@
 # FIXME(eddyb) provide `HOST_RUSTC` and `TARGET_RUSTC`
 # instead of hardcoding them everywhere they're needed.
 all:
- $(BARE_RUSTC) foo.rs --out-dir $(TMPDIR)
+ ifeq ($(IS_MUSL_HOST),1)
+ $(BARE_RUSTC) $(RUSTFLAGS) -Clinker=$(RUSTC_LINKER) foo.rs --out-dir $(TMPDIR)
+ else
+ $(BARE_RUSTC) foo.rs --out-dir $(TMPDIR)
+ endif
  $(RUSTC) bar.rs --target $(TARGET) --emit dep-info
  $(CGREP) -v "proc-macro source" < $(TMPDIR)/bar.d

src/tools/compiletest/src/runtest.rs

@@ -1602,18 +1602,15 @@ impl<'test> TestCx<'test> {
  None
  } else if self.config.target.contains("cloudabi")
  || self.config.target.contains("emscripten")
- || (self.config.target.contains("musl") && !aux_props.force_host)
  || self.config.target.contains("wasm32")
  {
  // We primarily compile all auxiliary libraries as dynamic libraries
  // to avoid code size bloat and large binaries as much as possible
  // for the test suite (otherwise including libstd statically in all
  // executables takes up quite a bit of space).
  //
- // For targets like MUSL or Emscripten, however, there is no support for
- // dynamic libraries so we just go back to building a normal library. Note,
- // however, that for MUSL if the library is built with `force_host` then
- // it's ok to be a dylib as the host should always support dylibs.
+ // For targets like Emscripten, however, there is no support for
+ // dynamic libraries so we just go back to building a normal library.
  Some("lib")
  } else {
  Some("dylib")
@@ -1845,19 +1842,36 @@ impl<'test> TestCx<'test> {
  None => {}
  }
 
+ // Musl toolchain is build on linux-gnu host
+ // but with proper setup it can behave almost* like native linux-musl.
+ // One difference is "cc" which will link to glibc; force musl cc.
  if self.props.force_host {
  self.maybe_add_external_args(&mut rustc,
  self.split_maybe_args(&self.config.host_rustcflags));
+ if self.config.target.contains("musl") {
+ if let Some(ref linker) = self.config.linker {
+ rustc.arg(format!("-Clinker={}", linker));
+ }
+ }
  } else {
  self.maybe_add_external_args(&mut rustc,
  self.split_maybe_args(&self.config.target_rustcflags));
  if !is_rustdoc {
  if let Some(ref linker) = self.config.linker {
  rustc.arg(format!("-Clinker={}", linker));
  }
+ } else if self.config.target.contains("musl") {
+ if let Some(ref linker) = self.config.linker {
+ rustc.arg(format!("--linker={}", linker));
+ }
  }
  }
 
+ // Use dynamic musl for tests because static doesn't allow creating dylibs
+ if self.config.target.contains("musl") {
+ rustc.arg("-Ctarget-feature=-crt-static");
+ }
+
  rustc.args(&self.props.compile_flags);
 
  rustc
@@ -2641,6 +2655,12 @@ impl<'test> TestCx<'test> {
  // compiler flags set in the test cases:
  cmd.env_remove("RUSTFLAGS");
 
+ // Use dynamic musl for tests because static doesn't allow creating dylibs
+ if self.config.target.contains("musl") {
+ cmd.env("RUSTFLAGS", "-Ctarget-feature=-crt-static")
+ .env("IS_MUSL_HOST", "1");
+ }
+
  if self.config.target.contains("msvc") && self.config.cc != "" {
  // We need to pass a path to `lib.exe`, so assume that `cc` is `cl.exe`
  // and that `lib.exe` lives next to it.
@@ -2663,8 +2683,13 @@ impl<'test> TestCx<'test> {
  .env("CC", format!("'{}' {}", self.config.cc, cflags))
  .env("CXX", format!("'{}'", &self.config.cxx));
  } else {
- cmd.env("CC", format!("{} {}", self.config.cc, self.config.cflags))
- .env("CXX", format!("{} {}", self.config.cxx, self.config.cflags))
+ let cflags = if self.config.target.contains("musl") {
+ self.config.cflags.replace("-static", "")
+ } else {
+ self.config.cflags.to_string()
+ };
+ cmd.env("CC", format!("{} {}", self.config.cc, cflags))
+ .env("CXX", format!("{} {}", self.config.cxx, cflags))
  .env("AR", &self.config.ar);
 
  if self.config.target.contains("windows") {