Rollup merge of rust-lang#45783 - kennytm:compiler-test-fixes, r=alexcrichton

Miscellaneous changes for CI, Docker and compiletest.

This PR contains 7 independent commits that improves interaction with CI, Docker and compiletest.

1. a4e5c91 — Forces a newline every 100 dots when testing in quiet mode. Prevents spurious timeouts when abusing the CI to test Android jobs.

2. 1b5aaf2 — Use vault.centos.org for dist-powerpc64le-linux, see rust-lang#45744.

3. 33400fb — Modify `src/ci/docker/run.sh` so that the docker images can be run from Docker Toolbox for Windows on Windows 7. I haven't checked the behavior of the newer Docker for Windows on Windows 10. Also, "can run" does not mean all the test can pass successfully (the UDP tests failed last time I checked)

4. d517668 — Don't emit a real warning the linker segfault, which affects UI tests like rust-lang#45489 (comment). Log it instead.

5. 51e2247 — During run-pass, trim the output if stdout/stderr exceeds 416 KB (top 160 KB + bottom 256 KB). This is an attempt to avoid spurious failures like rust-lang#45384 (comment)

6. 9cfdaba — Force `gem update --system` before deploy. This is an attempt to prevent spurious error rust-lang#44159.

7. eee10cc — Tries to print the crash log on macOS on failure. This is an attempt to debug rust-lang#45230.

Author: kennytm

.travis.yml

@@ -253,7 +253,14 @@ after_failure:
 
   # Random attempt at debugging currently. Just poking around in here to see if
   # anything shows up.
-  - ls $HOME/Library/Logs/DiagnosticReports/
+  - ls -lat $HOME/Library/Logs/DiagnosticReports/
+  - find $HOME/Library/Logs/DiagnosticReports/ ! \(
+      -name '*.stage2-*.crash'
+      -name 'com.apple.CoreSimulator.CoreSimulatorService-*.crash'
+    \)
+      -exec echo -e travis_fold":start:crashlog\n\033[31;1m" {} "\033[0m" \;
+      -exec head -750 {} \;
+      -exec echo travis_fold":"end:crashlog \;
 
   # attempt to debug anything killed by the oom killer on linux, just to see if
   # it happened
@@ -286,6 +293,7 @@ before_deploy:
           rm -rf obj/build/dist/doc &&
           cp -r obj/build/dist/* deploy/$TRAVIS_COMMIT;
       fi
+  - travis_retry gem update --system
 
 deploy:
   - provider: s3

src/Cargo.lock

@@ -22,6 +22,48 @@ Images will output artifacts in an `obj` dir at the root of a repository.
 - `scripts` contains files shared by docker images
 - `disabled` contains images that are not built on travis
 
+## Docker Toolbox on Windows
+
+For Windows before Windows 10, the docker images can be run on Windows via
+[Docker Toolbox]. There are several preparation needs to be made before running
+a Docker image.
+
+1. Stop the virtual machine from the terminal with `docker-machine stop`
+
+2. If your Rust source is placed outside of `C:\Users\**`, e.g. if you place the
+    repository in the `E:\rust` folder, please add a shared folder from
+    VirtualBox by:
+
+    1. Select the "default" virtual machine inside VirtualBox, then click
+        "Settings"
+    2. Go to "Shared Folders", click "Add shared foldrer" (the folder icon with
+        a plus sign), fill in the following information, then click "OK":
+
+        * Folder path: `E:\rust`
+        * Folder name: `e/rust`
+        * Read-only: ☐ *unchecked*
+        * Auto-mount: ☑ *checked*
+        * Make Permanant: ☑ *checked*
+
+3. VirtualBox might not support creating symbolic links inside a shared folder
+    by default. You can enable it manually by running these from `cmd.exe`:
+
+    ```bat
+    cd "C:\Program Files\Oracle\VirtualBox"
+    VBoxManage setextradata default VBoxInternal2/SharedFoldersEnableSymlinksCreate/e/rust 1
+    ::                                                                              ^~~~~~
+    ::                                                                              folder name
+    ```
+
+4. Restart the virtual machine from terminal with `docker-machine start`.
+
+To run the image,
+
+1. Launch the "Docker Quickstart Terminal".
+2. Execute `./src/ci/docker/run.sh $image_name` as explained at the beginning.
+
+[Docker Toolbox]: https://www.docker.com/products/docker-toolbox
+
 ## Cross toolchains
 
 A number of these images take quite a long time to compile as they're building
@@ -137,7 +179,7 @@ For targets: `armv7-unknown-linux-gnueabihf`
     libraries like jemalloc. See the mk/cfg/arm(v7)-uknown-linux-gnueabi{,hf}.mk
     file in Rust's source code.
 
-## `aarch64-linux-gnu.config`
+### `aarch64-linux-gnu.config`
 
 For targets: `aarch64-unknown-linux-gnu`
 
@@ -150,7 +192,7 @@ For targets: `aarch64-unknown-linux-gnu`
 - C compiler > gcc version = 5.2.0
 - C compiler > C++ = ENABLE -- to cross compile LLVM
 
-## `powerpc-linux-gnu.config`
+### `powerpc-linux-gnu.config`
 
 For targets: `powerpc-unknown-linux-gnu`
 
@@ -165,7 +207,7 @@ For targets: `powerpc-unknown-linux-gnu`
 - C compiler > gcc version = 4.9.3
 - C compiler > C++ = ENABLE -- to cross compile LLVM
 
-## `powerpc64-linux-gnu.config`
+### `powerpc64-linux-gnu.config`
 
 For targets: `powerpc64-unknown-linux-gnu`
 
@@ -184,7 +226,7 @@ For targets: `powerpc64-unknown-linux-gnu`
 
 (+) These CPU options match the configuration of the toolchains in RHEL6.
 
-## `s390x-linux-gnu.config`
+### `s390x-linux-gnu.config`
 
 For targets: `s390x-unknown-linux-gnu`
 

src/ci/docker/README.md

@@ -23,9 +23,9 @@ SYSROOT=/usr/local/$TARGET/sysroot
 mkdir -p $SYSROOT
 pushd $SYSROOT
 
-centos_base=http://mirror.centos.org/altarch/7/os/ppc64le/Packages
-glibc_v=2.17-196.el7
-kernel_v=3.10.0-693.el7
+centos_base=http://vault.centos.org/altarch/7.3.1611/os/ppc64le/Packages/
+glibc_v=2.17-157.el7
+kernel_v=3.10.0-514.el7
 for package in glibc{,-devel,-headers}-$glibc_v kernel-headers-$kernel_v; do
   curl $centos_base/$package.ppc64le.rpm | \
     rpm2cpio - | cpio -idm

src/ci/docker/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh

@@ -11,6 +11,8 @@
 
 set -e
 
+export MSYS_NO_PATHCONV=1
+
 script=`cd $(dirname $0) && pwd`/`basename $0`
 image=$1
 
@@ -25,12 +27,19 @@ travis_fold start build_docker
 travis_time_start
 
 if [ -f "$docker_dir/$image/Dockerfile" ]; then
+    dockerfile="$docker_dir/$image/Dockerfile"
+    if [ -x /usr/bin/cygpath ]; then
+        context="`cygpath -w $docker_dir`"
+        dockerfile="`cygpath -w $dockerfile`"
+    else
+        context="$docker_dir"
+    fi
     retry docker \
       build \
       --rm \
       -t rust-ci \
-      -f "$docker_dir/$image/Dockerfile" \
-      "$docker_dir"
+      -f "$dockerfile" \
+      "$context"
 elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then
     if [ -n "$TRAVIS_OS_NAME" ]; then
         echo Cannot run disabled images on travis!

src/ci/docker/run.sh

@@ -671,11 +671,12 @@ fn link_natively(sess: &Session,
             break
         }
 
-        sess.struct_warn("looks like the linker segfaulted when we tried to \
-                          call it, automatically retrying again")
-            .note(&format!("{:?}", cmd))
-            .note(&out)
-            .emit();
+        warn!(
+            "looks like the linker segfaulted when we tried to call it, \
+             automatically retrying again. cmd = {:?}, out = {}.",
+            cmd,
+            out,
+        );
     }
 
     match prog {

src/librustc_trans/back/link.rs

@@ -71,6 +71,7 @@ use std::thread;
 use std::time::{Instant, Duration};
 
 const TEST_WARN_TIMEOUT_S: u64 = 60;
+const QUIET_MODE_MAX_COLUMN: usize = 100; // insert a '\n' after 100 tests in quiet mode
 
 // to be used by rustc to compile tests in libtest
 pub mod test {
@@ -614,7 +615,14 @@ impl<T: Write> ConsoleTestState<T> {
     pub fn write_short_result(&mut self, verbose: &str, quiet: &str, color: term::color::Color)
                               -> io::Result<()> {
         if self.quiet {
-            self.write_pretty(quiet, color)
+            self.write_pretty(quiet, color)?;
+            if self.current_test_count() % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 {
+                // we insert a new line every 100 dots in order to flush the
+                // screen when dealing with line-buffered output (e.g. piping to
+                // `stamp` in the rust CI).
+                self.write_plain("\n")?;
+            }
+            Ok(())
         } else {
             self.write_pretty(verbose, color)?;
             self.write_plain("\n")
@@ -771,9 +779,12 @@ impl<T: Write> ConsoleTestState<T> {
         Ok(())
     }
 
+    fn current_test_count(&self) -> usize {
+        self.passed + self.failed + self.ignored + self.measured + self.allowed_fail
+    }
+
     pub fn write_run_finish(&mut self) -> io::Result<bool> {
-        assert!(self.passed + self.failed + self.ignored + self.measured +
-                    self.allowed_fail == self.total);
+        assert!(self.current_test_count() == self.total);
 
         if self.options.display_output {
             self.write_outputs()?;

src/libtest/lib.rs

@@ -11,3 +11,7 @@ getopts = "0.2"
 log = "0.3"
 rustc-serialize = "0.3"
 libc = "0.2"
+
+[target.'cfg(windows)'.dependencies]
+miow = "0.2"
+winapi = "0.2"

src/tools/compiletest/Cargo.toml

@@ -11,10 +11,11 @@
 #![crate_name = "compiletest"]
 
 #![feature(test)]
+#![feature(slice_rotate)]
 
 #![deny(warnings)]
 
-#[cfg(any(target_os = "macos", target_os = "ios"))]
+#[cfg(unix)]
 extern crate libc;
 extern crate test;
 extern crate getopts;
@@ -47,6 +48,7 @@ pub mod runtest;
 pub mod common;
 pub mod errors;
 mod raise_fd_limit;
+mod read2;
 
 fn main() {
     env_logger::init().unwrap();