Merge pull request rust-lang#591 from tgross35/testing-updates

Add a way to test more targets on non-linux machines

.gitignore

@@ -1,3 +1,5 @@
 *.rs.bk
 Cargo.lock
 target
+compiler-rt
+*.tar.gz

README.md

@@ -78,6 +78,26 @@ features = ["c"]
 [8]: http://en.cppreference.com/w/cpp/language/implicit_conversion
 [9]: https://doc.rust-lang.org/std/primitive.i32.html
 
+## Testing
+
+The easiest way to test locally is using Docker. This can be done by running
+`./ci/run-docker.sh [target]`. If no target is specified, all targets will be
+run.
+
+In order to run the full test suite, you will also need the C compiler runtime
+to test against, located in a directory called `compiler-rt`. This can be
+obtained with the following:
+
+```sh
+curl -L -o rustc-llvm-18.0.tar.gz https://github.com/rust-lang/llvm-project/archive/rustc/18.0-2024-02-13.tar.gz
+tar xzf rustc-llvm-18.0.tar.gz --strip-components 1 llvm-project-rustc-18.0-2024-02-13/compiler-rt
+````
+
+Local targets may also be tested with `./ci/run.sh [target]`.
+
+Note that testing may not work on all hosts, in which cases it is acceptable to
+rely on CI.
+
 ## Progress
 
 - [x] adddf3.c

build.rs

@@ -572,7 +572,9 @@ mod c {
         // rust-lang/rust.
         let root = match env::var_os("RUST_COMPILER_RT_ROOT") {
             Some(s) => PathBuf::from(s),
-            None => panic!("RUST_COMPILER_RT_ROOT is not set"),
+            None => {
+                panic!("RUST_COMPILER_RT_ROOT is not set. You may need to download compiler-rt.")
+            }
         };
         if !root.exists() {
             panic!("RUST_COMPILER_RT_ROOT={} does not exist", root.display());

ci/docker/aarch64-unknown-linux-gnu/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \

ci/docker/arm-unknown-linux-gnueabi/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \

ci/docker/arm-unknown-linux-gnueabihf/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \

ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \

ci/docker/i586-unknown-linux-gnu/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc-multilib libc6-dev ca-certificates

ci/docker/i686-unknown-linux-gnu/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc-multilib libc6-dev ca-certificates

ci/docker/mips-unknown-linux-gnu/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     ca-certificates \

ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     ca-certificates \

ci/docker/mipsel-unknown-linux-gnu/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

ci/docker/powerpc-unknown-linux-gnu/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

ci/docker/powerpc64-unknown-linux-gnu/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

ci/docker/thumbv6m-none-eabi/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \

ci/docker/thumbv7em-none-eabi/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \

ci/docker/thumbv7em-none-eabihf/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \

ci/docker/thumbv7m-none-eabi/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \

ci/docker/wasm32-unknown-unknown/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:20.04
+ARG IMAGE=ubuntu:20.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc clang libc6-dev ca-certificates

ci/docker/x86_64-unknown-linux-gnu/Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates

ci/run-docker.sh

@@ -1,38 +1,77 @@
+#!/bin/bash
+
 # Small script to run tests for a target (or all targets) inside all the
 # respective docker images.
 
-set -ex
+set -euxo pipefail
 
 run() {
-    local target=$1
+    local target="$1"
 
-    echo $target
+    echo "TESTING TARGET: $target"
 
     # This directory needs to exist before calling docker, otherwise docker will create it but it
     # will be owned by root
     mkdir -p target
 
-    docker build -t $target ci/docker/$target
+    if [ $(uname -s) = "Linux" ] && [ -z "${DOCKER_BASE_IMAGE:-}" ]; then
+      # Share the host rustc and target. Do this only on Linux and if the image
+      # isn't overridden
+      run_args=(
+           --user "$(id -u):$(id -g)"
+           -e "CARGO_HOME=/cargo"
+           -v "${HOME}/.cargo:/cargo"
+           -v "$(pwd)/target:/builtins-target" 
+           -v "$(rustc --print sysroot):/rust:ro"
+      )
+      run_cmd="HOME=/tmp PATH=\$PATH:/rust/bin ci/run.sh $target"
+    else
+      # Use rustc provided by a docker image
+      docker volume create compiler-builtins-cache
+      build_args=(
+        "--build-arg" "IMAGE=${DOCKER_BASE_IMAGE:-rustlang/rust:nightly}"
+      )
+      run_args=(
+        -v "compiler-builtins-cache:/builtins-target"
+      )
+      run_cmd="HOME=/tmp USING_CONTAINER_RUSTC=1 ci/run.sh $target"
+    fi
+
+    if [ -d compiler-rt ]; then
+      export RUST_COMPILER_RT_ROOT=./compiler-rt
+    fi
+
+    docker build \
+           -t "builtins-$target" \
+           ${build_args[@]:-} \
+           "ci/docker/$target"
     docker run \
            --rm \
-           --user $(id -u):$(id -g) \
-           -e CARGO_HOME=/cargo \
-           -e CARGO_TARGET_DIR=/target \
            -e RUST_COMPILER_RT_ROOT \
-           -v "${HOME}/.cargo":/cargo \
-           -v `pwd`/target:/target \
-           -v `pwd`:/checkout:ro \
-           -v `rustc --print sysroot`:/rust:ro \
+           -e "CARGO_TARGET_DIR=/builtins-target" \
+           -v "$(pwd):/checkout:ro" \
            -w /checkout \
+           ${run_args[@]:-} \
            --init \
-           $target \
-           sh -c "HOME=/tmp PATH=\$PATH:/rust/bin ci/run.sh $target"
+           "builtins-$target" \
+           sh -c "$run_cmd"
 }
 
-if [ -z "$1" ]; then
-  for d in `ls ci/docker/`; do
-    run $d
+if [ "${1:-}" = "--help" ] || [ "$#" -gt 1 ]; then
+  set +x
+  echo "\
+    usage: ./ci/run-docker.sh [target]
+
+    you can also set DOCKER_BASE_IMAGE to use something other than the default
+    ubuntu:18.04 (or rustlang/rust:nightly).
+  "
+  exit
+fi
+
+if [ -z "${1:-}" ]; then
+  for d in ci/docker/*; do
+    run $(basename "$d")
   done
 else
-  run $1
+  run "$1"
 fi