From c4a654d9cdbf259b4ec563df22787c31a3149b93 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Tue, 30 Jul 2019 21:23:59 +0200 Subject: [PATCH 01/11] bootstrap: get rid of TEST_MIRI env var --- src/bootstrap/bin/rustc.rs | 14 -------------- src/bootstrap/builder.rs | 1 - 2 files changed, 15 deletions(-) diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs index 23f81c2c87692..672b2ee14430c 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/bin/rustc.rs @@ -285,20 +285,6 @@ fn main() { } } - // When running miri tests, we need to generate MIR for all libraries - if env::var("TEST_MIRI").ok().map_or(false, |val| val == "true") { - // The flags here should be kept in sync with `add_miri_default_args` - // in miri's `src/lib.rs`. - cmd.arg("-Zalways-encode-mir"); - cmd.arg("--cfg=miri"); - // These options are preferred by miri, to be able to perform better validation, - // but the bootstrap compiler might not understand them. - if stage != "0" { - cmd.arg("-Zmir-emit-retag"); - cmd.arg("-Zmir-opt-level=0"); - } - } - if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") { cmd.arg("--remap-path-prefix").arg(&map); } diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 5a75497173eb3..52e37bfa1b0fd 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -981,7 +981,6 @@ impl<'a> Builder<'a> { PathBuf::from("/path/to/nowhere/rustdoc/not/required") }, ) - .env("TEST_MIRI", self.config.test_miri.to_string()) .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir()); if let Some(host_linker) = self.linker(compiler.host) { From ffb5f18a520c92f6a26039b75f9e72ddbb985435 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 31 Jul 2019 09:01:42 +0200 Subject: [PATCH 02/11] simplify a match --- src/bootstrap/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index c8ea3157dc92b..b72aa78f3de19 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -540,9 +540,7 @@ impl Build { Mode::Rustc => "-rustc", Mode::Codegen => "-codegen", Mode::ToolBootstrap => "-bootstrap-tools", - Mode::ToolStd => "-tools", - Mode::ToolTest => "-tools", - Mode::ToolRustc => "-tools", + Mode::ToolStd | Mode::ToolTest | Mode::ToolRustc => "-tools", }; self.out.join(&*compiler.host) .join(format!("stage{}{}", compiler.stage, suffix)) From 78b658004cce335b19743cd5406dd3c5ecfe824b Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 31 Jul 2019 15:24:09 +0200 Subject: [PATCH 03/11] Miri test: call 'cargo miri test' and use the sysroot it has set up --- src/bootstrap/bin/rustc.rs | 7 +++- src/bootstrap/test.rs | 82 +++++++++++++++++++++++++++++++++----- 2 files changed, 76 insertions(+), 13 deletions(-) diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs index 672b2ee14430c..54b689fb062a5 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/bin/rustc.rs @@ -143,8 +143,11 @@ fn main() { if let Some(target) = target { // The stage0 compiler has a special sysroot distinct from what we - // actually downloaded, so we just always pass the `--sysroot` option. - cmd.arg("--sysroot").arg(&sysroot); + // actually downloaded, so we just always pass the `--sysroot` option, + // unless one is already set. + if !args.iter().any(|arg| arg == "--sysroot") { + cmd.arg("--sysroot").arg(&sysroot); + } cmd.arg("-Zexternal-macro-backtrace"); diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index e5b0a46ba6fb6..5a1f8ead9f2c4 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -389,26 +389,86 @@ impl Step for Miri { extra_features: Vec::new(), }); if let Some(miri) = miri { - let mut cargo = tool::prepare_tool_cargo(builder, - compiler, - Mode::ToolRustc, - host, - "test", - "src/tools/miri", - SourceType::Submodule, - &[]); + // # Run `cargo miri setup`. + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "run", + "src/tools/miri", + SourceType::Submodule, + &[], + ); + cargo + .arg("--bin") + .arg("cargo-miri") + .arg("--") + .arg("miri") + .arg("setup"); + + // Tell `cargo miri` not to worry about the sysroot mismatch (we built with + // stage1 but run with stage2). + cargo.env("MIRI_SKIP_SYSROOT_CHECK", "1"); + // Tell `cargo miri setup` where to find the sources. + cargo.env("XARGO_RUST_SRC", builder.src.join("src")); + // Make sure the libstd gets built without debug assertions. + cargo.env("RUSTC_DEBUG_ASSERTIONS", "false"); + + if !try_run(builder, &mut cargo) { + return; + } + + // # Determine where Miri put its sysroot. + // To this end, we run `cargo miri setup --env` and capture the output. + // (We do this separately from the above so that when the setup actually + // happens we get some output.) + // We re-use the `cargo` from above. + cargo.arg("--env"); + + // FIXME: Is there a way in which we can re-use the usual `run` helpers? + let miri_sysroot = if builder.config.dry_run { + String::new() + } else { + let out = cargo.output() + .expect("We already ran `cargo miri setup` before and that worked"); + assert!(out.status.success(), "`cargo miri setup` returned with non-0 exit code"); + // Output is "MIRI_SYSROOT=\n". + let stdout = String::from_utf8(out.stdout) + .expect("`cargo miri setup` stdout is not valid UTF-8"); + let stdout = stdout.trim(); + builder.verbose(&format!("`cargo miri setup --env` returned: {:?}", stdout)); + let sysroot = stdout.splitn(2, '=') + .nth(1).expect("`cargo miri setup` stdout did not contain '='"); + sysroot.to_owned() + }; + + // # Run `cargo test`. + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/miri", + SourceType::Submodule, + &[], + ); // miri tests need to know about the stage sysroot - cargo.env("MIRI_SYSROOT", builder.sysroot(compiler)); + cargo.env("MIRI_SYSROOT", miri_sysroot); cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler)); cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); cargo.env("MIRI_PATH", miri); builder.add_rustc_lib_path(compiler, &mut cargo); - if try_run(builder, &mut cargo) { - builder.save_toolstate("miri", ToolState::TestPass); + if !try_run(builder, &mut cargo) { + return; } + + // # Done! + builder.save_toolstate("miri", ToolState::TestPass); } else { eprintln!("failed to test miri: could not build"); } From 946bed89da398187c77e9ef7d7b4c69e64deaba5 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 31 Jul 2019 15:26:56 +0200 Subject: [PATCH 04/11] remove test-miri flag from bootstrap --- src/bootstrap/builder.rs | 9 --------- src/bootstrap/config.rs | 4 ---- src/bootstrap/configure.py | 1 - src/bootstrap/test.rs | 3 +-- src/ci/azure-pipelines/auto.yml | 2 +- src/ci/docker/x86_64-gnu-tools/Dockerfile | 1 - 6 files changed, 2 insertions(+), 18 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 52e37bfa1b0fd..e54c9360baece 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -543,15 +543,6 @@ impl<'a> Builder<'a> { parent: Cell::new(None), }; - if kind == Kind::Dist { - assert!( - !builder.config.test_miri, - "Do not distribute with miri enabled.\n\ - The distributed libraries would include all MIR (increasing binary size). - The distributed MIR would include validation statements." - ); - } - builder } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 5a5f4ac725204..a5bfafdfdb4dc 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -128,7 +128,6 @@ pub struct Config { pub low_priority: bool, pub channel: String, pub verbose_tests: bool, - pub test_miri: bool, pub save_toolstates: Option, pub print_step_timings: bool, pub missing_tools: bool, @@ -315,7 +314,6 @@ struct Rust { debug: Option, dist_src: Option, verbose_tests: Option, - test_miri: Option, incremental: Option, save_toolstates: Option, codegen_backends: Option>, @@ -375,7 +373,6 @@ impl Config { config.codegen_tests = true; config.ignore_git = false; config.rust_dist_src = true; - config.test_miri = false; config.rust_codegen_backends = vec![INTERNER.intern_str("llvm")]; config.rust_codegen_backends_dir = "codegen-backends".to_owned(); config.deny_warnings = true; @@ -557,7 +554,6 @@ impl Config { set(&mut config.channel, rust.channel.clone()); set(&mut config.rust_dist_src, rust.dist_src); set(&mut config.verbose_tests, rust.verbose_tests); - set(&mut config.test_miri, rust.test_miri); // in the case "false" is set explicitly, do not overwrite the command line args if let Some(true) = rust.incremental { config.incremental = true; diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index 298330869b684..346f0cb2039c0 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -36,7 +36,6 @@ def v(*args): o("compiler-docs", "build.compiler-docs", "build compiler documentation") o("optimize-tests", "rust.optimize-tests", "build tests with optimizations") o("parallel-compiler", "rust.parallel-compiler", "build a multi-threaded rustc") -o("test-miri", "rust.test-miri", "run miri's test suite") o("verbose-tests", "rust.verbose-tests", "enable verbose output when running tests") o("ccache", "llvm.ccache", "invoke gcc/clang via ccache to reuse object files between builds") o("sccache", None, "invoke gcc/clang via sccache to reuse object files between builds") diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 5a1f8ead9f2c4..79d8b867cb61e 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -366,8 +366,7 @@ impl Step for Miri { const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let test_miri = run.builder.config.test_miri; - run.path("src/tools/miri").default_condition(test_miri) + run.path("src/tools/miri") } fn make_run(run: RunConfig<'_>) { diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 6ed1585c5168b..687856cca6b62 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -254,7 +254,7 @@ jobs: x86_64-msvc-tools: MSYS_BITS: 64 SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json # 32/64-bit MinGW builds. # diff --git a/src/ci/docker/x86_64-gnu-tools/Dockerfile b/src/ci/docker/x86_64-gnu-tools/Dockerfile index bab9145cbcb9c..f11ae7a34cb91 100644 --- a/src/ci/docker/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/x86_64-gnu-tools/Dockerfile @@ -23,6 +23,5 @@ COPY x86_64-gnu-tools/repo.sh /tmp/ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ - --enable-test-miri \ --save-toolstates=/tmp/toolstates.json ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstates.json linux From 2d49118fb4847de19c5263b1c8789c9ef600461a Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 31 Jul 2019 15:39:30 +0200 Subject: [PATCH 05/11] more comments --- src/bootstrap/test.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 79d8b867cb61e..58a0b5dde2e09 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -389,6 +389,7 @@ impl Step for Miri { }); if let Some(miri) = miri { // # Run `cargo miri setup`. + // As a side-effect, this will install xargo. let mut cargo = tool::prepare_tool_cargo( builder, compiler, From 295f894b18fedd1ff5748ef122166840a8b83e3c Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Thu, 1 Aug 2019 21:06:25 +0200 Subject: [PATCH 06/11] remove test-miri from config.toml.example --- config.toml.example | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config.toml.example b/config.toml.example index 6816eaeaa9486..cb9f388a8e47b 100644 --- a/config.toml.example +++ b/config.toml.example @@ -368,10 +368,6 @@ # When creating source tarballs whether or not to create a source tarball. #dist-src = false -# Whether to also run the Miri tests suite when running tests. -# As a side-effect also generates MIR for all libraries. -#test-miri = false - # After building or testing extended tools (e.g. clippy and rustfmt), append the # result (broken, compiling, testing) into this JSON file. #save-toolstates = "/path/to/toolstates.json" From ceda77486f8a37f85326f38bfc7509eea4c38ce9 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Fri, 2 Aug 2019 17:56:55 +0200 Subject: [PATCH 07/11] dont test Miri by default (also be more verbose when being verbose) --- src/bootstrap/test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 58a0b5dde2e09..00a2802ba8143 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -363,7 +363,6 @@ pub struct Miri { impl Step for Miri { type Output = (); const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path("src/tools/miri") @@ -430,6 +429,7 @@ impl Step for Miri { let miri_sysroot = if builder.config.dry_run { String::new() } else { + builder.verbose(&format!("running: {:?}", cargo)); let out = cargo.output() .expect("We already ran `cargo miri setup` before and that worked"); assert!(out.status.success(), "`cargo miri setup` returned with non-0 exit code"); From 05aa983f4c3fa10de98be229451b32fdd5fa7c49 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Fri, 2 Aug 2019 14:20:42 +0200 Subject: [PATCH 08/11] let us try to find out where that panic is coming from --- src/bootstrap/test.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 00a2802ba8143..8b8b38b15098f 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -413,6 +413,8 @@ impl Step for Miri { cargo.env("XARGO_RUST_SRC", builder.src.join("src")); // Make sure the libstd gets built without debug assertions. cargo.env("RUSTC_DEBUG_ASSERTIONS", "false"); + // Debug things. + cargo.env("RUST_BACKTRACE", "1"); if !try_run(builder, &mut cargo) { return; From 03de7fdbfacb176394c257e7ce921466c1af378c Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 3 Aug 2019 14:30:57 +0200 Subject: [PATCH 09/11] install and use xargo inside the build dir --- src/bootstrap/test.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 8b8b38b15098f..1850d60f139c9 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -415,6 +415,10 @@ impl Step for Miri { cargo.env("RUSTC_DEBUG_ASSERTIONS", "false"); // Debug things. cargo.env("RUST_BACKTRACE", "1"); + // Configure `cargo install` path, and let cargo-miri know that that's where + // xargo ends up. + cargo.env("CARGO_INSTALL_ROOT", &builder.out); // cargo adds a `bin/` + cargo.env("XARGO", builder.out.join("bin").join("xargo")); if !try_run(builder, &mut cargo) { return; From 82b3b8229b2a99d499220caa0712cca02b697c70 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 3 Aug 2019 17:26:01 +0200 Subject: [PATCH 10/11] don't set RUSTC_DEBUG_ASSERTIONS here; let cargo-miri do that --- src/bootstrap/test.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 1850d60f139c9..c2c134bfd1d7d 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -411,8 +411,6 @@ impl Step for Miri { cargo.env("MIRI_SKIP_SYSROOT_CHECK", "1"); // Tell `cargo miri setup` where to find the sources. cargo.env("XARGO_RUST_SRC", builder.src.join("src")); - // Make sure the libstd gets built without debug assertions. - cargo.env("RUSTC_DEBUG_ASSERTIONS", "false"); // Debug things. cargo.env("RUST_BACKTRACE", "1"); // Configure `cargo install` path, and let cargo-miri know that that's where From e6be1d713497a47ddbcedd2776d999fe4324cc16 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Thu, 8 Aug 2019 19:32:33 +0200 Subject: [PATCH 11/11] update miri --- src/tools/miri | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/miri b/src/tools/miri index 39a524f694e42..b12ebfc3de853 160000 --- a/src/tools/miri +++ b/src/tools/miri @@ -1 +1 @@ -Subproject commit 39a524f694e42cfb178639d490d3fdbbaf8569dc +Subproject commit b12ebfc3de853abf6b4260c44a71cd51323803c5