From f58d107e7c907063715622632e7d47f68a98ccc1 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Fri, 28 Dec 2018 17:47:50 -0800 Subject: [PATCH] testsuite: Require failing commands to check output. --- tests/testsuite/alt_registry.rs | 1 + tests/testsuite/build.rs | 51 +++++++++++--- tests/testsuite/check.rs | 8 ++- tests/testsuite/features.rs | 5 +- tests/testsuite/fix.rs | 2 + tests/testsuite/freshness.rs | 10 ++- tests/testsuite/install.rs | 5 +- tests/testsuite/path.rs | 5 +- tests/testsuite/publish.rs | 4 ++ tests/testsuite/rename_deps.rs | 5 +- tests/testsuite/rustdocflags.rs | 1 + tests/testsuite/rustflags.rs | 115 ++++++++++++++++++++++++++------ tests/testsuite/support/mod.rs | 27 ++++++++ tests/testsuite/workspaces.rs | 8 ++- 14 files changed, 211 insertions(+), 36 deletions(-) diff --git a/tests/testsuite/alt_registry.rs b/tests/testsuite/alt_registry.rs index 6906395ad6e..2bdc1b57e96 100644 --- a/tests/testsuite/alt_registry.rs +++ b/tests/testsuite/alt_registry.rs @@ -310,6 +310,7 @@ fn cannot_publish_to_crates_io_with_registry_dependency() { .arg(registry::registry().to_string()) .masquerade_as_nightly_cargo() .with_status(101) + .with_stderr_contains("[ERROR] crates cannot be published to crates.io[..]") .run(); } diff --git a/tests/testsuite/build.rs b/tests/testsuite/build.rs index c89f7ba8fbc..4d9cc80e04d 100644 --- a/tests/testsuite/build.rs +++ b/tests/testsuite/build.rs @@ -542,15 +542,19 @@ fn cargo_compile_with_invalid_code_in_deps() { .build(); let _bar = project() .at("bar") - .file("Cargo.toml", &basic_bin_manifest("bar")) + .file("Cargo.toml", &basic_manifest("bar", "0.1.0")) .file("src/lib.rs", "invalid rust code!") .build(); let _baz = project() .at("baz") - .file("Cargo.toml", &basic_bin_manifest("baz")) + .file("Cargo.toml", &basic_manifest("baz", "0.1.0")) .file("src/lib.rs", "invalid rust code!") .build(); - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr_contains("[..]invalid rust code[..]") + .with_stderr_contains("[ERROR] Could not compile [..]") + .run(); } #[test] @@ -1266,7 +1270,10 @@ fn compile_offline_while_transitive_dep_not_cached() { .build(); // simulate download bar, but fail to download baz - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr_contains("[..]failed to verify the checksum of `baz[..]") + .run(); drop(File::create(baz_path).ok().unwrap().write_all(&content)); @@ -2169,7 +2176,10 @@ fn deletion_causes_failure() { p.cargo("build").run(); p.change_file("Cargo.toml", &basic_manifest("foo", "0.0.1")); - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr_contains("[..]can't find crate for `bar`") + .run(); } #[test] @@ -2541,11 +2551,14 @@ fn bad_platform_specific_dependency() { .file("bar/Cargo.toml", &basic_manifest("bar", "0.5.0")) .file( "bar/src/lib.rs", - r#"extern crate baz; pub fn gimme() -> String { format!("") }"#, + r#"pub fn gimme() -> String { format!("") }"#, ) .build(); - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr_contains("[..]can't find crate for `bar`") + .run(); } #[test] @@ -2858,7 +2871,17 @@ fn dashes_in_crate_name_bad() { .file("src/main.rs", "extern crate foo_bar; fn main() {}") .build(); - p.cargo("build -v").with_status(101).run(); + p.cargo("build -v") + .with_status(101) + .with_stderr( + "\ +[ERROR] failed to parse manifest at `[..]/foo/Cargo.toml` + +Caused by: + library target names cannot contain hyphens: foo-bar +", + ) + .run(); } #[test] @@ -4513,7 +4536,17 @@ fn avoid_dev_deps() { .file("src/main.rs", "fn main() {}") .build(); - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr( + "\ +[UPDATING] [..] +[ERROR] no matching package named `baz` found +location searched: registry `https://github.com/rust-lang/crates.io-index` +required by package `bar v0.1.0 ([..]/foo)` +", + ) + .run(); p.cargo("build -Zavoid-dev-deps") .masquerade_as_nightly_cargo() .run(); diff --git a/tests/testsuite/check.rs b/tests/testsuite/check.rs index c4d83244398..390293d0f8e 100644 --- a/tests/testsuite/check.rs +++ b/tests/testsuite/check.rs @@ -61,7 +61,10 @@ fn check_fail() { .file("src/lib.rs", "pub fn baz() {}") .build(); - foo.cargo("check").with_status(101).run(); + foo.cargo("check") + .with_status(101) + .with_stderr_contains("[..]this function takes 0 parameters but 1 parameter was supplied") + .run(); } #[test] @@ -363,6 +366,9 @@ fn rustc_check_err() { foo.cargo("rustc --profile check -- --emit=metadata") .with_status(101) + .with_stderr_contains("[CHECKING] bar [..]") + .with_stderr_contains("[CHECKING] foo [..]") + .with_stderr_contains("[..]cannot find function `qux` in module `bar`") .run(); } diff --git a/tests/testsuite/features.rs b/tests/testsuite/features.rs index 7b0386723df..52f2c44f5e5 100644 --- a/tests/testsuite/features.rs +++ b/tests/testsuite/features.rs @@ -1171,7 +1171,10 @@ fn dep_feature_in_cmd_line() { // The foo project requires that feature "some-feat" in "bar" is enabled. // Building without any features enabled should fail: - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr_contains("[..]unresolved import `bar::test`") + .run(); // We should be able to enable the feature "derived-feat", which enables "some-feat", // on the command line. The feature is enabled, thus building should be successful: diff --git a/tests/testsuite/fix.rs b/tests/testsuite/fix.rs index 51a698024a0..b37e3756937 100644 --- a/tests/testsuite/fix.rs +++ b/tests/testsuite/fix.rs @@ -29,6 +29,7 @@ fn do_not_fix_broken_builds() { p.cargo("fix --allow-no-vcs") .env("__CARGO_FIX_YOLO", "1") .with_status(101) + .with_stderr_contains("[ERROR] Could not compile `foo`.") .run(); assert!(p.read_file("src/lib.rs").contains("let mut x = 3;")); } @@ -1247,6 +1248,7 @@ fn fix_to_broken_code() { .cwd(p.root().join("bar")) .env("RUSTC", p.root().join("foo/target/debug/foo")) .with_status(101) + .with_stderr_contains("[WARNING] failed to automatically apply fixes [..]") .run(); assert_eq!( diff --git a/tests/testsuite/freshness.rs b/tests/testsuite/freshness.rs index 74daba86518..359c08b9142 100644 --- a/tests/testsuite/freshness.rs +++ b/tests/testsuite/freshness.rs @@ -40,7 +40,10 @@ fn modifying_and_moving() { .run(); fs::rename(&p.root().join("src/a.rs"), &p.root().join("src/b.rs")).unwrap(); - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr_contains("[..]file not found[..]") + .run(); } #[test] @@ -513,7 +516,10 @@ fn rebuild_tests_if_lib_changes() { File::create(&p.root().join("src/lib.rs")).unwrap(); p.cargo("build -v").run(); - p.cargo("test -v").with_status(101).run(); + p.cargo("test -v") + .with_status(101) + .with_stderr_contains("[..]cannot find function `foo`[..]") + .run(); } #[test] diff --git a/tests/testsuite/install.rs b/tests/testsuite/install.rs index 90917f69e6c..5c202ee3b07 100644 --- a/tests/testsuite/install.rs +++ b/tests/testsuite/install.rs @@ -1014,7 +1014,10 @@ fn dev_dependencies_no_check() { .file("src/main.rs", "fn main() {}") .build(); - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr_contains("[..] no matching package named `baz` found") + .run(); p.cargo("install").run(); } diff --git a/tests/testsuite/path.rs b/tests/testsuite/path.rs index 01c605307c1..8e46a604b23 100644 --- a/tests/testsuite/path.rs +++ b/tests/testsuite/path.rs @@ -138,7 +138,10 @@ fn cargo_compile_with_root_dev_deps() { ) .build(); - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr_contains("[..]can't find crate for `bar`") + .run(); } #[test] diff --git a/tests/testsuite/publish.rs b/tests/testsuite/publish.rs index 05bbdfb6788..57ddd3428ba 100644 --- a/tests/testsuite/publish.rs +++ b/tests/testsuite/publish.rs @@ -597,6 +597,10 @@ fn new_crate_rejected() { p.cargo("publish --index") .arg(publish::registry().to_string()) .with_status(101) + .with_stderr_contains( + "[ERROR] 3 files in the working directory contain \ + changes that were not yet committed into git:", + ) .run(); } diff --git a/tests/testsuite/rename_deps.rs b/tests/testsuite/rename_deps.rs index 8a7f0e264b6..915a94d6ded 100644 --- a/tests/testsuite/rename_deps.rs +++ b/tests/testsuite/rename_deps.rs @@ -228,7 +228,10 @@ fn rename_affects_fingerprint() { "#, ); - p.cargo("build -v").with_status(101).run(); + p.cargo("build -v") + .with_status(101) + .with_stderr_contains("[..]can't find crate for `foo`") + .run(); } #[test] diff --git a/tests/testsuite/rustdocflags.rs b/tests/testsuite/rustdocflags.rs index 11b88f75571..7ac963da710 100644 --- a/tests/testsuite/rustdocflags.rs +++ b/tests/testsuite/rustdocflags.rs @@ -35,6 +35,7 @@ fn bad_flags() { p.cargo("doc") .env("RUSTDOCFLAGS", "--bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); } diff --git a/tests/testsuite/rustflags.rs b/tests/testsuite/rustflags.rs index 796e36be805..84ee109748b 100644 --- a/tests/testsuite/rustflags.rs +++ b/tests/testsuite/rustflags.rs @@ -24,22 +24,27 @@ fn env_rustflags_normal_source() { p.cargo("build --lib") .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); p.cargo("build --bin=a") .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); p.cargo("build --example=b") .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); p.cargo("test") .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); p.cargo("bench") .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); } @@ -202,26 +207,31 @@ fn env_rustflags_normal_source_with_target() { .arg(host) .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); p.cargo("build --bin=a --target") .arg(host) .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); p.cargo("build --example=b --target") .arg(host) .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); p.cargo("test --target") .arg(host) .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); p.cargo("bench --target") .arg(host) .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); } @@ -386,6 +396,7 @@ fn env_rustflags_recompile() { p.cargo("build") .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); } @@ -398,6 +409,7 @@ fn env_rustflags_recompile2() { p.cargo("build") .env("RUSTFLAGS", "-Z bogus") .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); } @@ -435,11 +447,26 @@ fn build_rustflags_normal_source() { ) .build(); - p.cargo("build --lib").with_status(101).run(); - p.cargo("build --bin=a").with_status(101).run(); - p.cargo("build --example=b").with_status(101).run(); - p.cargo("test").with_status(101).run(); - p.cargo("bench").with_status(101).run(); + p.cargo("build --lib") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("build --bin=a") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("build --example=b") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("test") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("bench") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); } #[test] @@ -635,17 +662,28 @@ fn build_rustflags_normal_source_with_target() { p.cargo("build --lib --target") .arg(host) .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); p.cargo("build --bin=a --target") .arg(host) .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); p.cargo("build --example=b --target") .arg(host) .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("test --target") + .arg(host) + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("bench --target") + .arg(host) + .with_status(101) + .with_stderr_contains("[..]bogus[..]") .run(); - p.cargo("test --target").arg(host).with_status(101).run(); - p.cargo("bench --target").arg(host).with_status(101).run(); } #[test] @@ -832,7 +870,10 @@ fn build_rustflags_recompile() { let mut config_file = File::create(config_file).unwrap(); config_file.write_all(config.as_bytes()).unwrap(); - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); } #[test] @@ -851,7 +892,10 @@ fn build_rustflags_recompile2() { let mut config_file = File::create(config_file).unwrap(); config_file.write_all(config.as_bytes()).unwrap(); - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); } #[test] @@ -923,11 +967,26 @@ fn target_rustflags_normal_source() { ) .build(); - p.cargo("build --lib").with_status(101).run(); - p.cargo("build --bin=a").with_status(101).run(); - p.cargo("build --example=b").with_status(101).run(); - p.cargo("test").with_status(101).run(); - p.cargo("bench").with_status(101).run(); + p.cargo("build --lib") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("build --bin=a") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("build --example=b") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("test") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("bench") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); } // target.{}.rustflags takes precedence over build.rustflags @@ -935,6 +994,9 @@ fn target_rustflags_normal_source() { fn target_rustflags_precedence() { let p = project() .file("src/lib.rs", "") + .file("src/bin/a.rs", "fn main() {}") + .file("examples/b.rs", "fn main() {}") + .file("tests/c.rs", "#[test] fn f() { }") .file( ".cargo/config", &format!( @@ -950,11 +1012,26 @@ fn target_rustflags_precedence() { ) .build(); - p.cargo("build --lib").with_status(101).run(); - p.cargo("build --bin=a").with_status(101).run(); - p.cargo("build --example=b").with_status(101).run(); - p.cargo("test").with_status(101).run(); - p.cargo("bench").with_status(101).run(); + p.cargo("build --lib") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("build --bin=a") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("build --example=b") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("test") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); + p.cargo("bench") + .with_status(101) + .with_stderr_contains("[..]bogus[..]") + .run(); } #[test] diff --git a/tests/testsuite/support/mod.rs b/tests/testsuite/support/mod.rs index 7416a21b187..16537f61577 100644 --- a/tests/testsuite/support/mod.rs +++ b/tests/testsuite/support/mod.rs @@ -781,6 +781,32 @@ impl Execs { } } + fn verify_checks_output(&self, output: &Output) { + if self.expect_exit_code.unwrap_or(0) != 0 + && self.expect_stdout.is_none() + && self.expect_stdin.is_none() + && self.expect_stderr.is_none() + && self.expect_stdout_contains.is_empty() + && self.expect_stderr_contains.is_empty() + && self.expect_either_contains.is_empty() + && self.expect_stdout_contains_n.is_empty() + && self.expect_stdout_not_contains.is_empty() + && self.expect_stderr_not_contains.is_empty() + && self.expect_stderr_unordered.is_empty() + && self.expect_neither_contains.is_empty() + && self.expect_json.is_none() + && self.expect_json_contains_unordered.is_empty() + { + panic!( + "`with_status()` is used, but no output is checked.\n\ + The test must check the output to ensure the correct error is triggered.\n\ + --- stdout\n{}\n--- stderr\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + } + } + fn match_process(&self, process: &ProcessBuilder) -> MatchResult { println!("running {}", process); let res = if self.stream_output { @@ -817,6 +843,7 @@ impl Execs { } fn match_output(&self, actual: &Output) -> MatchResult { + self.verify_checks_output(actual); self.match_status(actual) .and(self.match_stdout(actual)) .and(self.match_stderr(actual)) diff --git a/tests/testsuite/workspaces.rs b/tests/testsuite/workspaces.rs index 199b4194283..1063e9837a7 100644 --- a/tests/testsuite/workspaces.rs +++ b/tests/testsuite/workspaces.rs @@ -545,7 +545,12 @@ fn cycle() { .file("bar/src/main.rs", "fn main() {}"); let p = p.build(); - p.cargo("build").with_status(101).run(); + p.cargo("build") + .with_status(101) + .with_stderr( + "[ERROR] root of a workspace inferred but wasn't a root: [..]/foo/bar/Cargo.toml", + ) + .run(); } #[test] @@ -1082,6 +1087,7 @@ fn rebuild_please() { p.cargo("run") .cwd(p.root().join("bin")) .with_status(101) + .with_stderr_contains("[..]assertion[..]") .run(); }