From 54f80a5b7c254d1df302680eac6552665b4c8945 Mon Sep 17 00:00:00 2001 From: nathanwhit Date: Thu, 22 Aug 2019 12:01:37 -0400 Subject: [PATCH 1/7] Add header flag to check run-pass test output --- src/tools/compiletest/src/header.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs index 076ad87c70fc4..dcd4f14f354d1 100644 --- a/src/tools/compiletest/src/header.rs +++ b/src/tools/compiletest/src/header.rs @@ -326,6 +326,8 @@ pub struct TestProps { pub force_host: bool, // Check stdout for error-pattern output as well as stderr pub check_stdout: bool, + // Check stdout & stderr for output of run-pass test + pub check_run_results: bool, // For UI tests, allows compiler to generate arbitrary output to stdout pub dont_check_compiler_stdout: bool, // For UI tests, allows compiler to generate arbitrary output to stderr @@ -388,6 +390,7 @@ impl TestProps { build_aux_docs: false, force_host: false, check_stdout: false, + check_run_results: false, dont_check_compiler_stdout: false, dont_check_compiler_stderr: false, no_prefer_dynamic: false, @@ -468,6 +471,10 @@ impl TestProps { self.check_stdout = config.parse_check_stdout(ln); } + if !self.check_run_results { + self.check_run_results = config.parse_check_run_results(ln); + } + if !self.dont_check_compiler_stdout { self.dont_check_compiler_stdout = config.parse_dont_check_compiler_stdout(ln); } @@ -712,6 +719,10 @@ impl Config { self.parse_name_directive(line, "check-stdout") } + fn parse_check_run_results(&self, line: &str) -> bool { + self.parse_name_directive(line, "check-run-results") + } + fn parse_dont_check_compiler_stdout(&self, line: &str) -> bool { self.parse_name_directive(line, "dont-check-compiler-stdout") } From 0c379dff2919321944440cfe7c306c2d35bffaaa Mon Sep 17 00:00:00 2001 From: Nathan Date: Thu, 22 Aug 2019 19:02:13 -0400 Subject: [PATCH 2/7] Extract loading and comparison of compiletest outputs to new fn Moved into ```fn load_compare_outputs(&self, proc_res: &ProcRes, explicit_format: bool) -> usize``` --- src/tools/compiletest/src/runtest.rs | 35 ++++++++++++++++------------ 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 3da6be74129f4..86c5e4ac89dd6 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -2934,33 +2934,20 @@ impl<'test> TestCx<'test> { } } - fn run_ui_test(&self) { - // if the user specified a format in the ui test - // print the output to the stderr file, otherwise extract - // the rendered error messages from json and print them - let explicit = self - .props - .compile_flags - .iter() - .any(|s| s.contains("--error-format")); - let proc_res = self.compile_test(); - self.check_if_test_should_compile(&proc_res); - + fn load_compare_outputs(&self, proc_res: &ProcRes, explicit_format: bool) -> usize { let expected_stderr = self.load_expected_output(UI_STDERR); let expected_stdout = self.load_expected_output(UI_STDOUT); - let expected_fixed = self.load_expected_output(UI_FIXED); let normalized_stdout = self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout); - let stderr = if explicit { + let stderr = if explicit_format { proc_res.stderr.clone() } else { json::extract_rendered(&proc_res.stderr) }; let normalized_stderr = self.normalize_output(&stderr, &self.props.normalize_stderr); - let mut errors = 0; if !self.props.dont_check_compiler_stdout { errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout); @@ -2968,10 +2955,28 @@ impl<'test> TestCx<'test> { if !self.props.dont_check_compiler_stderr { errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr); } + errors + } + + fn run_ui_test(&self) { + // if the user specified a format in the ui test + // print the output to the stderr file, otherwise extract + // the rendered error messages from json and print them + let explicit = self + .props + .compile_flags + .iter() + .any(|s| s.contains("--error-format")); + let proc_res = self.compile_test(); + self.check_if_test_should_compile(&proc_res); + + let expected_fixed = self.load_expected_output(UI_FIXED); let modes_to_prune = vec![CompareMode::Nll]; self.prune_duplicate_outputs(&modes_to_prune); + let mut errors = self.load_compare_outputs(&proc_res, explicit); + if self.config.compare_mode.is_some() { // don't test rustfix with nll right now } else if self.config.rustfix_coverage { From b9ba8f9596ac282b3254edbb9a7176771fb6ba24 Mon Sep 17 00:00:00 2001 From: Nathan Date: Thu, 22 Aug 2019 19:08:51 -0400 Subject: [PATCH 3/7] Compare run-pass outputs if `check-run-results` flag enabled Addresses #63751 --- src/tools/compiletest/src/runtest.rs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 86c5e4ac89dd6..c1f8caa474e2d 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -2975,7 +2975,14 @@ impl<'test> TestCx<'test> { let modes_to_prune = vec![CompareMode::Nll]; self.prune_duplicate_outputs(&modes_to_prune); - let mut errors = self.load_compare_outputs(&proc_res, explicit); + // if the user specified to check the results of the + // run-pass test, delay loading and comparing output + // until execution of the binary + let mut errors = if !self.props.check_run_results { + self.load_compare_outputs(&proc_res, explicit) + } else { + 0 + }; if self.config.compare_mode.is_some() { // don't test rustfix with nll right now @@ -3054,7 +3061,17 @@ impl<'test> TestCx<'test> { if self.should_run_successfully() { let proc_res = self.exec_compiled_test(); - + let run_output_errors = if self.props.check_run_results { + self.load_compare_outputs(&proc_res, explicit) + } else { + 0 + }; + if run_output_errors > 0 { + self.fatal_proc_rec( + &format!("{} errors occured comparing run output.", run_output_errors), + &proc_res, + ); + } if !proc_res.status.success() { self.fatal_proc_rec("test run failed!", &proc_res); } From 0e3bc62f399b262bb56eb3de146a85729cdb882c Mon Sep 17 00:00:00 2001 From: Nathan Date: Sun, 25 Aug 2019 19:17:32 -0400 Subject: [PATCH 4/7] Allow checking of both compile & run output for run-pass tests --- src/tools/compiletest/src/common.rs | 4 ++- src/tools/compiletest/src/runtest.rs | 48 ++++++++++++++++++---------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index 66e030e979355..edb9eb7d860e2 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -333,10 +333,12 @@ pub fn expected_output_path( testpaths.file.with_extension(extension) } -pub const UI_EXTENSIONS: &[&str] = &[UI_STDERR, UI_STDOUT, UI_FIXED]; +pub const UI_EXTENSIONS: &[&str] = &[UI_STDERR, UI_STDOUT, UI_FIXED, UI_RUN_STDERR, UI_RUN_STDOUT]; pub const UI_STDERR: &str = "stderr"; pub const UI_STDOUT: &str = "stdout"; pub const UI_FIXED: &str = "fixed"; +pub const UI_RUN_STDERR: &str = "run.stderr"; +pub const UI_RUN_STDOUT: &str = "run.stdout"; /// Absolute path to the directory where all output for all tests in the given /// `relative_dir` group should reside. Example: diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index c1f8caa474e2d..7b5e0752b3673 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -2,6 +2,7 @@ use crate::common::{CompareMode, PassMode}; use crate::common::{expected_output_path, UI_EXTENSIONS, UI_FIXED, UI_STDERR, UI_STDOUT}; +use crate::common::{UI_RUN_STDERR, UI_RUN_STDOUT}; use crate::common::{output_base_dir, output_base_name, output_testname_unique}; use crate::common::{Codegen, CodegenUnits, Rustdoc}; use crate::common::{DebugInfoCdb, DebugInfoGdbLldb, DebugInfoGdb, DebugInfoLldb}; @@ -288,6 +289,11 @@ enum ReadFrom { Stdin(String), } +enum TestOutput { + Compile, + Run, +} + impl<'test> TestCx<'test> { /// Code executed for each revision in turn (or, if there are no /// revisions, exactly once, with revision == None). @@ -2934,9 +2940,16 @@ impl<'test> TestCx<'test> { } } - fn load_compare_outputs(&self, proc_res: &ProcRes, explicit_format: bool) -> usize { - let expected_stderr = self.load_expected_output(UI_STDERR); - let expected_stdout = self.load_expected_output(UI_STDOUT); + fn load_compare_outputs(&self, proc_res: &ProcRes, + output_kind: TestOutput, explicit_format: bool) -> usize { + + let (stderr_kind, stdout_kind) = match output_kind { + TestOutput::Compile => (UI_STDERR, UI_STDOUT), + TestOutput::Run => (UI_RUN_STDERR, UI_RUN_STDOUT) + }; + + let expected_stderr = self.load_expected_output(stderr_kind); + let expected_stdout = self.load_expected_output(stdout_kind); let normalized_stdout = self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout); @@ -2949,11 +2962,19 @@ impl<'test> TestCx<'test> { let normalized_stderr = self.normalize_output(&stderr, &self.props.normalize_stderr); let mut errors = 0; - if !self.props.dont_check_compiler_stdout { - errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout); - } - if !self.props.dont_check_compiler_stderr { - errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr); + match output_kind { + TestOutput::Compile => { + if !self.props.dont_check_compiler_stdout { + errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout); + } + if !self.props.dont_check_compiler_stderr { + errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr); + } + } + TestOutput::Run => { + errors += self.compare_output(stdout_kind, &normalized_stdout, &expected_stdout); + errors += self.compare_output(stderr_kind, &normalized_stderr, &expected_stderr); + } } errors } @@ -2975,14 +2996,7 @@ impl<'test> TestCx<'test> { let modes_to_prune = vec![CompareMode::Nll]; self.prune_duplicate_outputs(&modes_to_prune); - // if the user specified to check the results of the - // run-pass test, delay loading and comparing output - // until execution of the binary - let mut errors = if !self.props.check_run_results { - self.load_compare_outputs(&proc_res, explicit) - } else { - 0 - }; + let mut errors = self.load_compare_outputs(&proc_res, TestOutput::Compile, explicit); if self.config.compare_mode.is_some() { // don't test rustfix with nll right now @@ -3062,7 +3076,7 @@ impl<'test> TestCx<'test> { if self.should_run_successfully() { let proc_res = self.exec_compiled_test(); let run_output_errors = if self.props.check_run_results { - self.load_compare_outputs(&proc_res, explicit) + self.load_compare_outputs(&proc_res, TestOutput::Run, explicit) } else { 0 }; From 12e0420268a3855eea981bb8907a8bbfb1cb9d9a Mon Sep 17 00:00:00 2001 From: Nathan Date: Sun, 25 Aug 2019 21:10:33 -0400 Subject: [PATCH 5/7] Update dbg macro test to use `check-run-results` flag for run-pass test Updates the dbg-macro-expected behavior test to remove the workaround and use the `check-run-result` flag/feature in compiletest. This serves to test the feature on a real use-case (as mentioned in #63751) --- .../dbg-macro-expected-behavior.rs | 84 +------------------ .../dbg-macro-expected-behavior.run.stderr | 28 +++++++ 2 files changed, 30 insertions(+), 82 deletions(-) create mode 100644 src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.run.stderr diff --git a/src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.rs b/src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.rs index e885263aa8084..04d924a9aed20 100644 --- a/src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.rs +++ b/src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.rs @@ -1,7 +1,5 @@ // run-pass -// ignore-cloudabi no processes -// ignore-emscripten no processes -// ignore-sgx no processes +// check-run-results // Tests ensuring that `dbg!(expr)` has the expected run-time behavior. // as well as some compile time properties we expect. @@ -18,7 +16,7 @@ struct Point { #[derive(Debug, PartialEq)] struct NoCopy(usize); -fn test() { +fn main() { let a: Unit = dbg!(Unit); let _: Unit = dbg!(a); // We can move `a` because it's Copy. @@ -67,81 +65,3 @@ fn test() { assert_eq!((1u8, 2u32, "Yeah"), dbg!(1u8, 2u32, "Yeah",)); } - -fn validate_stderr(stderr: Vec) { - assert_eq!(stderr, &[ - ":22] Unit = Unit", - - ":23] a = Unit", - - ":29] Point{x: 42, y: 24,} = Point {", - " x: 42,", - " y: 24,", - "}", - - ":30] b = Point {", - " x: 42,", - " y: 24,", - "}", - - ":38]", - - ":42] &a = NoCopy(", - " 1337,", - ")", - - ":42] dbg!(& a) = NoCopy(", - " 1337,", - ")", - ":47] f(&42) = 42", - - "before", - ":52] { foo += 1; eprintln!(\"before\"); 7331 } = 7331", - - ":60] (\"Yeah\",) = (", - " \"Yeah\",", - ")", - - ":63] 1 = 1", - ":63] 2 = 2", - - ":67] 1u8 = 1", - ":67] 2u32 = 2", - ":67] \"Yeah\" = \"Yeah\"", - ]); -} - -fn main() { - // The following is a hack to deal with compiletest's inability - // to check the output (to stdout) of run-pass tests. - use std::env; - use std::process::Command; - - let mut args = env::args(); - let prog = args.next().unwrap(); - let child = args.next(); - if let Some("child") = child.as_ref().map(|s| &**s) { - // Only run the test if we've been spawned as 'child' - test() - } else { - // This essentially spawns as 'child' to run the tests - // and then it collects output of stderr and checks the output - // against what we expect. - let out = Command::new(&prog).arg("child").output().unwrap(); - assert!(out.status.success()); - assert!(out.stdout.is_empty()); - - let stderr = String::from_utf8(out.stderr).unwrap(); - let stderr = stderr.lines().map(|mut s| { - if s.starts_with("[") { - // Strip `[` and file path: - s = s.trim_start_matches("["); - assert!(s.starts_with(file!())); - s = s.trim_start_matches(file!()); - } - s.to_owned() - }).collect(); - - validate_stderr(stderr); - } -} diff --git a/src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.run.stderr b/src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.run.stderr new file mode 100644 index 0000000000000..707b38cf37af0 --- /dev/null +++ b/src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.run.stderr @@ -0,0 +1,28 @@ +[$DIR/dbg-macro-expected-behavior.rs:20] Unit = Unit +[$DIR/dbg-macro-expected-behavior.rs:21] a = Unit +[$DIR/dbg-macro-expected-behavior.rs:27] Point{x: 42, y: 24,} = Point { + x: 42, + y: 24, +} +[$DIR/dbg-macro-expected-behavior.rs:28] b = Point { + x: 42, + y: 24, +} +[$DIR/dbg-macro-expected-behavior.rs:36] +[$DIR/dbg-macro-expected-behavior.rs:40] &a = NoCopy( + 1337, +) +[$DIR/dbg-macro-expected-behavior.rs:40] dbg!(& a) = NoCopy( + 1337, +) +[$DIR/dbg-macro-expected-behavior.rs:45] f(&42) = 42 +before +[$DIR/dbg-macro-expected-behavior.rs:50] { foo += 1; eprintln!("before"); 7331 } = 7331 +[$DIR/dbg-macro-expected-behavior.rs:58] ("Yeah",) = ( + "Yeah", +) +[$DIR/dbg-macro-expected-behavior.rs:61] 1 = 1 +[$DIR/dbg-macro-expected-behavior.rs:61] 2 = 2 +[$DIR/dbg-macro-expected-behavior.rs:65] 1u8 = 1 +[$DIR/dbg-macro-expected-behavior.rs:65] 2u32 = 2 +[$DIR/dbg-macro-expected-behavior.rs:65] "Yeah" = "Yeah" From bf44f12d66b011bf14c4bbe4a652a853ef3e52cb Mon Sep 17 00:00:00 2001 From: nathanwhit Date: Mon, 2 Sep 2019 12:00:55 -0400 Subject: [PATCH 6/7] Ignore check-run-results tests for wasm32-bare Ignores run-pass tests with the `check-run-results` flag enabled for the wasm32-bare ("wasm32-unknown-unknown") target, as it does not support printing to stdout/stderr. --- src/tools/compiletest/src/header.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs index dcd4f14f354d1..3ba8cffe2b559 100644 --- a/src/tools/compiletest/src/header.rs +++ b/src/tools/compiletest/src/header.rs @@ -137,6 +137,11 @@ impl EarlyProps { config.parse_needs_sanitizer_support(ln) { props.ignore = Ignore::Ignore; } + + if config.target == "wasm32-unknown-unknown" && config.parse_check_run_results(ln) { + props.ignore = Ignore::Ignore; + } + } if (config.mode == common::DebugInfoGdb || config.mode == common::DebugInfoGdbLldb) && From 12adc395c375d4ab14d24624a0ccdd519d5a5978 Mon Sep 17 00:00:00 2001 From: nathanwhit Date: Tue, 3 Sep 2019 11:56:08 -0400 Subject: [PATCH 7/7] Strip remote-test-client output from run stdout The remote-test-client outputs a message of the form "uploaded "/", waiting for result" onto stdout when executing a test, which is then captured in the process result. This needs to be removed when comparing the results of the run-pass test execution. --- src/tools/compiletest/src/runtest.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 7b5e0752b3673..8fb1302936c66 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -2951,8 +2951,24 @@ impl<'test> TestCx<'test> { let expected_stderr = self.load_expected_output(stderr_kind); let expected_stdout = self.load_expected_output(stdout_kind); - let normalized_stdout = - self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout); + let normalized_stdout = match output_kind { + TestOutput::Run if self.config.remote_test_client.is_some() => { + // When tests are run using the remote-test-client, the string + // 'uploaded "$TEST_BUILD_DIR/, waiting for result"' + // is printed to stdout by the client and then captured in the ProcRes, + // so it needs to be removed when comparing the run-pass test execution output + lazy_static! { + static ref REMOTE_TEST_RE: Regex = Regex::new( + "^uploaded \"\\$TEST_BUILD_DIR(/[[:alnum:]_\\-]+)+\", waiting for result\n" + ).unwrap(); + } + REMOTE_TEST_RE.replace( + &self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout), + "" + ).to_string() + } + _ => self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout) + }; let stderr = if explicit_format { proc_res.stderr.clone()