Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for TeamCity Test history via Teamcity Service Messages #44813

Closed
wants to merge 3 commits into from
Closed

Support for TeamCity Test history via Teamcity Service Messages #44813

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1211c5d
Implementation of Teamcity service messages to allow easy integration…
gilescope Sep 22, 2017
6ce7552
Added TeamCity service messages
gilescope Sep 24, 2017
05db1cb
Removing debugging statements.
gilescope Sep 24, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 135 additions & 38 deletions src/libtest/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,12 +258,14 @@ impl Clone for MetricMap {
#[derive(Copy, Clone, Debug)]
pub struct Options {
display_output: bool,
display_service_messages: bool
}

impl Options {
pub fn new() -> Options {
Options {
display_output: false,
display_service_messages: false
}
}

Expand All @@ -276,12 +278,11 @@ impl Options {
// The default console test runner. It accepts the command line
// arguments and a vector of test_descs.
pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Options) {
let mut opts = match parse_opts(args) {
let opts = match parse_opts(args, options) {
Some(Ok(o)) => o,
Some(Err(msg)) => panic!("{:?}", msg),
None => return,
};
opts.options = options;
if opts.list {
if let Err(e) = list_tests_console(&opts, tests) {
panic!("io error when listing tests: {:?}", e);
Expand Down Expand Up @@ -385,6 +386,7 @@ fn optgroups() -> getopts::Options {
of stdout", "PATH")
.optflag("", "nocapture", "don't capture stdout/stderr of each \
task, allow printing directly")
.optflag("", "teamcity", "Print Teamcity Service Messages to stdout to track test history.")
.optopt("", "test-threads", "Number of threads used for running tests \
in parallel", "n_threads")
.optmulti("", "skip", "Skip tests whose names contain FILTER (this flag can \
Expand Down Expand Up @@ -431,7 +433,7 @@ Test Attributes:
}

// Parses command line arguments into test options
pub fn parse_opts(args: &[String]) -> Option<OptRes> {
pub fn parse_opts(args: &[String], mut options: Options) -> Option<OptRes> {
let opts = optgroups();
let matches = match opts.parse(&args[1..]) {
Ok(m) => m,
Expand Down Expand Up @@ -468,6 +470,14 @@ pub fn parse_opts(args: &[String]) -> Option<OptRes> {
};
}

let mut teamcity = matches.opt_present("teamcity");
if !teamcity {
teamcity = match env::var("RUST_TEST_TEAMCITY_SERVICE_MESSAGES") {
Ok(val) => &val != "0",
Err(_) => false
};
}

let test_threads = match matches.opt_str("test-threads") {
Some(n_str) =>
match n_str.parse::<usize>() {
Expand All @@ -494,6 +504,10 @@ pub fn parse_opts(args: &[String]) -> Option<OptRes> {
}
};

if teamcity {
options.display_service_messages = true
}

let test_opts = TestOpts {
list,
filter,
Expand All @@ -507,7 +521,7 @@ pub fn parse_opts(args: &[String]) -> Option<OptRes> {
quiet,
test_threads,
skip: matches.opt_strs("skip"),
options: Options::new(),
options: options
};

Some(Ok(test_opts))
Expand Down Expand Up @@ -587,16 +601,33 @@ impl<T: Write> ConsoleTestState<T> {
})
}

pub fn write_ok(&mut self) -> io::Result<()> {
self.write_short_result("ok", ".", term::color::GREEN)
pub fn write_ok(&mut self, test: &TestDesc) -> io::Result<()> {
self.write_short_result("ok", ".", term::color::GREEN)?;

let name : String = service_message_escape(test.name.as_slice());
if self.options.display_service_messages {
self.write_plain(&format!("\n##teamcity[testFinished name='{}']\n", name))?;
}
std::result::Result::Ok(())
}

pub fn write_failed(&mut self) -> io::Result<()> {
self.write_short_result("FAILED", "F", term::color::RED)
pub fn write_failed(&mut self, test: &TestDesc) -> io::Result<()> {

self.write_short_result("FAILED", "F", term::color::RED)?;
if self.options.display_service_messages {
self.write_plain(&format!("\n##teamcity[testFailed name='{}']\n",
service_message_escape(test.name.as_slice())))?;
}
std::result::Result::Ok(())
}

pub fn write_ignored(&mut self) -> io::Result<()> {
self.write_short_result("ignored", "i", term::color::YELLOW)
pub fn write_ignored(&mut self, test: &TestDesc) -> io::Result<()> {
self.write_short_result("ignored", "i", term::color::YELLOW)?;
if self.options.display_service_messages {
self.write_plain(&format!("\n##teamcity[testIgnored name='{}']\n",
service_message_escape(test.name.as_slice())))?;
}
std::result::Result::Ok(())
}

pub fn write_allowed_fail(&mut self) -> io::Result<()> {
Expand Down Expand Up @@ -669,28 +700,43 @@ impl<T: Write> ConsoleTestState<T> {
Ok(())
} else {
let name = test.padded_name(self.max_name_len, align);
if self.options.display_service_messages {
let use_std_out = if stdout_isatty() {
"true"
} else {
"false"
};
self.write_plain(
&format!("\n##teamcity[testStarted name='{}' captureStandardOutput='{}']\n",
service_message_escape(test.name.as_slice()), use_std_out))?;
}
self.write_plain(&format!("test {} ... ", name))
}
}

pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> {
pub fn write_result(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> {
match *result {
TrOk => self.write_ok(),
TrFailed | TrFailedMsg(_) => self.write_failed(),
TrIgnored => self.write_ignored(),
TrOk => self.write_ok(&test),
TrFailed | TrFailedMsg(_) => self.write_failed(&test),
TrIgnored => self.write_ignored(&test),
TrAllowedFail => self.write_allowed_fail(),
TrMetrics(ref mm) => {
self.write_metric()?;
self.write_plain(&format!(": {}\n", mm.fmt_metrics()))
}
TrBench(ref bs) => {
self.write_bench()?;
self.write_plain(&format!(": {}\n", fmt_bench_samples(bs)))
let msg = self.fmt_bench_samples(test, bs).clone();
self.write_plain(&format!(": {}\n", msg))
}
}
}

pub fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> {
if self.options.display_service_messages {
self.write_plain(&format!("\n##teamcity[testFailed name='{}']\n",
service_message_escape(desc.name.as_slice())))?;
}
self.write_plain(&format!("test {} has been running for over {} seconds\n",
desc.name,
TEST_WARN_TIMEOUT_S))
Expand All @@ -705,18 +751,54 @@ impl<T: Write> ConsoleTestState<T> {
}

pub fn write_log_result(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> {
self.write_log(
format!("{} {}\n",
let msg =format!("{} {}\n",
match *result {
TrOk => "ok".to_owned(),
TrFailed => "failed".to_owned(),
TrFailedMsg(ref msg) => format!("failed: {}", msg),
TrIgnored => "ignored".to_owned(),
TrAllowedFail => "failed (allowed)".to_owned(),
TrMetrics(ref mm) => mm.fmt_metrics(),
TrBench(ref bs) => fmt_bench_samples(bs),
TrBench(ref bs) => self.fmt_bench_samples(test, bs),
},
test.name))
test.name);
self.write_log(msg)
}

pub fn fmt_bench_samples(&self, test: &TestDesc, bs: &BenchSamples) -> String {
use std::fmt::Write;
let mut output = String::new();

let median = bs.ns_iter_summ.median as usize;
let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;

output.write_fmt(format_args!("{:>11} ns/iter (+/- {})",
fmt_thousands_sep(median, ','),
fmt_thousands_sep(deviation, ','))).unwrap();

if bs.mb_s != 0 {
output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap();

if self.options.display_service_messages {
output.write_fmt(
format_args!("##teamcity[buildStatisticValue key='{}_MB_s' value='{}']\n",
test.name.as_slice(), bs.mb_s)).unwrap();
}
}
if self.options.display_service_messages {
output.write_fmt(
format_args!(
"\n##teamcity[buildStatisticValue key='{}_ns_iter_median' value='{}']\n",
test.name.as_slice(), median)).unwrap();

output.write_fmt(
format_args!("##teamcity[buildStatisticValue key='{}_std_dev' value='{}']\n",
test.name.as_slice(), deviation)).unwrap();
}
output.write_fmt(
format_args!("##teamcity[testFinished name='{}']\n",
test.name.as_slice())).unwrap();
output
}

pub fn write_failures(&mut self) -> io::Result<()> {
Expand Down Expand Up @@ -813,6 +895,17 @@ impl<T: Write> ConsoleTestState<T> {
}
}

// Follow TeamCity escaping rules for service messages.
fn service_message_escape(message: &str) -> String {
message.replace('|', "||")
.replace('\'', "|'")
.replace('\n', "|n")
.replace('\r', "|r")
.replace('[', "|[")
.replace(']', "|]")
//FIXME \uNNNN (unicode symbol with code 0xNNNN) to "|0xNNNN'
}

// Format a number with thousands separators
fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
use std::fmt::Write;
Expand All @@ -837,22 +930,6 @@ fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
output
}

pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
use std::fmt::Write;
let mut output = String::new();

let median = bs.ns_iter_summ.median as usize;
let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;

output.write_fmt(format_args!("{:>11} ns/iter (+/- {})",
fmt_thousands_sep(median, ','),
fmt_thousands_sep(deviation, ',')))
.unwrap();
if bs.mb_s != 0 {
output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap();
}
output
}

// List the tests to console, and optionally to logfile. Filters are honored.
pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> {
Expand Down Expand Up @@ -908,7 +985,7 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Resu
TeTimeout(ref test) => st.write_timeout(test),
TeResult(test, result, stdout) => {
st.write_log_result(&test, &result)?;
st.write_result(&result)?;
st.write_result(&test, &result)?;
match result {
TrOk => {
st.passed += 1;
Expand Down Expand Up @@ -1850,11 +1927,31 @@ mod tests {
#[test]
fn parse_ignored_flag() {
let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
let opts = match parse_opts(&args) {
let opts = match parse_opts(&args, TestOpts::new().options) {
Some(Ok(o)) => o,
_ => panic!("Malformed arg in parse_ignored_flag"),
};
assert!(opts.run_ignored);
}

#[test]
fn parse_teamcity_flag() {
let args = vec!["progname".to_string(), "filter".to_string(), "--teamcity".to_string()];
let opts = match parse_opts(&args, TestOpts::new().options) {
Some(Ok(o)) => o,
_ => panic!("Malformed arg in parse_ignored_flag"),
};
assert!(opts.options.display_service_messages);
}

#[test]
fn teamcity_service_messages_defaults_to_off() {
let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
let opts = match parse_opts(&args, TestOpts::new().options) {
Some(Ok(o)) => o,
_ => panic!("Malformed arg in parse_ignored_flag"),
};
assert!((opts.run_ignored));
assert!(!opts.options.display_service_messages);
}

#[test]
Expand Down