Auto merge of #108148 - parthopdas:master, r=oli-obk

Implementing "<test_binary> --list --format json" for use by IDE test explorers / runners

Fixes #107307

PR 1 of 2 - wiring up just the new information + implement the command line changes i.e. --format json + tests

upcoming:
PR 2 of 2 - clean up "#[cfg(not(bootstrap))]" from PR 1

As per the discussions on
- MCP: https://rust-lang.zulipchat.com/#narrow/stream/233931-t-compiler.2Fmajor-changes/topic/Implementing.20.22.3Ctest_binary.3E.20--list.20--form.E2.80.A6.20compiler-team.23592/near/328747548
- preRFC: https://internals.rust-lang.org/t/pre-rfc-implementing-test-binary-list-format-json-for-use-by-ide-test-explorers-runners/18308
- FYI on Discord: https://discord.com/channels/442252698964721669/459149169546887178/1075581549409484820

Author: bors

Diff for: .gitignore

@@ -21,6 +21,7 @@ Session.vim
 .project
 .favorites.json
 .settings/
+.vs/
 
 ## Tool
 .valgrindrc

Diff for: compiler/rustc_builtin_macros/src/test.rs

@@ -8,7 +8,7 @@ use rustc_errors::Applicability;
 use rustc_expand::base::*;
 use rustc_session::Session;
 use rustc_span::symbol::{sym, Ident, Symbol};
-use rustc_span::Span;
+use rustc_span::{FileNameDisplayPreference, Span};
 use std::iter;
 use thin_vec::{thin_vec, ThinVec};
 
@@ -231,6 +231,8 @@ pub fn expand_test_or_bench(
         &item.ident,
     ));
 
+    let location_info = get_location_info(cx, &item);
+
     let mut test_const = cx.item(
         sp,
         Ident::new(item.ident.name, sp),
@@ -280,6 +282,16 @@ pub fn expand_test_or_bench(
                                             cx.expr_none(sp)
                                         },
                                     ),
+                                    // source_file: <relative_path_of_source_file>
+                                    field("source_file", cx.expr_str(sp, location_info.0)),
+                                    // start_line: start line of the test fn identifier.
+                                    field("start_line", cx.expr_usize(sp, location_info.1)),
+                                    // start_col: start column of the test fn identifier.
+                                    field("start_col", cx.expr_usize(sp, location_info.2)),
+                                    // end_line: end line of the test fn identifier.
+                                    field("end_line", cx.expr_usize(sp, location_info.3)),
+                                    // end_col: end column of the test fn identifier.
+                                    field("end_col", cx.expr_usize(sp, location_info.4)),
                                     // compile_fail: true | false
                                     field("compile_fail", cx.expr_bool(sp, false)),
                                     // no_run: true | false
@@ -364,6 +376,19 @@ pub fn expand_test_or_bench(
     }
 }
 
+fn get_location_info(cx: &ExtCtxt<'_>, item: &ast::Item) -> (Symbol, usize, usize, usize, usize) {
+    let span = item.ident.span;
+    let (source_file, lo_line, lo_col, hi_line, hi_col) =
+        cx.sess.source_map().span_to_location_info(span);
+
+    let file_name = match source_file {
+        Some(sf) => sf.name.display(FileNameDisplayPreference::Remapped).to_string(),
+        None => "no-location".to_string(),
+    };
+
+    (Symbol::intern(&file_name), lo_line, lo_col, hi_line, hi_col)
+}
+
 fn item_path(mod_path: &[Ident], item_ident: &Ident) -> String {
     mod_path
         .iter()

Diff for: compiler/rustc_span/src/source_map.rs

@@ -448,25 +448,36 @@ impl SourceMap {
         sp: Span,
         filename_display_pref: FileNameDisplayPreference,
     ) -> String {
-        if self.files.borrow().source_files.is_empty() || sp.is_dummy() {
-            return "no-location".to_string();
-        }
+        let (source_file, lo_line, lo_col, hi_line, hi_col) = self.span_to_location_info(sp);
+
+        let file_name = match source_file {
+            Some(sf) => sf.name.display(filename_display_pref).to_string(),
+            None => return "no-location".to_string(),
+        };
 
-        let lo = self.lookup_char_pos(sp.lo());
-        let hi = self.lookup_char_pos(sp.hi());
         format!(
-            "{}:{}:{}{}",
-            lo.file.name.display(filename_display_pref),
-            lo.line,
-            lo.col.to_usize() + 1,
+            "{file_name}:{lo_line}:{lo_col}{}",
             if let FileNameDisplayPreference::Short = filename_display_pref {
                 String::new()
             } else {
-                format!(": {}:{}", hi.line, hi.col.to_usize() + 1)
+                format!(": {hi_line}:{hi_col}")
             }
         )
     }
 
+    pub fn span_to_location_info(
+        &self,
+        sp: Span,
+    ) -> (Option<Lrc<SourceFile>>, usize, usize, usize, usize) {
+        if self.files.borrow().source_files.is_empty() || sp.is_dummy() {
+            return (None, 0, 0, 0, 0);
+        }
+
+        let lo = self.lookup_char_pos(sp.lo());
+        let hi = self.lookup_char_pos(sp.hi());
+        (Some(lo.file), lo.line, lo.col.to_usize() + 1, hi.line, hi.col.to_usize() + 1)
+    }
+
     /// Format the span location suitable for embedding in build artifacts
     pub fn span_to_embeddable_string(&self, sp: Span) -> String {
         self.span_to_string(sp, FileNameDisplayPreference::Remapped)

Diff for: library/test/src/console.rs

@@ -41,6 +41,46 @@ impl<T: Write> Write for OutputLocation<T> {
     }
 }
 
+pub struct ConsoleTestDiscoveryState {
+    pub log_out: Option<File>,
+    pub tests: usize,
+    pub benchmarks: usize,
+    pub ignored: usize,
+    pub options: Options,
+}
+
+impl ConsoleTestDiscoveryState {
+    pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestDiscoveryState> {
+        let log_out = match opts.logfile {
+            Some(ref path) => Some(File::create(path)?),
+            None => None,
+        };
+
+        Ok(ConsoleTestDiscoveryState {
+            log_out,
+            tests: 0,
+            benchmarks: 0,
+            ignored: 0,
+            options: opts.options,
+        })
+    }
+
+    pub fn write_log<F, S>(&mut self, msg: F) -> io::Result<()>
+    where
+        S: AsRef<str>,
+        F: FnOnce() -> S,
+    {
+        match self.log_out {
+            None => Ok(()),
+            Some(ref mut o) => {
+                let msg = msg();
+                let msg = msg.as_ref();
+                o.write_all(msg.as_bytes())
+            }
+        }
+    }
+}
+
 pub struct ConsoleTestState {
     pub log_out: Option<File>,
     pub total: usize,
@@ -138,53 +178,44 @@ impl ConsoleTestState {
 
 // List the tests to console, and optionally to logfile. Filters are honored.
 pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> {
-    let mut output = match term::stdout() {
+    let output = match term::stdout() {
         None => OutputLocation::Raw(io::stdout().lock()),
         Some(t) => OutputLocation::Pretty(t),
     };
 
-    let quiet = opts.format == OutputFormat::Terse;
-    let mut st = ConsoleTestState::new(opts)?;
-
-    let mut ntest = 0;
-    let mut nbench = 0;
+    let mut out: Box<dyn OutputFormatter> = match opts.format {
+        OutputFormat::Pretty | OutputFormat::Junit => {
+            Box::new(PrettyFormatter::new(output, false, 0, false, None))
+        }
+        OutputFormat::Terse => Box::new(TerseFormatter::new(output, false, 0, false)),
+        OutputFormat::Json => Box::new(JsonFormatter::new(output)),
+    };
+    let mut st = ConsoleTestDiscoveryState::new(opts)?;
 
+    out.write_discovery_start()?;
     for test in filter_tests(opts, tests).into_iter() {
         use crate::TestFn::*;
 
-        let TestDescAndFn { desc: TestDesc { name, .. }, testfn } = test;
+        let TestDescAndFn { desc, testfn } = test;
 
         let fntype = match testfn {
             StaticTestFn(..) | DynTestFn(..) => {
-                ntest += 1;
+                st.tests += 1;
                 "test"
             }
             StaticBenchFn(..) | DynBenchFn(..) => {
-                nbench += 1;
+                st.benchmarks += 1;
                 "benchmark"
             }
         };
 
-        writeln!(output, "{name}: {fntype}")?;
-        st.write_log(|| format!("{fntype} {name}\n"))?;
-    }
+        st.ignored += if desc.ignore { 1 } else { 0 };
 
-    fn plural(count: u32, s: &str) -> String {
-        match count {
-            1 => format!("1 {s}"),
-            n => format!("{n} {s}s"),
-        }
+        out.write_test_discovered(&desc, fntype)?;
+        st.write_log(|| format!("{fntype} {}\n", desc.name))?;
     }
 
-    if !quiet {
-        if ntest != 0 || nbench != 0 {
-            writeln!(output)?;
-        }
-
-        writeln!(output, "{}, {}", plural(ntest, "test"), plural(nbench, "benchmark"))?;
-    }
-
-    Ok(())
+    out.write_discovery_finish(&st)
 }
 
 // Updates `ConsoleTestState` depending on result of the test execution.

Diff for: library/test/src/formatters/json.rs

@@ -2,7 +2,7 @@ use std::{borrow::Cow, io, io::prelude::Write};
 
 use super::OutputFormatter;
 use crate::{
-    console::{ConsoleTestState, OutputLocation},
+    console::{ConsoleTestDiscoveryState, ConsoleTestState, OutputLocation},
     test_result::TestResult,
     time,
     types::TestDesc,
@@ -60,6 +60,56 @@ impl<T: Write> JsonFormatter<T> {
 }
 
 impl<T: Write> OutputFormatter for JsonFormatter<T> {
+    fn write_discovery_start(&mut self) -> io::Result<()> {
+        self.writeln_message(&format!(r#"{{ "type": "suite", "event": "discovery" }}"#))
+    }
+
+    fn write_test_discovered(&mut self, desc: &TestDesc, test_type: &str) -> io::Result<()> {
+        let TestDesc {
+            name,
+            ignore,
+            ignore_message,
+            #[cfg(not(bootstrap))]
+            source_file,
+            #[cfg(not(bootstrap))]
+            start_line,
+            #[cfg(not(bootstrap))]
+            start_col,
+            #[cfg(not(bootstrap))]
+            end_line,
+            #[cfg(not(bootstrap))]
+            end_col,
+            ..
+        } = desc;
+
+        #[cfg(bootstrap)]
+        let source_file = "";
+        #[cfg(bootstrap)]
+        let start_line = 0;
+        #[cfg(bootstrap)]
+        let start_col = 0;
+        #[cfg(bootstrap)]
+        let end_line = 0;
+        #[cfg(bootstrap)]
+        let end_col = 0;
+
+        self.writeln_message(&format!(
+            r#"{{ "type": "{test_type}", "event": "discovered", "name": "{}", "ignore": {ignore}, "ignore_message": "{}", "source_path": "{}", "start_line": {start_line}, "start_col": {start_col}, "end_line": {end_line}, "end_col": {end_col} }}"#,
+            EscapedString(name.as_slice()),
+            ignore_message.unwrap_or(""),
+            EscapedString(source_file),
+        ))
+    }
+
+    fn write_discovery_finish(&mut self, state: &ConsoleTestDiscoveryState) -> io::Result<()> {
+        let ConsoleTestDiscoveryState { tests, benchmarks, ignored, .. } = state;
+
+        let total = tests + benchmarks;
+        self.writeln_message(&format!(
+            r#"{{ "type": "suite", "event": "completed", "tests": {tests}, "benchmarks": {benchmarks}, "total": {total}, "ignored": {ignored} }}"#
+        ))
+    }
+
     fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option<u64>) -> io::Result<()> {
         let shuffle_seed_json = if let Some(shuffle_seed) = shuffle_seed {
             format!(r#", "shuffle_seed": {shuffle_seed}"#)

Diff for: library/test/src/formatters/junit.rs

@@ -3,7 +3,7 @@ use std::time::Duration;
 
 use super::OutputFormatter;
 use crate::{
-    console::{ConsoleTestState, OutputLocation},
+    console::{ConsoleTestDiscoveryState, ConsoleTestState, OutputLocation},
     test_result::TestResult,
     time,
     types::{TestDesc, TestType},
@@ -27,6 +27,18 @@ impl<T: Write> JunitFormatter<T> {
 }
 
 impl<T: Write> OutputFormatter for JunitFormatter<T> {
+    fn write_discovery_start(&mut self) -> io::Result<()> {
+        Err(io::Error::new(io::ErrorKind::NotFound, "Not yet implemented!"))
+    }
+
+    fn write_test_discovered(&mut self, _desc: &TestDesc, _test_type: &str) -> io::Result<()> {
+        Err(io::Error::new(io::ErrorKind::NotFound, "Not yet implemented!"))
+    }
+
+    fn write_discovery_finish(&mut self, _state: &ConsoleTestDiscoveryState) -> io::Result<()> {
+        Err(io::Error::new(io::ErrorKind::NotFound, "Not yet implemented!"))
+    }
+
     fn write_run_start(
         &mut self,
         _test_count: usize,

Diff for: library/test/src/formatters/mod.rs

@@ -1,7 +1,7 @@
 use std::{io, io::prelude::Write};
 
 use crate::{
-    console::ConsoleTestState,
+    console::{ConsoleTestDiscoveryState, ConsoleTestState},
     test_result::TestResult,
     time,
     types::{TestDesc, TestName},
@@ -18,6 +18,10 @@ pub(crate) use self::pretty::PrettyFormatter;
 pub(crate) use self::terse::TerseFormatter;
 
 pub(crate) trait OutputFormatter {
+    fn write_discovery_start(&mut self) -> io::Result<()>;
+    fn write_test_discovered(&mut self, desc: &TestDesc, test_type: &str) -> io::Result<()>;
+    fn write_discovery_finish(&mut self, state: &ConsoleTestDiscoveryState) -> io::Result<()>;
+
     fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option<u64>) -> io::Result<()>;
     fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>;
     fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>;

Diff for: library/test/src/formatters/pretty.rs

@@ -3,7 +3,7 @@ use std::{io, io::prelude::Write};
 use super::OutputFormatter;
 use crate::{
     bench::fmt_bench_samples,
-    console::{ConsoleTestState, OutputLocation},
+    console::{ConsoleTestDiscoveryState, ConsoleTestState, OutputLocation},
     term,
     test_result::TestResult,
     time,
@@ -181,6 +181,33 @@ impl<T: Write> PrettyFormatter<T> {
 }
 
 impl<T: Write> OutputFormatter for PrettyFormatter<T> {
+    fn write_discovery_start(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+
+    fn write_test_discovered(&mut self, desc: &TestDesc, test_type: &str) -> io::Result<()> {
+        self.write_plain(format!("{}: {test_type}\n", desc.name))
+    }
+
+    fn write_discovery_finish(&mut self, state: &ConsoleTestDiscoveryState) -> io::Result<()> {
+        fn plural(count: usize, s: &str) -> String {
+            match count {
+                1 => format!("1 {s}"),
+                n => format!("{n} {s}s"),
+            }
+        }
+
+        if state.tests != 0 || state.benchmarks != 0 {
+            self.write_plain("\n")?;
+        }
+
+        self.write_plain(format!(
+            "{}, {}\n",
+            plural(state.tests, "test"),
+            plural(state.benchmarks, "benchmark")
+        ))
+    }
+
     fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option<u64>) -> io::Result<()> {
         let noun = if test_count != 1 { "tests" } else { "test" };
         let shuffle_seed_msg = if let Some(shuffle_seed) = shuffle_seed {