Auto merge of #114843 - Zalathar:test-coverage-map, r=oli-obk

coverage: Explicitly test the coverage maps produced by codegen/LLVM Our existing coverage tests verify the output of end-to-end coverage reports, but we don't have any way to test the specific mapping information (code regions and their associated counters) that are emitted by `rustc_codegen_llvm` and LLVM. That makes it harder to to be confident in changes that would modify those mappings (whether deliberately or accidentally). This PR addresses that by adding a new `coverage-map` test suite that does the following: - Compiles test files to LLVM IR assembly (`.ll`) - Feeds those IR files to a custom tool (`src/tools/coverage-dump`) that extracts and decodes coverage mappings, and prints them in a more human-readable format - Checks the output of that tool against known-good snapshots --- I recommend excluding the last commit while reviewing the main changes, because that last commit is just ~40 test files copied over from `tests/run-coverage`, plus their blessed coverage-map snapshots and a readme file. Those snapshots aren't really intended to be checked by hand; they're mostly there to increase the chances that an unintended change to coverage maps will be observable (even if it requires relatively specific circumstances to manifest).
rust-lang · Sep 5, 2023 · ab45885 · ab45885
2 parents f222a2d + 3141177
commit ab45885
Show file tree

Hide file tree

Showing 102 changed files with 6,395 additions and 6 deletions.
diff --git a/Cargo.lock b/Cargo.lock
@@ -722,6 +722,18 @@ version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
 
+[[package]]
+name = "coverage-dump"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "leb128",
+ "md-5",
+ "miniz_oxide",
+ "regex",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "coverage_test_macros"
 version = "0.0.0"
@@ -2041,6 +2053,12 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
+[[package]]
+name = "leb128"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
+
 [[package]]
 name = "levenshtein"
 version = "1.0.5"

diff --git a/Cargo.toml b/Cargo.toml
@@ -43,6 +43,7 @@ members = [
   "src/tools/generate-windows-sys",
   "src/tools/rustdoc-gui-test",
   "src/tools/opt-dist",
+  "src/tools/coverage-dump",
 ]
 
 exclude = [

diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
@@ -703,7 +703,8 @@ impl<'a> Builder<'a> {
                 llvm::Lld,
                 llvm::CrtBeginEnd,
                 tool::RustdocGUITest,
-                tool::OptimizedDist
+                tool::OptimizedDist,
+                tool::CoverageDump,
             ),
             Kind::Check | Kind::Clippy | Kind::Fix => describe!(
                 check::Std,
@@ -725,6 +726,7 @@ impl<'a> Builder<'a> {
                 test::Tidy,
                 test::Ui,
                 test::RunPassValgrind,
+                test::CoverageMap,
                 test::RunCoverage,
                 test::MirOpt,
                 test::Codegen,

diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
@@ -1340,6 +1340,12 @@ host_test!(RunMakeFullDeps {
 
 default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" });
 
+default_test!(CoverageMap {
+    path: "tests/coverage-map",
+    mode: "coverage-map",
+    suite: "coverage-map"
+});
+
 host_test!(RunCoverage { path: "tests/run-coverage", mode: "run-coverage", suite: "run-coverage" });
 host_test!(RunCoverageRustdoc {
     path: "tests/run-coverage-rustdoc",
@@ -1545,6 +1551,14 @@ note: if you're sure you want to do this, please open an issue as to why. In the
                 .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }));
         }
 
+        if mode == "coverage-map" {
+            let coverage_dump = builder.ensure(tool::CoverageDump {
+                compiler: compiler.with_stage(0),
+                target: compiler.host,
+            });
+            cmd.arg("--coverage-dump-path").arg(coverage_dump);
+        }
+
         if mode == "run-make" || mode == "run-coverage" {
             let rust_demangler = builder
                 .ensure(tool::RustDemangler {

diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs
@@ -306,6 +306,7 @@ bootstrap_tool!(
     GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys";
     RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test";
     OptimizedDist, "src/tools/opt-dist", "opt-dist";
+    CoverageDump, "src/tools/coverage-dump", "coverage-dump";
 );
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]

diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs
@@ -66,6 +66,7 @@ string_enum! {
         JsDocTest => "js-doc-test",
         MirOpt => "mir-opt",
         Assembly => "assembly",
+        CoverageMap => "coverage-map",
         RunCoverage => "run-coverage",
     }
 }
@@ -161,6 +162,9 @@ pub struct Config {
     /// The rust-demangler executable.
     pub rust_demangler_path: Option<PathBuf>,
 
+    /// The coverage-dump executable.
+    pub coverage_dump_path: Option<PathBuf>,
+
     /// The Python executable to use for LLDB and htmldocck.
     pub python: String,
 
@@ -639,6 +643,7 @@ pub const UI_EXTENSIONS: &[&str] = &[
     UI_STDERR_32,
     UI_STDERR_16,
     UI_COVERAGE,
+    UI_COVERAGE_MAP,
 ];
 pub const UI_STDERR: &str = "stderr";
 pub const UI_STDOUT: &str = "stdout";
@@ -649,6 +654,7 @@ pub const UI_STDERR_64: &str = "64bit.stderr";
 pub const UI_STDERR_32: &str = "32bit.stderr";
 pub const UI_STDERR_16: &str = "16bit.stderr";
 pub const UI_COVERAGE: &str = "coverage";
+pub const UI_COVERAGE_MAP: &str = "cov-map";
 
 /// Absolute path to the directory where all output for all tests in the given
 /// `relative_dir` group should reside. Example:

diff --git a/src/tools/compiletest/src/lib.rs b/src/tools/compiletest/src/lib.rs
@@ -48,6 +48,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH")
         .optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
         .optopt("", "rust-demangler-path", "path to rust-demangler to use in tests", "PATH")
+        .optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
         .reqopt("", "python", "path to python to use for doc tests", "PATH")
         .optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH")
         .optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH")
@@ -218,6 +219,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         rustc_path: opt_path(matches, "rustc-path"),
         rustdoc_path: matches.opt_str("rustdoc-path").map(PathBuf::from),
         rust_demangler_path: matches.opt_str("rust-demangler-path").map(PathBuf::from),
+        coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from),
         python: matches.opt_str("python").unwrap(),
         jsondocck_path: matches.opt_str("jsondocck-path"),
         jsondoclint_path: matches.opt_str("jsondoclint-path"),

diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
@@ -6,8 +6,8 @@ use crate::common::{Assembly, Incremental, JsDocTest, MirOpt, RunMake, RustdocJs
 use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
 use crate::common::{CompareMode, FailMode, PassMode};
 use crate::common::{Config, TestPaths};
-use crate::common::{Pretty, RunCoverage, RunPassValgrind};
-use crate::common::{UI_COVERAGE, UI_RUN_STDERR, UI_RUN_STDOUT};
+use crate::common::{CoverageMap, Pretty, RunCoverage, RunPassValgrind};
+use crate::common::{UI_COVERAGE, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT};
 use crate::compute_diff::{write_diff, write_filtered_diff};
 use crate::errors::{self, Error, ErrorKind};
 use crate::header::TestProps;
@@ -254,6 +254,7 @@ impl<'test> TestCx<'test> {
             MirOpt => self.run_mir_opt_test(),
             Assembly => self.run_assembly_test(),
             JsDocTest => self.run_js_doc_test(),
+            CoverageMap => self.run_coverage_map_test(),
             RunCoverage => self.run_coverage_test(),
         }
     }
@@ -467,6 +468,46 @@ impl<'test> TestCx<'test> {
         }
     }
 
+    fn run_coverage_map_test(&self) {
+        let Some(coverage_dump_path) = &self.config.coverage_dump_path else {
+            self.fatal("missing --coverage-dump");
+        };
+
+        let proc_res = self.compile_test_and_save_ir();
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("compilation failed!", &proc_res);
+        }
+        drop(proc_res);
+
+        let llvm_ir_path = self.output_base_name().with_extension("ll");
+
+        let mut dump_command = Command::new(coverage_dump_path);
+        dump_command.arg(llvm_ir_path);
+        let proc_res = self.run_command_to_procres(&mut dump_command);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("coverage-dump failed!", &proc_res);
+        }
+
+        let kind = UI_COVERAGE_MAP;
+
+        let expected_coverage_dump = self.load_expected_output(kind);
+        let actual_coverage_dump = self.normalize_output(&proc_res.stdout, &[]);
+
+        let coverage_dump_errors = self.compare_output(
+            kind,
+            &actual_coverage_dump,
+            &expected_coverage_dump,
+            self.props.compare_output_lines_by_subset,
+        );
+
+        if coverage_dump_errors > 0 {
+            self.fatal_proc_rec(
+                &format!("{coverage_dump_errors} errors occurred comparing coverage output."),
+                &proc_res,
+            );
+        }
+    }
+
     fn run_coverage_test(&self) {
         let should_run = self.run_if_enabled();
         let proc_res = self.compile_test(should_run, Emit::None);
@@ -650,6 +691,10 @@ impl<'test> TestCx<'test> {
         let mut cmd = Command::new(tool_path);
         configure_cmd_fn(&mut cmd);
 
+        self.run_command_to_procres(&mut cmd)
+    }
+
+    fn run_command_to_procres(&self, cmd: &mut Command) -> ProcRes {
         let output = cmd.output().unwrap_or_else(|_| panic!("failed to exec `{cmd:?}`"));
 
         let proc_res = ProcRes {
@@ -2321,9 +2366,11 @@ impl<'test> TestCx<'test> {
                     }
                 }
                 DebugInfo => { /* debuginfo tests must be unoptimized */ }
-                RunCoverage => {
-                    // Coverage reports are affected by optimization level, and
-                    // the current snapshots assume no optimization by default.
+                CoverageMap | RunCoverage => {
+                    // Coverage mappings and coverage reports are affected by
+                    // optimization level, so they ignore the optimize-tests
+                    // setting and set an optimization level in their mode's
+                    // compile flags (below) or in per-test `compile-flags`.
                 }
                 _ => {
                     rustc.arg("-O");
@@ -2392,8 +2439,22 @@ impl<'test> TestCx<'test> {
 
                 rustc.arg(dir_opt);
             }
+            CoverageMap => {
+                rustc.arg("-Cinstrument-coverage");
+                // These tests only compile to MIR, so they don't need the
+                // profiler runtime to be present.
+                rustc.arg("-Zno-profiler-runtime");
+                // Coverage mappings are sensitive to MIR optimizations, and
+                // the current snapshots assume `opt-level=2` unless overridden
+                // by `compile-flags`.
+                rustc.arg("-Copt-level=2");
+            }
             RunCoverage => {
                 rustc.arg("-Cinstrument-coverage");
+                // Coverage reports are sometimes sensitive to optimizations,
+                // and the current snapshots assume no optimization unless
+                // overridden by `compile-flags`.
+                rustc.arg("-Copt-level=0");
             }
             RunPassValgrind | Pretty | DebugInfo | Codegen | Rustdoc | RustdocJson | RunMake
             | CodegenUnits | JsDocTest | Assembly => {

diff --git a/src/tools/coverage-dump/Cargo.toml b/src/tools/coverage-dump/Cargo.toml
@@ -0,0 +1,14 @@
+[package]
+name = "coverage-dump"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.71"
+leb128 = "0.2.5"
+md5 = { package = "md-5" , version = "0.10.5" }
+miniz_oxide = "0.7.1"
+regex = "1.8.4"
+rustc-demangle = "0.1.23"
diff --git a/src/tools/coverage-dump/README.md b/src/tools/coverage-dump/README.md
@@ -0,0 +1,8 @@
+This tool extracts coverage mapping information from an LLVM IR assembly file
+(`.ll`), and prints it in a more human-readable form that can be used for
+snapshot tests.
+
+The output format is mostly arbitrary, so it's OK to change the output as long
+as any affected tests are also re-blessed. However, the output should be
+consistent across different executions on different platforms, so avoid
+printing any information that is platform-specific or non-deterministic.