Detect which reference examples should check/codegen/verify. (rust-la…

…ng#363)

* Copy options from the rust code blocks in the markdown files.

* Fix typos and add comments.

scripts/rmc.py

@@ -142,6 +142,8 @@ def compile_single_rust_file(input_filename, output_filename, verbose=False, deb
  "-Z", f"symbol-mangling-version={mangler}", 
  "-Z", f"symbol_table_passes={' '.join(symbol_table_passes)}",
  f"--cfg={RMC_CFG}", "-o", output_filename, input_filename]
+ if "RUSTFLAGS" in os.environ:
+ build_cmd += os.environ["RUSTFLAGS"].split(" ")
  build_env = os.environ
  if debug:
  add_rmc_rustc_debug_to_env(build_env)

src/tools/compiletest/src/runtest.rs

@@ -2394,12 +2394,12 @@ impl<'test> TestCx<'test> {
 
  /// Adds rmc scripts directory to the `PATH` environment variable.
  fn add_rmc_dir_to_path(&self, command: &mut Command) {
- // If the PATH enviornment variable is already defined,
+ // If the PATH environment variable is already defined,
  if let Some((key, val)) = env::vars().find(|(key, _)| key == "PATH") {
  // Add the RMC scripts directory to the PATH.
  command.env(key, format!("{}:{}", self.config.rmc_dir_path.to_str().unwrap(), val));
  } else {
- // Otherwise, insert PATH as a new enviornment variable and set its value to the RMC scripts directory.
+ // Otherwise, insert PATH as a new environment variable and set its value to the RMC scripts directory.
  command.env(
  String::from("PATH"),
  String::from(self.config.rmc_dir_path.to_str().unwrap()),
@@ -2411,7 +2411,10 @@ impl<'test> TestCx<'test> {
  /// error message is printed to stdout if the check result is not expected.
  fn check(&self) {
  let mut rustc = Command::new("rmc-rustc");
- rustc.args(["-Z", "no-codegen"]).arg(&self.testpaths.file);
+ rustc
+ .args(self.props.compile_flags.clone())
+ .args(["-Z", "no-codegen"])
+ .arg(&self.testpaths.file);
  self.add_rmc_dir_to_path(&mut rustc);
  let proc_res = self.compose_and_run_compiler(rustc, None);
  if self.props.rmc_panic_step == Some(RMCFailStep::Check) {
@@ -2431,6 +2434,7 @@ impl<'test> TestCx<'test> {
  fn codegen(&self) {
  let mut rustc = Command::new("rmc-rustc");
  rustc
+ .args(self.props.compile_flags.clone())
  .args(["-Z", "codegen-backend=gotoc", "--cfg=rmc", "--out-dir"])
  .arg(self.output_base_dir())
  .arg(&self.testpaths.file);
@@ -2461,6 +2465,13 @@ impl<'test> TestCx<'test> {
  // 2. It may pass some options that do not make sense for RMC
  // So we create our own command to execute RMC and pass it to self.compose_and_run_compiler(...) directly.
  let mut rmc = Command::new("rmc");
+ // We cannot pass rustc flags directly to RMC. Instead, we add them
+ // to the current environment through the `RUSTFLAGS` environment
+ // variable. RMC recognizes the variable and adds those flags to its
+ // internal call to rustc.
+ if !self.props.compile_flags.is_empty() {
+ rmc.env("RUSTFLAGS", self.props.compile_flags.join(" "));
+ }
  // Pass the test path along with RMC and CBMC flags parsed from comments at the top of the test file.
  rmc.args(&self.props.rmc_flags)
  .arg("--input")

src/tools/dashboard/src/reference.rs

@@ -8,7 +8,10 @@ use crate::dashboard;
 use pulldown_cmark::{Parser, Tag};
 use std::{
  collections::HashMap,
- env, fs,
+ env,
+ fmt::{Debug, Formatter, Result},
+ fs::{self, File},
+ hash::Hash,
  io::{BufRead, BufReader},
  path::{Path, PathBuf},
  process::{Command, Stdio},
@@ -19,17 +22,18 @@ use std::{
 /// code to corresponding directories where the extracted rust code should
 /// reside.
 fn parse_hierarchy(summary_path: &Path) -> HashMap<PathBuf, PathBuf> {
+ let summary_dir = summary_path.parent().unwrap().to_path_buf();
  let start = "# The Rust Reference\n\n[Introduction](introduction.md)";
  let summary = fs::read_to_string(summary_path).unwrap();
  assert!(summary.starts_with(start), "Error: The start of the summary file changed.");
  // Skip the title and introduction.
  let n = Parser::new(start).count();
  let parser = Parser::new(&summary).skip(n);
  // Set "ref" as the root of the hierarchical path.
- let mut hierarchy = PathBuf::from("ref");
+ let mut hierarchy: PathBuf = ["src", "test", "ref"].iter().collect();
  let mut map = HashMap::new();
  // Introduction is a especial case, so handle it separately.
- map.insert(PathBuf::from("introduction.md"), hierarchy.join("Introduction"));
+ map.insert(summary_dir.join("introduction.md"), hierarchy.join("Introduction"));
  for event in parser {
  match event {
  pulldown_cmark::Event::End(Tag::Item) => {
@@ -42,7 +46,9 @@ fn parse_hierarchy(summary_path: &Path) -> HashMap<PathBuf, PathBuf> {
  // contain the title of the current chapter/section. So, we wait
  // for the end of the link tag before adding the path and
  // hierarchy of the current chapter/section to the map.
- map.insert(path.split('/').collect(), hierarchy.clone());
+ let mut full_path = summary_dir.clone();
+ full_path.extend(path.split('/'));
+ map.insert(full_path, hierarchy.clone());
  }
  pulldown_cmark::Event::Text(text) => {
  // Add the current chapter/section title to the hierarchy.
@@ -54,87 +60,131 @@ fn parse_hierarchy(summary_path: &Path) -> HashMap<PathBuf, PathBuf> {
  map
 }
 
-/// Extracts examples from the given relative `paths` in the `book_dir` and
-/// saves them in `gen_dir`.
-fn extract_examples(paths: Vec<&PathBuf>, book_dir: &Path, gen_dir: &Path) {
- for path in paths {
- let mut cmd = Command::new("rustdoc");
- cmd.args([
- "+nightly",
- "--test",
- "-Z",
- "unstable-options",
- book_dir.join(path).to_str().unwrap(),
- "--test-builder",
- &["src", "tools", "dashboard", "print.sh"]
- .iter()
- .collect::<PathBuf>()
- .to_str()
- .unwrap(),
- "--persist-doctests",
- gen_dir.to_str().unwrap(),
- "--no-run",
- ]);
- cmd.stdout(Stdio::null());
- cmd.spawn().unwrap().wait().unwrap();
+/// The data structure represents the "full" path to examples in the Rust books.
+#[derive(PartialEq, Eq, Hash)]
+struct Example {
+ /// Path to the markdown file containing the example.
+ path: PathBuf,
+ /// Line number of the code block introducing the example.
+ line: usize,
+}
+
+impl Example {
+ /// Creates a new [`Example`] instance representing "full" path to the
+ /// Rust example.
+ fn new(path: PathBuf, line: usize) -> Example {
+ Example { path, line }
  }
 }
 
-/// Copies the extracted rust code in `from_dir` to `src/test` following the
-/// hierarchy specified by `map`.
-fn organize_examples(map: &HashMap<PathBuf, PathBuf>, book_dir: &Path, from_dir: &Path) {
- // The names of the extracted examples generated by `rustdoc` have the
- // format `<path>_<line-num>_<test-num>` where occurrences of '/', '-', and
- // '.' in <path> are replaced by '_'. This transformation is not injective,
- // so we cannot map those names back to the original markdown file path.
- // Instead, we apply the same transformation on the keys of `map` in the for
- // loop below and lookup <path> in those modified keys.
- let mut modified_map = HashMap::new();
- for (path, hierarchy) in map.iter() {
- modified_map.insert(
- book_dir.join(path).to_str().unwrap().replace(&['\\', '/', '-', '.'][..], "_"),
- hierarchy.clone(),
- );
+impl Debug for Example {
+ fn fmt(&self, f: &mut Formatter<'_>) -> Result {
+ f.write_fmt(format_args!("{}:{}", self.path.to_str().unwrap(), self.line))
  }
- for dir in from_dir.read_dir().unwrap() {
- let dir = dir.unwrap().path();
+}
+
+/// Extracts examples from the markdown files specified by each key in the given
+/// `map` and saves them in the directory specified by the corresponding value.
+/// Returns a mapping from the original location of **_each_** example to the
+/// path it was extracted to.
+fn extract_examples(par_map: HashMap<PathBuf, PathBuf>) -> HashMap<Example, PathBuf> {
+ let mut full_map = HashMap::new();
+ for (par_from, par_to) in par_map {
+ let pairs = extract(&par_from, &par_to);
+ for (key, val) in pairs {
+ full_map.insert(key, val);
+ }
+ }
+ full_map
+}
+
+/// Extracts examples from the markdown files specified by `par_from` and saves
+/// them in the directory specified by `par_to`. Returns a mapping from the
+/// original location of **_each_** example to the path it was extracted to.
+fn extract(par_from: &Path, par_to: &Path) -> Vec<(Example, PathBuf)> {
+ let build_dir = &env::var("BUILD_DIR").unwrap();
+ let triple = &env::var("TRIPLE").unwrap();
+ // Create a temporary directory to save the files generated by `rustdoc`.
+ let gen_dir: PathBuf = [build_dir, triple, "dashboard", "ref"].iter().collect();
+ fs::create_dir_all(&gen_dir).unwrap();
+ let mut cmd = Command::new("rustdoc");
+ cmd.args([
+ "+nightly",
+ "--test",
+ "-Z",
+ "unstable-options",
+ par_from.to_str().unwrap(),
+ "--test-builder",
+ &["src", "tools", "dashboard", "print.sh"].iter().collect::<PathBuf>().to_str().unwrap(),
+ "--persist-doctests",
+ gen_dir.to_str().unwrap(),
+ "--no-run",
+ ]);
+ cmd.stdout(Stdio::null());
+ cmd.spawn().unwrap().wait().unwrap();
+ // Mapping from path and line number of rust example to where it was extracted to.
+ let mut pairs = Vec::new();
+
+ for dir in gen_dir.read_dir().unwrap() {
  // Some directories do not contain tests because the markdown file
- // instructs `rustdoc` to ignore those tests.
- if let Some(example) = dir.read_dir().unwrap().next() {
- let example = example.unwrap().path();
- copy(&example, &modified_map);
+ // instructs `rustdoc` to "ignore" those tests.
+ let dir = dir.unwrap().path();
+ if let Some(from) = dir.read_dir().unwrap().next() {
+ // The path to each example extracted by `rustdoc` has the form:
+ // <from> = `<gen_dir>/<par_from>_<line>_<test-num>/rust_out`
+ // where occurrences of '/', '-', and '.' in <par_from> are replaced
+ // by '_'. We copy the file in this path to a new path of the form:
+ // <to> = `<par_to>/<line>.rs`
+ // We omit <test-num> because all tests have the same number, 0.
+ let from = from.unwrap().path();
+ let path_line_test = dir.file_name().unwrap().to_str().unwrap();
+ let splits: Vec<_> = path_line_test.rsplitn(3, '_').collect();
+ let line: usize = splits[1].parse().unwrap();
+ let to = par_to.join(format!("{}.rs", line));
+ fs::create_dir_all(par_to).unwrap();
+ fs::copy(&from, &to).unwrap();
+ pairs.push((Example::new(par_from.to_path_buf(), line), to));
  }
  }
+ // Delete the temporary directory.
+ fs::remove_dir_all(gen_dir).unwrap();
+ pairs
 }
 
-/// Copy the file specified by `from` to the corresponding location specified by
-/// `map`.
-fn copy(from: &Path, map: &HashMap<String, PathBuf>) {
- // The path specified by `from` has the form:
- // `build/<triple>/dashboard/ref/<key>_<line-num>_<test-num>/rust_out`
- // We copy the file in this path to a new path of the form:
- // `src/test/<val>/<line-num>.rs
- // where `map[<key>] == <val>`. We omit <test-num> because all tests have
- // the same number, 0.
- // Extract `<key>_<line-num>_<test-num>`.
- let key_line_test = from.parent().unwrap().file_name().unwrap().to_str().unwrap();
- // Extract <key> and <line-num> from `key_line_test` to get <val> and
- // construct destination path.
- let splits: Vec<_> = key_line_test.rsplitn(3, '_').collect();
- let key = splits[2];
- let line = splits[1];
- let val = &map[key];
- let name = &format!("{}.rs", line);
- let to = Path::new("src").join("test").join(val).join(name);
- fs::create_dir_all(to.parent().unwrap()).unwrap();
- fs::copy(&from, &to).unwrap();
+/// Prepends the text in `path` with the given `text`.
+fn prepend_text(path: &Path, text: &str) {
+ let code = fs::read_to_string(&path).unwrap();
+ let code = format!("{}\n{}", text, code);
+ fs::write(&path, code).unwrap();
 }
 
-/// Pre-processes the tests in the specified `paths` before running them with
-/// `compiletest`.
-fn preprocess_examples(_paths: Vec<&PathBuf>) {
- // For now, we will only pre-process the tests that cause infinite loops.
- // TODO: properly implement this step (see issue #324).
+/// Pre-processes the examples in `map` before running them with `compiletest`.
+fn preprocess_examples(map: &HashMap<Example, PathBuf>) {
+ // Copy compiler configurations specified in the original markdown code
+ // block.
+ for (from, to) in map.iter() {
+ let file = File::open(&from.path).unwrap();
+ // Skip to the first line of the example code block.
+ // Line numbers in files start with 1 but `nth(...)` starts with 0.
+ // Subtract 1 to account for the difference.
+ let line = BufReader::new(file).lines().nth(from.line - 1).unwrap().unwrap();
+ if line.contains("edition2015") {
+ prepend_text(to, "// compile-flags: --edition 2015");
+ } else {
+ prepend_text(to, "// compile-flags: --edition 2018");
+ }
+ // Most examples with `compile_fail` configuration fail because of
+ // check errors.
+ if line.contains("compile_fail") {
+ prepend_text(to, "// rmc-check-fail");
+ }
+ // RMC should catch run-time errors.
+ if line.contains("should_panic") {
+ prepend_text(to, "// rmc-verify-fail");
+ }
+ }
+ // For now, we will only manually pre-process the tests that cause infinite loops.
+ // TODO: Add support for manually adding options and assertions (see issue #324).
  let loop_tests: [PathBuf; 4] = [
  ["src", "test", "ref", "Appendices", "Glossary", "263.rs"].iter().collect(),
  ["src", "test", "ref", "Linkage", "190.rs"].iter().collect(),
@@ -193,7 +243,6 @@ fn run_examples(suite: &str, log_path: &Path) {
  ]);
  cmd.env_clear().envs(filtered_env);
  cmd.stdout(Stdio::null());
-
  cmd.spawn().unwrap().wait().unwrap();
 }
 
@@ -258,20 +307,17 @@ fn display_dashboard(dashboard: dashboard::Tree) {
 /// displays their results in a terminal dashboard.
 pub fn display_reference_dashboard() {
  let summary_path: PathBuf = ["src", "doc", "reference", "src", "SUMMARY.md"].iter().collect();
- let ref_dir: PathBuf = ["src", "doc", "reference", "src"].iter().collect();
  let build_dir = &env::var("BUILD_DIR").unwrap();
  let triple = &env::var("TRIPLE").unwrap();
- let gen_dir: PathBuf = [build_dir, triple, "dashboard", "ref"].iter().collect();
  let log_path: PathBuf = [build_dir, triple, "dashboard", "ref.log"].iter().collect();
  // Parse the chapter/section hierarchy from the table of contents in The
  // Rust Reference.
  let map = parse_hierarchy(&summary_path);
- // Extract examples from The Rust Reference.
- extract_examples(map.keys().collect(), &ref_dir, &gen_dir);
- // Reorganize those examples following the The Rust Reference hierarchy.
- organize_examples(&map, &ref_dir, &gen_dir);
+ // Extract examples from The Rust Reference, organize them following the
+ // partial hierarchy in map, and return the full hierarchy map.
+ let map = extract_examples(map);
  // Pre-process the examples before running them through `compiletest`.
- preprocess_examples(map.values().collect());
+ preprocess_examples(&map);
  // Run `compiletest` on the reference examples.
  run_examples("ref", &log_path);
  // Parse `compiletest` log file.