diff --git a/.github/workflows/copyright.yml b/.github/workflows/copyright.yml index cc3a972e66b2a..b9a962f4eab79 100644 --- a/.github/workflows/copyright.yml +++ b/.github/workflows/copyright.yml @@ -15,7 +15,7 @@ jobs: - name: Get paths for files added id: git-diff run: | - ignore='(.md|expected|ignore|gitignore)$' + ignore='(.md|.props|expected|ignore|gitignore)$' files=$(git diff --ignore-submodules=all --name-only --diff-filter=A ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | grep -v -E $ignore | xargs) echo "::set-output name=paths::$files" diff --git a/Cargo.lock b/Cargo.lock index dfb60b8a8fd5a..91113929daf53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -890,6 +890,7 @@ name = "dashboard" version = "0.1.0" dependencies = [ "pulldown-cmark 0.8.0", + "walkdir", ] [[package]] @@ -5595,9 +5596,9 @@ dependencies = [ [[package]] name = "walkdir" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" dependencies = [ "same-file", "winapi", diff --git a/src/tools/dashboard/Cargo.toml b/src/tools/dashboard/Cargo.toml index 4e4ee120afc23..bb5efea2e9789 100644 --- a/src/tools/dashboard/Cargo.toml +++ b/src/tools/dashboard/Cargo.toml @@ -8,3 +8,4 @@ edition = "2018" [dependencies] pulldown-cmark = { version = "0.8.0", default-features = false } +walkdir = "2.3.2" diff --git a/src/tools/dashboard/configs/ref/Appendices/Glossary/263.props b/src/tools/dashboard/configs/ref/Appendices/Glossary/263.props new file mode 100644 index 0000000000000..b72125d5f0bea --- /dev/null +++ b/src/tools/dashboard/configs/ref/Appendices/Glossary/263.props @@ -0,0 +1 @@ +// rmc-flags: --cbmc-args --unwind 4 diff --git a/src/tools/dashboard/configs/ref/Attributes/Limits/45.props b/src/tools/dashboard/configs/ref/Attributes/Limits/45.props new file mode 100644 index 0000000000000..89b9bfc2ca3d9 --- /dev/null +++ b/src/tools/dashboard/configs/ref/Attributes/Limits/45.props @@ -0,0 +1 @@ +// rmc-codegen-fail diff --git a/src/tools/dashboard/configs/ref/Linkage/190.props b/src/tools/dashboard/configs/ref/Linkage/190.props new file mode 100644 index 0000000000000..64cfe4cf8caa4 --- /dev/null +++ b/src/tools/dashboard/configs/ref/Linkage/190.props @@ -0,0 +1 @@ +// rmc-flags: --cbmc-args --unwind 1 diff --git a/src/tools/dashboard/configs/ref/Statements and expressions/Expressions/Loop expressions/133.props b/src/tools/dashboard/configs/ref/Statements and expressions/Expressions/Loop expressions/133.props new file mode 100644 index 0000000000000..b72125d5f0bea --- /dev/null +++ b/src/tools/dashboard/configs/ref/Statements and expressions/Expressions/Loop expressions/133.props @@ -0,0 +1 @@ +// rmc-flags: --cbmc-args --unwind 4 diff --git a/src/tools/dashboard/configs/ref/Statements and expressions/Expressions/Method call expressions/10.props b/src/tools/dashboard/configs/ref/Statements and expressions/Expressions/Method call expressions/10.props new file mode 100644 index 0000000000000..64cfe4cf8caa4 --- /dev/null +++ b/src/tools/dashboard/configs/ref/Statements and expressions/Expressions/Method call expressions/10.props @@ -0,0 +1 @@ +// rmc-flags: --cbmc-args --unwind 1 diff --git a/src/tools/dashboard/src/reference.rs b/src/tools/dashboard/src/reference.rs index d223b970ddbc4..a6a3184d15c12 100644 --- a/src/tools/dashboard/src/reference.rs +++ b/src/tools/dashboard/src/reference.rs @@ -4,18 +4,23 @@ //! [The Rust Reference](https://doc.rust-lang.org/nightly/reference), //! run them through RMC, and display their results. -use crate::{dashboard, litani::Litani, util}; +use crate::{ + dashboard, + litani::Litani, + util::{self, FailStep, TestProps}, +}; use pulldown_cmark::{Parser, Tag}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, env, - fmt::{Debug, Formatter, Result}, + fmt::{Debug, Formatter, Result, Write}, fs::{self, File}, hash::Hash, io::{BufRead, BufReader}, path::{Path, PathBuf}, process::{Command, Stdio}, }; +use walkdir::WalkDir; /// Parses the chapter/section hierarchy in the markdown file specified by /// `summary_path` and returns a mapping from markdown files containing rust @@ -151,72 +156,98 @@ fn extract(par_from: &Path, par_to: &Path) -> Vec<(Example, PathBuf)> { pairs } -/// Prepends the text in `path` with the given `text`. -fn prepend_text(path: &Path, text: &str) { - let code = fs::read_to_string(&path).unwrap(); - let code = format!("{}\n{}", text, code); - fs::write(&path, code).unwrap(); +/// Returns a set of paths to the config files for examples in the Rust books. +fn get_config_paths() -> HashSet { + let config_dir: PathBuf = ["src", "tools", "dashboard", "configs"].iter().collect(); + let mut config_paths = HashSet::new(); + for entry in WalkDir::new(config_dir) { + let entry = entry.unwrap().into_path(); + if entry.is_file() { + config_paths.insert(entry); + } + } + config_paths +} + +/// Prepends the given `props` to the test file in `props.test`. +fn prepend_props(props: &TestProps) { + let code = fs::read_to_string(&props.path).unwrap(); + let code = format!("{}{}", props, code); + fs::write(&props.path, code).unwrap(); +} + +/// Pretty prints the `paths` set. +fn paths_to_string(paths: HashSet) -> String { + let mut f = String::new(); + for path in paths { + f.write_fmt(format_args!(" {:?}\n", path.to_str().unwrap())).unwrap(); + } + f } /// Pre-processes the examples in `map` before running them with `compiletest`. fn preprocess_examples(map: &HashMap) { - // Copy compiler configurations specified in the original markdown code - // block. + let config_dir: PathBuf = ["src", "tools", "dashboard", "configs"].iter().collect(); + let test_dir: PathBuf = ["src", "test"].iter().collect(); + let mut config_paths = get_config_paths(); + // Copy compiler annotations specified in the original markdown code blocks + // and custom configurations under the `config` directory. for (from, to) in map.iter() { + // Path `to` has the following form: + // `src/test/ref//.rs` + // If it has a custom props file, the path to the props file will have + // the following form: + // `src/tools/dashboard/configs/ref//.props` + // where and are the same for both paths. + let mut props_path = config_dir.join(to.strip_prefix(&test_dir).unwrap()); + props_path.set_extension("props"); + let mut props = if props_path.exists() { + config_paths.remove(&props_path); + // Parse the properties in the file. The format follows the same + // conventions for the headers in RMC regressions. + let mut props = util::parse_test_header(&props_path); + // `util::parse_test_header` thinks `props_path` is the path to the + // test. That is not the case, `to` is the actual path to the + // test/example. + props.path = to.clone(); + props + } else { + TestProps::new(to.clone(), None, Vec::new(), Vec::new()) + }; let file = File::open(&from.path).unwrap(); // Skip to the first line of the example code block. // Line numbers in files start with 1 but `nth(...)` starts with 0. // Subtract 1 to account for the difference. let line = BufReader::new(file).lines().nth(from.line - 1).unwrap().unwrap(); - if line.contains("edition2015") { - prepend_text(to, "// compile-flags: --edition 2015"); - } else { - prepend_text(to, "// compile-flags: --edition 2018"); + if !line.contains("edition2015") { + props.rustc_args.push(String::from("--edition")); + props.rustc_args.push(String::from("2018")); } - // Most examples with `compile_fail` configuration fail because of - // check errors. - if line.contains("compile_fail") { - prepend_text(to, "// rmc-check-fail"); + // Most examples with `compile_fail` annotation fail because of check + // errors. This heuristic can be overridden by manually specifying the + // fail step in the corresponding config file. + if props.fail_step.is_none() && line.contains("compile_fail") { + props.fail_step = Some(FailStep::Check); } // RMC should catch run-time errors. - if line.contains("should_panic") { - prepend_text(to, "// rmc-verify-fail"); + if props.fail_step.is_none() && line.contains("should_panic") { + props.fail_step = Some(FailStep::Verification); } + // Prepend those properties to test/example file. + prepend_props(&props); } - // For now, we will only manually pre-process the tests that cause infinite loops. - // TODO: Add support for manually adding options and assertions (see issue #324). - let loop_tests: [PathBuf; 4] = [ - ["src", "test", "ref", "Appendices", "Glossary", "263.rs"].iter().collect(), - ["src", "test", "ref", "Linkage", "190.rs"].iter().collect(), - [ - "src", - "test", - "ref", - "Statements and expressions", - "Expressions", - "Loop expressions", - "133.rs", - ] - .iter() - .collect(), - [ - "src", - "test", - "ref", - "Statements and expressions", - "Expressions", - "Method call expressions", - "10.rs", - ] - .iter() - .collect(), - ]; - - for test in loop_tests { - let code = fs::read_to_string(&test).unwrap(); - let code = format!("// rmc-flags: --cbmc-args --unwind 1\n{}", code); - fs::write(&test, code).unwrap(); + if !config_paths.is_empty() { + panic!( + "Error: The examples corresponding to the following config files \ + were not encountered in the pre-processing step:\n{}This is most \ + likely because the line numbers of the config files are not in \ + sync with the line numbers of the corresponding code blocks in \ + the latest versions of the Rust books. Please update the line \ + numbers of the config files and rerun the program.", + paths_to_string(config_paths) + ); } + // TODO: Add support for manually adding assertions (see issue #324). } /// Runs `compiletest` on the `suite` and logs the results to `log_path`. @@ -310,18 +341,12 @@ fn litani_run_tests() { let ref_dir: PathBuf = ["src", "test", "ref"].iter().collect(); util::add_rmc_and_litani_to_path(); let mut litani = Litani::init("RMC", &output_prefix, &output_symlink); - let mut stack = vec![ref_dir]; // Run all tests under the `src/test/ref` directory. - while !stack.is_empty() { - let cur_dir = stack.pop().unwrap(); - for child in cur_dir.read_dir().unwrap() { - let child = child.unwrap().path(); - if child.is_file() { - let test_props = util::parse_test_header(&child); - util::add_test_pipeline(&mut litani, &test_props); - } else { - stack.push(child); - } + for entry in WalkDir::new(ref_dir) { + let entry = entry.unwrap().into_path(); + if entry.is_file() { + let test_props = util::parse_test_header(&entry); + util::add_test_pipeline(&mut litani, &test_props); } } litani.run_build(); diff --git a/src/tools/dashboard/src/util.rs b/src/tools/dashboard/src/util.rs index 718cd490164f7..bc22062660d6c 100644 --- a/src/tools/dashboard/src/util.rs +++ b/src/tools/dashboard/src/util.rs @@ -9,6 +9,7 @@ use crate::litani::Litani; use std::{ env, + fmt::{self, Display, Formatter, Write}, fs::File, io::{BufRead, BufReader}, path::{Path, PathBuf}, @@ -30,6 +31,17 @@ pub enum FailStep { Verification, } +impl Display for FailStep { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let str = match self { + FailStep::Check => "check", + FailStep::Codegen => "codegen", + FailStep::Verification => "verify", + }; + f.write_str(str) + } +} + /// Data structure representing properties specific to each test. pub struct TestProps { pub path: PathBuf, @@ -53,6 +65,29 @@ impl TestProps { } } +impl Display for TestProps { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + if let Some(fail_step) = &self.fail_step { + f.write_fmt(format_args!("// rmc-{}-fail\n", fail_step))?; + } + if !self.rustc_args.is_empty() { + f.write_str("// compile-flags:")?; + for arg in &self.rustc_args { + f.write_fmt(format_args!(" {}", arg))?; + } + f.write_char('\n')?; + } + if !self.rmc_args.is_empty() { + f.write_str("// rmc-flags:")?; + for arg in &self.rmc_args { + f.write_fmt(format_args!(" {}", arg))?; + } + f.write_char('\n')?; + } + Ok(()) + } +} + /// Parses strings of the form `rmc-*-fail` and returns the step at which RMC is /// expected to panic. fn try_parse_fail_step(cur_fail_step: Option, line: &str) -> Option { @@ -140,7 +175,7 @@ pub fn add_check_job(litani: &mut Litani, test_props: &TestProps) { test_props.path.to_str().unwrap(), "build", exit_status, - 1, + 5, ); } @@ -166,7 +201,7 @@ pub fn add_codegen_job(litani: &mut Litani, test_props: &TestProps) { test_props.path.to_str().unwrap(), "test", exit_status, - 1, + 5, ); } @@ -174,7 +209,7 @@ pub fn add_codegen_job(litani: &mut Litani, test_props: &TestProps) { pub fn add_verification_job(litani: &mut Litani, test_props: &TestProps) { let exit_status = if test_props.fail_step == Some(FailStep::Verification) { 10 } else { 0 }; let mut rmc = Command::new("rmc"); - rmc.args(&test_props.rmc_args).arg(&test_props.path); + rmc.arg(&test_props.path).args(&test_props.rmc_args); if !test_props.rustc_args.is_empty() { rmc.env("RUSTFLAGS", test_props.rustc_args.join(" ")); }