test: allow the test filter to be a regex.

This is fully backwards compatible, since test names are Rust
identifiers + `:`, and hence not special regex characters.

Fixes #2866.

Author: huonw

mk/crates.mk

@@ -80,7 +80,7 @@ DEPS_collections := std rand
 DEPS_fourcc := syntax std
 DEPS_hexfloat := syntax std
 DEPS_num := std rand
-DEPS_test := std collections getopts serialize term time
+DEPS_test := std collections getopts serialize term time regex
 DEPS_time := std serialize
 DEPS_rand := std
 DEPS_url := std collections

src/compiletest/common.rs

@@ -10,6 +10,7 @@
 
 use std::from_str::FromStr;
 use std::fmt;
+use regex::Regex;
 
 #[deriving(Clone, Eq)]
 pub enum Mode {
@@ -88,7 +89,7 @@ pub struct Config {
     pub run_ignored: bool,
 
     // Only run tests that match this filter
-    pub filter: Option<~str>,
+    pub filter: Option<Regex>,
 
     // Write out a parseable log of tests that were run
     pub logfile: Option<Path>,

src/compiletest/compiletest.rs

@@ -23,6 +23,8 @@ extern crate log;
 extern crate green;
 extern crate rustuv;
 
+extern crate regex;
+
 use std::os;
 use std::io;
 use std::io::fs;
@@ -113,6 +115,19 @@ pub fn parse_config(args: Vec<~str> ) -> Config {
         Path::new(m.opt_str(nm).unwrap())
     }
 
+    let filter = if !matches.free.is_empty() {
+        let s = matches.free.get(0).as_slice();
+        match regex::Regex::new(s) {
+            Ok(re) => Some(re),
+            Err(e) => {
+                println!("failed to parse filter /{}/: {}", s, e);
+                fail!()
+            }
+        }
+    } else {
+        None
+    };
+
     Config {
         compile_lib_path: matches.opt_str("compile-lib-path").unwrap(),
         run_lib_path: matches.opt_str("run-lib-path").unwrap(),
@@ -125,12 +140,7 @@ pub fn parse_config(args: Vec<~str> ) -> Config {
         stage_id: matches.opt_str("stage-id").unwrap(),
         mode: FromStr::from_str(matches.opt_str("mode").unwrap()).expect("invalid mode"),
         run_ignored: matches.opt_present("ignored"),
-        filter:
-            if !matches.free.is_empty() {
-                 Some((*matches.free.get(0)).clone())
-            } else {
-                None
-            },
+        filter: filter,
         logfile: matches.opt_str("logfile").map(|s| Path::new(s)),
         save_metrics: matches.opt_str("save-metrics").map(|s| Path::new(s)),
         ratchet_metrics:
@@ -169,7 +179,7 @@ pub fn log_config(config: &Config) {
     logv(c, format!("stage_id: {}", config.stage_id));
     logv(c, format!("mode: {}", config.mode));
     logv(c, format!("run_ignored: {}", config.run_ignored));
-    logv(c, format!("filter: {}", opt_str(&config.filter)));
+    logv(c, format!("filter: {}", opt_str(&config.filter.as_ref().map(|re| re.to_str()))));
     logv(c, format!("runtool: {}", opt_str(&config.runtool)));
     logv(c, format!("host-rustcflags: {}", opt_str(&config.host_rustcflags)));
     logv(c, format!("target-rustcflags: {}", opt_str(&config.target_rustcflags)));
@@ -238,7 +248,7 @@ pub fn test_opts(config: &Config) -> test::TestOpts {
     test::TestOpts {
         filter: match config.filter {
             None => None,
-            Some(ref filter) => Some(filter.to_strbuf()),
+            Some(ref filter) => Some(filter.clone()),
         },
         run_ignored: config.run_ignored,
         logfile: config.logfile.clone(),

src/doc/guide-testing.md

@@ -90,10 +90,15 @@ fn test_out_of_bounds_failure() {
 ~~~
 
 A test runner built with the `--test` flag supports a limited set of
-arguments to control which tests are run: the first free argument
-passed to a test runner specifies a filter used to narrow down the set
-of tests being run; the `--ignored` flag tells the test runner to run
-only tests with the `ignore` attribute.
+arguments to control which tests are run:
+
+- the first free argument passed to a test runner is interpreted as a
+  regular expression
+  ([syntax reference](regex/index.html#syntax))
+  and is used to narrow down the set of tests being run. Note: a plain
+  string is a valid regular expression that matches itself.
+- the `--ignored` flag tells the test runner to run only tests with the
+  `ignore` attribute.
 
 ## Parallelism
 
@@ -146,16 +151,31 @@ result: FAILED. 1 passed; 1 failed; 0 ignored
 
 ### Running a subset of tests
 
+Using a plain string:
+
+~~~ {.notrust}
+$ mytests mytest23
+
+running 1 tests
+running driver::tests::mytest23 ... ok
+
+result: ok. 1 passed; 0 failed; 0 ignored
+~~~
+
+Using some regular expression features:
+
 ~~~ {.notrust}
-$ mytests mytest1
+$ mytests 'mytest[145]'
 
-running 11 tests
+running 13 tests
 running driver::tests::mytest1 ... ok
+running driver::tests::mytest4 ... ok
+running driver::tests::mytest5 ... ok
 running driver::tests::mytest10 ... ignored
 ... snip ...
 running driver::tests::mytest19 ... ok
 
-result: ok. 11 passed; 0 failed; 1 ignored
+result: ok. 13 passed; 0 failed; 1 ignored
 ~~~
 
 # Microbenchmarking

src/libtest/lib.rs

@@ -37,6 +37,7 @@
 
 extern crate collections;
 extern crate getopts;
+extern crate regex;
 extern crate serialize;
 extern crate term;
 extern crate time;
@@ -45,6 +46,7 @@ use collections::TreeMap;
 use stats::Stats;
 use time::precise_time_ns;
 use getopts::{OptGroup, optflag, optopt};
+use regex::Regex;
 use serialize::{json, Decodable};
 use serialize::json::{Json, ToJson};
 use term::Terminal;
@@ -263,7 +265,7 @@ pub fn test_main_static_x(args: &[~str], tests: &[TestDescAndFn]) {
 }
 
 pub struct TestOpts {
-    pub filter: Option<StrBuf>,
+    pub filter: Option<Regex>,
     pub run_ignored: bool,
     pub run_tests: bool,
     pub run_benchmarks: bool,
@@ -324,8 +326,8 @@ fn usage(binary: &str, helpstr: &str) {
     println!("");
     if helpstr == "help" {
         println!("{}", "\
-The FILTER is matched against the name of all tests to run, and if any tests
-have a substring match, only those tests are run.
+The FILTER regex is matched against the name of all tests to run, and
+only those tests that match are run.
 
 By default, all tests are run in parallel. This can be altered with the
 RUST_TEST_TASKS environment variable when running tests (set it to 1).
@@ -372,12 +374,15 @@ pub fn parse_opts(args: &[StrBuf]) -> Option<OptRes> {
         return None;
     }
 
-    let filter =
-        if matches.free.len() > 0 {
-            Some((*matches.free.get(0)).to_strbuf())
-        } else {
-            None
-        };
+    let filter = if matches.free.len() > 0 {
+        let s = matches.free.get(0).as_slice();
+        match Regex::new(s) {
+            Ok(re) => Some(re),
+            Err(e) => return Some(Err(format_strbuf!("could not parse /{}/: {}", s, e)))
+        }
+    } else {
+        None
+    };
 
     let run_ignored = matches.opt_present("ignored");
 
@@ -945,26 +950,12 @@ pub fn filter_tests(
     let mut filtered = tests;
 
     // Remove tests that don't match the test filter
-    filtered = if opts.filter.is_none() {
-        filtered
-    } else {
-        let filter_str = match opts.filter {
-          Some(ref f) => (*f).clone(),
-          None => "".to_strbuf()
-        };
-
-        fn filter_fn(test: TestDescAndFn, filter_str: &str) ->
-            Option<TestDescAndFn> {
-            if test.desc.name.to_str().contains(filter_str) {
-                return Some(test);
-            } else {
-                return None;
-            }
+    filtered = match opts.filter {
+        None => filtered,
+        Some(ref re) => {
+            filtered.move_iter()
+                .filter(|test| re.is_match(test.desc.name.as_slice())).collect()
         }
-
-        filtered.move_iter()
-                .filter_map(|x| filter_fn(x, filter_str.as_slice()))
-                .collect()
     };
 
     // Maybe pull out the ignored test and unignore them
@@ -1451,12 +1442,12 @@ mod tests {
 
     #[test]
     fn first_free_arg_should_be_a_filter() {
-        let args = vec!("progname".to_strbuf(), "filter".to_strbuf());
+        let args = vec!("progname".to_strbuf(), "some_regex_filter".to_strbuf());
         let opts = match parse_opts(args.as_slice()) {
             Some(Ok(o)) => o,
             _ => fail!("Malformed arg in first_free_arg_should_be_a_filter")
         };
-        assert!("filter" == opts.filter.clone().unwrap().as_slice());
+        assert!(opts.filter.expect("should've found filter").is_match("some_regex_filter"))
     }
 
     #[test]
@@ -1555,6 +1546,37 @@ mod tests {
         }
     }
 
+    #[test]
+    pub fn filter_tests_regex() {
+        let mut opts = TestOpts::new();
+        opts.filter = Some(::regex::Regex::new("a.*b.+c").unwrap());
+
+        let mut names = ["yes::abXc", "yes::aXXXbXXXXc",
+                         "no::XYZ", "no::abc"];
+        names.sort();
+
+        fn test_fn() {}
+        let tests = names.iter().map(|name| {
+            TestDescAndFn {
+                desc: TestDesc {
+                    name: DynTestName(name.to_strbuf()),
+                    ignore: false,
+                    should_fail: false
+                },
+                testfn: DynTestFn(test_fn)
+            }
+        }).collect();
+        let filtered = filter_tests(&opts, tests);
+
+        let expected: Vec<&str> =
+            names.iter().map(|&s| s).filter(|name| name.starts_with("yes")).collect();
+
+        assert_eq!(filtered.len(), expected.len());
+        for (test, expected_name) in filtered.iter().zip(expected.iter()) {
+            assert_eq!(test.desc.name.as_slice(), *expected_name);
+        }
+    }
+
     #[test]
     pub fn test_metricmap_compare() {
         let mut m1 = MetricMap::new();