Merge pull request #1234 from wasmerio/feature/spectest-check-excluded

Check for unused excluded spectest failures.
wasmerio · Feb 21, 2020 · f632b81 · f632b81
2 parents 210eeae + f717d4c
commit f632b81
Show file tree

Hide file tree

Showing 3 changed files with 113 additions and 58 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## **[Unreleased]**
 
+- [#1234](https://github.com/wasmerio/wasmer/pull/1234) Check for unused excluded spectest failures.
 - [#1232](https://github.com/wasmerio/wasmer/pull/1232) `wasmer-interface-types` has a WAT decoder.
 
 ## 0.14.0 - 2020-02-20

diff --git a/lib/spectests/tests/excludes.txt b/lib/spectests/tests/excludes.txt
@@ -267,10 +267,6 @@ clif:fail:data.wast:186:windows # AssertUnlinkable - caught panic Any
 clif:fail:data.wast:194:windows # AssertUnlinkable - caught panic Any
 
 # LLVM
-llvm:fail:f32.wast:1621 # AssertReturn - result F32(0) ("0x0") does not match expected F32(2147483648) ("0x80000000")
-llvm:fail:f32.wast:2020 # AssertReturn - result F32(2147483648) ("0x80000000") does not match expected F32(0) ("0x0")
-llvm:fail:f64.wast:1621 # AssertReturn - result F64(0) ("0x0") does not match expected F64(9223372036854775808) ("0x8000000000000000")
-llvm:fail:f64.wast:2020 # AssertReturn - result F64(9223372036854775808) ("0x8000000000000000") does not match expected F64(0) ("0x0")
 llvm:fail:linking.wast:388 # AssertReturn - Call failed RuntimeError: WebAssembly trap occurred during runtime: incorrect `call_indirect` signature
 
 # LLVM AArch64
@@ -312,48 +308,65 @@ singlepass:skip:simd_binaryen.wast:* # SIMD not implemented
 
 singlepass:skip:atomic.wast:*:*:aarch64 # Threads not yet supported on singlepass
 
-singlepass:fail:address.wast:194 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:195 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:196 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:197 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:198 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:200 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:201 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:202 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:203 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:204 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:481 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:482 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:483 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:484 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:485 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:486 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:487 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:489 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:490 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:491 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:492 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:493 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:494 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:495 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:541 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:542 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:588 # AssertTrap - expected trap, got []
-singlepass:fail:address.wast:589 # AssertTrap - expected trap, got []
-singlepass:fail:globals.wast:243 # AssertInvalid - Should be invalid
-singlepass:fail:linking.wast:134 # AssertReturn - Call failed RuntimeError: unknown error
-singlepass:fail:linking.wast:137 # AssertTrap - expected trap, got Runtime:Error unknown error
-singlepass:fail:linking.wast:139 # AssertTrap - expected trap, got Runtime:Error unknown error
-singlepass:fail:linking.wast:142 # AssertTrap - expected trap, got Runtime:Error unknown error
-singlepass:fail:linking.wast:144 # AssertTrap - expected trap, got Runtime:Error unknown error
-singlepass:fail:linking.wast:147 # AssertTrap - expected trap, got Runtime:Error unknown error
-singlepass:fail:linking.wast:149 # AssertTrap - expected trap, got Runtime:Error unknown error
-singlepass:fail:linking.wast:169 # AssertReturn - Call failed RuntimeError: unknown error
-singlepass:fail:linking.wast:175 # AssertReturn - Call failed RuntimeError: unknown error
-singlepass:fail:linking.wast:181 # AssertReturn - Call failed RuntimeError: unknown error
-singlepass:fail:linking.wast:185 # AssertTrap - expected trap, got Runtime:Error unknown error
-singlepass:fail:linking.wast:187 # AssertTrap - expected trap, got Runtime:Error unknown error
-singlepass:fail:linking.wast:388 # AssertReturn - Call failed RuntimeError: unknown error
+singlepass:fail:address.wast:200:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:201:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:202:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:203:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:204:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:489:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:490:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:491:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:492:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:495:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:542:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:589:linux:x86_64 # AssertTrap - expected trap, got []
+singlepass:fail:linking.wast:137:linux # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:139:linux # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:142:linux # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:144:linux # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:147:linux # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:149:linux # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:185:linux # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:187:linux # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:388:linux # AssertReturn - Call failed RuntimeError: unknown error
+
+singlepass:fail:address.wast:194:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:195:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:196:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:197:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:198:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:200:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:201:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:202:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:203:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:204:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:481:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:482:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:483:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:484:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:485:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:486:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:487:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:489:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:490:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:491:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:492:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:493:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:494:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:495:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:541:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:542:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:588:macos # AssertTrap - expected trap, got []
+singlepass:fail:address.wast:589:macos # AssertTrap - expected trap, got []
+singlepass:fail:linking.wast:137:macos # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:139:macos # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:142:macos # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:144:macos # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:147:macos # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:149:macos # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:185:macos # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:187:macos # AssertTrap - expected trap, got Runtime:Error unknown error
+singlepass:fail:linking.wast:388:macos # AssertReturn - Call failed RuntimeError: unknown error
 
 # These failures only happen on AArch64 and not on x86-64. 
 singlepass:fail:conversions.wast:83:*:aarch64 # AssertTrap - expected trap, got [I32(2147483647)]

diff --git a/lib/spectests/tests/spectest.rs b/lib/spectests/tests/spectest.rs
@@ -48,12 +48,21 @@ mod tests {
             &mut self,
             failure: SpecFailure,
             _testkey: &str,
-            excludes: &Vec<Exclude>,
+            excludes: &mut Vec<Option<Exclude>>,
             line: u64,
         ) {
-            if excludes
-                .iter()
-                .any(|e| e.line_matches(line) && e.exclude_kind == ExcludeKind::Fail)
+            // Ensure that each exclude is only used once.
+            if let Some(_) = excludes
+                .iter_mut()
+                .find(|e| {
+                    if let Some(ref e) = e {
+                        e.line_matches(line) && e.exclude_kind == ExcludeKind::Fail
+                    } else {
+                        false
+                    }
+                })
+                .take()
+                .and_then(|x| x.take())
             {
                 self.allowed_failure += 1;
                 return;
@@ -88,6 +97,21 @@ mod tests {
         "windows"
     }
 
+    #[cfg(target_os = "linux")]
+    fn get_target_os() -> &'static str {
+        "linux"
+    }
+
+    #[cfg(target_os = "macos")]
+    fn get_target_os() -> &'static str {
+        "macos"
+    }
+
+    #[cfg(target_os = "windows")]
+    fn get_target_os() -> &'static str {
+        "windows"
+    }
+
     fn get_target_arch() -> &'static str {
         if cfg!(target_arch = "x86_64") {
             "x86_64"
@@ -110,6 +134,7 @@ mod tests {
 
     //  clif:skip:data.wast:172:unix:x86
     #[allow(dead_code)]
+    #[derive(Clone)]
     struct Exclude {
         backend: Option<String>,
         exclude_kind: ExcludeKind,
@@ -261,13 +286,11 @@ mod tests {
         let mut named_modules: HashMap<String, Arc<Mutex<Instance>>> = HashMap::new();
 
         let mut registered_modules: HashMap<String, Arc<Mutex<Instance>>> = HashMap::new();
-        //
-        let empty_excludes = vec![];
-        let excludes = if excludes.contains_key(filename) {
-            excludes.get(filename).unwrap()
-        } else {
-            &empty_excludes
-        };
+        let mut excludes: Vec<_> = excludes
+            .get(filename)
+            .map(|file| file.iter().map(|x| Some(x.clone())).collect())
+            .unwrap_or(vec![]);
+        let excludes = &mut excludes;
 
         let backend = get_compiler_name();
 
@@ -281,6 +304,7 @@ mod tests {
             // Skip tests that match this line
             if excludes
                 .iter()
+                .filter_map(|x| x.as_ref())
                 .any(|e| e.line_exact_match(line) && e.exclude_kind == ExcludeKind::Skip)
             {
                 continue;
@@ -1074,6 +1098,21 @@ mod tests {
                 },
             }
         }
+
+        // Check for unused excludes.
+        for excl in excludes {
+            if let Some(ref excl) = *excl {
+                if excl.exclude_kind == ExcludeKind::Fail {
+                    test_report.failed += 1;
+                    test_report.failures.push(SpecFailure {
+                        file: filename.to_string(),
+                        line: excl.line.unwrap_or(0),
+                        kind: format!("{}", "Exclude"),
+                        message: format!("Excluded failure did not occur"),
+                    });
+                }
+            }
+        }
         Ok(test_report)
     }
 
@@ -1235,6 +1274,7 @@ mod tests {
         let mut result = HashMap::new();
         let mut file_excludes = HashSet::new();
         let current_backend = get_compiler_name();
+        let current_target_os = get_target_os();
         let current_target_family = get_target_family();
         let current_target_arch = get_target_arch();
 
@@ -1283,7 +1323,8 @@ mod tests {
                 };
 
                 if exclude.matches_backend(current_backend)
-                    && exclude.matches_target_family(current_target_family)
+                    && (exclude.matches_target_family(current_target_os)
+                        || exclude.matches_target_family(current_target_family))
                     && exclude.matches_target_arch(current_target_arch)
                 {
                     // Skip the whole file for line * and skip