diff --git a/CHANGELOG.md b/CHANGELOG.md index 551870d6dd7..d4e429cdffe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## **[Unreleased]** +- [#1234](https://github.com/wasmerio/wasmer/pull/1234) Check for unused excluded spectest failures. - [#1232](https://github.com/wasmerio/wasmer/pull/1232) `wasmer-interface-types` has a WAT decoder. ## 0.14.0 - 2020-02-20 diff --git a/lib/spectests/tests/excludes.txt b/lib/spectests/tests/excludes.txt index b6690cc99d2..8d1ccdaa7a9 100644 --- a/lib/spectests/tests/excludes.txt +++ b/lib/spectests/tests/excludes.txt @@ -267,10 +267,6 @@ clif:fail:data.wast:186:windows # AssertUnlinkable - caught panic Any clif:fail:data.wast:194:windows # AssertUnlinkable - caught panic Any # LLVM -llvm:fail:f32.wast:1621 # AssertReturn - result F32(0) ("0x0") does not match expected F32(2147483648) ("0x80000000") -llvm:fail:f32.wast:2020 # AssertReturn - result F32(2147483648) ("0x80000000") does not match expected F32(0) ("0x0") -llvm:fail:f64.wast:1621 # AssertReturn - result F64(0) ("0x0") does not match expected F64(9223372036854775808) ("0x8000000000000000") -llvm:fail:f64.wast:2020 # AssertReturn - result F64(9223372036854775808) ("0x8000000000000000") does not match expected F64(0) ("0x0") llvm:fail:linking.wast:388 # AssertReturn - Call failed RuntimeError: WebAssembly trap occurred during runtime: incorrect `call_indirect` signature # LLVM AArch64 @@ -312,48 +308,65 @@ singlepass:skip:simd_binaryen.wast:* # SIMD not implemented singlepass:skip:atomic.wast:*:*:aarch64 # Threads not yet supported on singlepass -singlepass:fail:address.wast:194 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:195 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:196 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:197 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:198 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:200 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:201 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:202 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:203 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:204 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:481 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:482 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:483 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:484 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:485 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:486 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:487 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:489 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:490 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:491 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:492 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:493 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:494 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:495 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:541 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:542 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:588 # AssertTrap - expected trap, got [] -singlepass:fail:address.wast:589 # AssertTrap - expected trap, got [] -singlepass:fail:globals.wast:243 # AssertInvalid - Should be invalid -singlepass:fail:linking.wast:134 # AssertReturn - Call failed RuntimeError: unknown error -singlepass:fail:linking.wast:137 # AssertTrap - expected trap, got Runtime:Error unknown error -singlepass:fail:linking.wast:139 # AssertTrap - expected trap, got Runtime:Error unknown error -singlepass:fail:linking.wast:142 # AssertTrap - expected trap, got Runtime:Error unknown error -singlepass:fail:linking.wast:144 # AssertTrap - expected trap, got Runtime:Error unknown error -singlepass:fail:linking.wast:147 # AssertTrap - expected trap, got Runtime:Error unknown error -singlepass:fail:linking.wast:149 # AssertTrap - expected trap, got Runtime:Error unknown error -singlepass:fail:linking.wast:169 # AssertReturn - Call failed RuntimeError: unknown error -singlepass:fail:linking.wast:175 # AssertReturn - Call failed RuntimeError: unknown error -singlepass:fail:linking.wast:181 # AssertReturn - Call failed RuntimeError: unknown error -singlepass:fail:linking.wast:185 # AssertTrap - expected trap, got Runtime:Error unknown error -singlepass:fail:linking.wast:187 # AssertTrap - expected trap, got Runtime:Error unknown error -singlepass:fail:linking.wast:388 # AssertReturn - Call failed RuntimeError: unknown error +singlepass:fail:address.wast:200:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:201:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:202:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:203:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:204:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:489:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:490:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:491:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:492:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:495:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:542:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:589:linux:x86_64 # AssertTrap - expected trap, got [] +singlepass:fail:linking.wast:137:linux # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:139:linux # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:142:linux # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:144:linux # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:147:linux # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:149:linux # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:185:linux # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:187:linux # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:388:linux # AssertReturn - Call failed RuntimeError: unknown error + +singlepass:fail:address.wast:194:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:195:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:196:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:197:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:198:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:200:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:201:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:202:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:203:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:204:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:481:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:482:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:483:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:484:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:485:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:486:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:487:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:489:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:490:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:491:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:492:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:493:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:494:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:495:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:541:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:542:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:588:macos # AssertTrap - expected trap, got [] +singlepass:fail:address.wast:589:macos # AssertTrap - expected trap, got [] +singlepass:fail:linking.wast:137:macos # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:139:macos # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:142:macos # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:144:macos # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:147:macos # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:149:macos # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:185:macos # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:187:macos # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:linking.wast:388:macos # AssertReturn - Call failed RuntimeError: unknown error # These failures only happen on AArch64 and not on x86-64. singlepass:fail:conversions.wast:83:*:aarch64 # AssertTrap - expected trap, got [I32(2147483647)] diff --git a/lib/spectests/tests/spectest.rs b/lib/spectests/tests/spectest.rs index f8b7939f372..f958f12585c 100644 --- a/lib/spectests/tests/spectest.rs +++ b/lib/spectests/tests/spectest.rs @@ -48,12 +48,21 @@ mod tests { &mut self, failure: SpecFailure, _testkey: &str, - excludes: &Vec, + excludes: &mut Vec>, line: u64, ) { - if excludes - .iter() - .any(|e| e.line_matches(line) && e.exclude_kind == ExcludeKind::Fail) + // Ensure that each exclude is only used once. + if let Some(_) = excludes + .iter_mut() + .find(|e| { + if let Some(ref e) = e { + e.line_matches(line) && e.exclude_kind == ExcludeKind::Fail + } else { + false + } + }) + .take() + .and_then(|x| x.take()) { self.allowed_failure += 1; return; @@ -88,6 +97,21 @@ mod tests { "windows" } + #[cfg(target_os = "linux")] + fn get_target_os() -> &'static str { + "linux" + } + + #[cfg(target_os = "macos")] + fn get_target_os() -> &'static str { + "macos" + } + + #[cfg(target_os = "windows")] + fn get_target_os() -> &'static str { + "windows" + } + fn get_target_arch() -> &'static str { if cfg!(target_arch = "x86_64") { "x86_64" @@ -110,6 +134,7 @@ mod tests { // clif:skip:data.wast:172:unix:x86 #[allow(dead_code)] + #[derive(Clone)] struct Exclude { backend: Option, exclude_kind: ExcludeKind, @@ -261,13 +286,11 @@ mod tests { let mut named_modules: HashMap>> = HashMap::new(); let mut registered_modules: HashMap>> = HashMap::new(); - // - let empty_excludes = vec![]; - let excludes = if excludes.contains_key(filename) { - excludes.get(filename).unwrap() - } else { - &empty_excludes - }; + let mut excludes: Vec<_> = excludes + .get(filename) + .map(|file| file.iter().map(|x| Some(x.clone())).collect()) + .unwrap_or(vec![]); + let excludes = &mut excludes; let backend = get_compiler_name(); @@ -281,6 +304,7 @@ mod tests { // Skip tests that match this line if excludes .iter() + .filter_map(|x| x.as_ref()) .any(|e| e.line_exact_match(line) && e.exclude_kind == ExcludeKind::Skip) { continue; @@ -1074,6 +1098,21 @@ mod tests { }, } } + + // Check for unused excludes. + for excl in excludes { + if let Some(ref excl) = *excl { + if excl.exclude_kind == ExcludeKind::Fail { + test_report.failed += 1; + test_report.failures.push(SpecFailure { + file: filename.to_string(), + line: excl.line.unwrap_or(0), + kind: format!("{}", "Exclude"), + message: format!("Excluded failure did not occur"), + }); + } + } + } Ok(test_report) } @@ -1235,6 +1274,7 @@ mod tests { let mut result = HashMap::new(); let mut file_excludes = HashSet::new(); let current_backend = get_compiler_name(); + let current_target_os = get_target_os(); let current_target_family = get_target_family(); let current_target_arch = get_target_arch(); @@ -1283,7 +1323,8 @@ mod tests { }; if exclude.matches_backend(current_backend) - && exclude.matches_target_family(current_target_family) + && (exclude.matches_target_family(current_target_os) + || exclude.matches_target_family(current_target_family)) && exclude.matches_target_arch(current_target_arch) { // Skip the whole file for line * and skip