chore(coverage): ignore v8 failed cases from runtime.snap (#6780)

tasks/coverage/Cargo.toml

@@ -32,7 +32,6 @@ encoding_rs = { workspace = true }
 encoding_rs_io = { workspace = true }
 futures = { workspace = true }
 lazy_static = { workspace = true }
-phf = { workspace = true, features = ["macros"] }
 pico-args = { workspace = true }
 rayon = { workspace = true }
 regex = { workspace = true }

tasks/coverage/src/runtime/.gitignore

@@ -0,0 +1 @@
+v8_test262.status

tasks/coverage/src/runtime/mod.rs

@@ -1,3 +1,5 @@
+mod test262_status;
+
 use std::{
     path::{Path, PathBuf},
     time::Duration,
@@ -13,7 +15,6 @@ use oxc::{
     transformer::{TransformOptions, Transformer},
 };
 use oxc_tasks_common::agent;
-use phf::{phf_set, Set};
 use serde_json::json;
 
 use crate::{
@@ -22,54 +23,51 @@ use crate::{
     workspace_root,
 };
 
-static SKIP_EVALUATING_FEATURES: Set<&'static str> = phf_set! {
-  // Node's version of V8 doesn't implement these
-  "hashbang",
-  "legacy-regexp",
-  "regexp-duplicate-named-groups",
-  "symbols-as-weakmap-keys",
-  "tail-call-optimization",
-
-  // We don't care about API-related things
-  "ArrayBuffer",
-  "change-array-by-copy",
-  "DataView",
-  "resizable-arraybuffer",
-  "ShadowRealm",
-  "cross-realm",
-  "SharedArrayBuffer",
-  "String.prototype.toWellFormed",
-  "Symbol.match",
-  "Symbol.replace",
-  "Symbol.unscopables",
-  "Temporal",
-  "TypedArray",
-
-  // Added in oxc
-  "Array.fromAsync",
-  "IsHTMLDDA",
-  "iterator-helpers",
-  "set-methods",
-  "array-grouping",
-
-  // stage 2
-  "Intl.DurationFormat"
-};
-
-static SKIP_EVALUATING_THESE_INCLUDES: Set<&'static str> = phf_set! {
+use test262_status::get_v8_test262_failure_paths;
+
+static SKIP_EVALUATING_FEATURES: &[&str] = &[
+    // Node's version of V8 doesn't implement these
+    "hashbang",
+    "legacy-regexp",
+    "regexp-duplicate-named-groups",
+    "symbols-as-weakmap-keys",
+    "tail-call-optimization",
+    // We don't care about API-related things
+    "ArrayBuffer",
+    "change-array-by-copy",
+    "DataView",
+    "resizable-arraybuffer",
+    "ShadowRealm",
+    "cross-realm",
+    "SharedArrayBuffer",
+    "String.prototype.toWellFormed",
+    "Symbol.match",
+    "Symbol.replace",
+    "Symbol.unscopables",
+    "Temporal",
+    "TypedArray",
+    // Added in oxc
+    "Array.fromAsync",
+    "IsHTMLDDA",
+    "iterator-helpers",
+    "set-methods",
+    "array-grouping",
+    // stage 2
+    "Intl.DurationFormat",
+];
+
+static SKIP_EVALUATING_THESE_INCLUDES: &[&str] = &[
     // We don't preserve "toString()" on functions
     "nativeFunctionMatcher.js",
-};
+];
 
-static SKIP_TEST_CASES: Set<&'static str> = phf_set! {
+static SKIP_TEST_CASES: &[&str] = &[
     // For some unknown reason these tests are unstable, so we'll skip them for now.
     "language/identifiers/start-unicode",
     // Properly misconfigured test setup for `eval`, but can't figure out where
     "annexB/language/eval-code",
-    "language/eval-code"
-};
-
-const FIXTURES_PATH: &str = "test262/test";
+    "language/eval-code",
+];
 
 pub struct Test262RuntimeCase {
     base: Test262Case,
@@ -78,7 +76,7 @@ pub struct Test262RuntimeCase {
 
 impl Case for Test262RuntimeCase {
     fn new(path: PathBuf, code: String) -> Self {
-        Self { base: Test262Case::new(path, code), test_root: workspace_root().join(FIXTURES_PATH) }
+        Self { base: Test262Case::new(path, code), test_root: workspace_root() }
     }
 
     fn code(&self) -> &str {
@@ -95,23 +93,18 @@ impl Case for Test262RuntimeCase {
 
     fn skip_test_case(&self) -> bool {
         let base_path = self.base.path().to_string_lossy();
+        let includes = &self.base.meta().includes;
+        let features = &self.base.meta().features;
         self.base.should_fail()
             || self.base.skip_test_case()
             || base_path.contains("built-ins")
             || base_path.contains("staging")
             || base_path.contains("intl402")
-            || self
-                .base
-                .meta()
-                .includes
-                .iter()
-                .any(|include| SKIP_EVALUATING_THESE_INCLUDES.contains(include))
-            || self
-                .base
-                .meta()
-                .features
+            || get_v8_test262_failure_paths().iter().any(|test| base_path.contains(test))
+            || includes
                 .iter()
-                .any(|feature| SKIP_EVALUATING_FEATURES.contains(feature))
+                .any(|include| SKIP_EVALUATING_THESE_INCLUDES.contains(&include.as_ref()))
+            || features.iter().any(|feature| SKIP_EVALUATING_FEATURES.contains(&feature.as_ref()))
             || SKIP_TEST_CASES.iter().any(|path| base_path.contains(path))
             || self.base.code().contains("$262")
             || self.base.code().contains("$DONOTEVALUATE()")

tasks/coverage/src/runtime/runtime.js

@@ -15,6 +15,12 @@ async function runCodeInHarness(options = {}) {
   const { code = '', includes = [], importDir = '', isAsync = false, isModule = true, isRaw = false } = options;
   const context = {};
 
+  if (process.env.DEBUG) {
+    const { code: c, ...o } = options;
+    console.log(c);
+    console.log(o);
+  }
+
   // See: https://github.com/nodejs/node/issues/36351
   const unique = () => '//' + Math.random();
 

tasks/coverage/src/runtime/test262_status.rs

@@ -0,0 +1,44 @@
+use std::{fs, sync::OnceLock, time::Duration};
+
+use oxc_tasks_common::agent;
+
+use crate::workspace_root;
+
+/// Generate v8 test262 status file, which is used to skip failed tests
+/// see <https://chromium.googlesource.com/v8/v8/+/refs/heads/main/test/test262/test262.status>
+///
+/// # Panics
+pub fn get_v8_test262_failure_paths() -> &'static Vec<String> {
+    static STATUS: OnceLock<Vec<String>> = OnceLock::new();
+    STATUS.get_or_init(|| {
+        let path = workspace_root().join("src/runtime/v8_test262.status");
+
+        let lines = if path.exists() {
+            fs::read_to_string(&path).unwrap().lines().map(ToString::to_string).collect::<Vec<_>>()
+        } else {
+            let res = agent()
+                .get("http://raw.githubusercontent.com/v8/v8/main/test/test262/test262.status")
+                .timeout(Duration::from_secs(10))
+                .call()
+                .unwrap()
+                .into_string()
+                .unwrap();
+            let mut tests = vec![];
+            regex::Regex::new(r"'(.+)': \[(FAIL|SKIP)\]").unwrap().captures_iter(&res).for_each(
+                |caps| {
+                    if let Some(name) = caps.get(1).map(|f| f.as_str()) {
+                        if !name.eq("*") {
+                            tests.push(name.to_string());
+                        }
+                    }
+                },
+            );
+            tests.sort_unstable();
+            tests.dedup();
+            fs::write(path, tests.join("\n")).unwrap();
+            tests
+        };
+
+        lines
+    })
+}