Update grading.py to handle XFAIL

princeton-nlp · Aug 9, 2024 · 5b2d7ad · 5b2d7ad
1 parent 1a6eee3
commit 5b2d7ad
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/swebench/harness/grading.py b/swebench/harness/grading.py
@@ -21,7 +21,7 @@
 
 # MARK: Utility functions
 def test_passed(case: str, sm: dict[str, str]) -> bool:
-    return case in sm and sm[case] == TestStatus.PASSED.value
+    return case in sm and sm[case] in [TestStatus.PASSED.value, TestStatus.XFAIL.value]
 
 
 def test_failed(case: str, sm: dict[str, str]) -> bool: