Merge pull request #197 from wenting-zhao/main

Handle xfail in grading
princeton-nlp · Oct 7, 2024 · 98dde0e · 98dde0e
2 parents 58d3d81 + 5b2d7ad
commit 98dde0e
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/swebench/harness/constants.py b/swebench/harness/constants.py
@@ -40,6 +40,7 @@ class TestStatus(Enum):
     PASSED = "PASSED"
     SKIPPED = "SKIPPED"
     ERROR = "ERROR"
+    XFAIL = "XFAIL"
 
 TEST_PYTEST = "pytest --no-header -rA --tb=no -p no:cacheprovider"
 TEST_PYTEST_VERBOSE = "pytest -rA --tb=long -p no:cacheprovider"

diff --git a/swebench/harness/grading.py b/swebench/harness/grading.py
@@ -21,7 +21,7 @@
 
 # MARK: Utility functions
 def test_passed(case: str, sm: dict[str, str]) -> bool:
-    return case in sm and sm[case] == TestStatus.PASSED.value
+    return case in sm and sm[case] in [TestStatus.PASSED.value, TestStatus.XFAIL.value]
 
 
 def test_failed(case: str, sm: dict[str, str]) -> bool: