Start lightweight client-side validation of formats #666

python-client/tests/format_check/__init__.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+RESOURCES = Path(__file__).parent.parent / "resources"
+VALID_RUN_OUTPUT = RESOURCES / "ranking-outputs"
+EMPTY_OUTPUT = RESOURCES / "input-run-01" / "1"
+IR_QUERY_OUTPUT = RESOURCES / "query-processing-outputs" / "query-segmentation"

python-client/tests/format_check/test_check_format_for_non_existing_formats.py

@@ -0,0 +1,31 @@
+import unittest
+
+from tira.check_format import check_format
+
+from . import EMPTY_OUTPUT, IR_QUERY_OUTPUT, VALID_RUN_OUTPUT
+
+
+class TestCheckFormatForNonExistingFormats(unittest.TestCase):
+    def test_invalid_validator_on_empty_output(self):
+        with self.assertRaises(Exception):
+            check_format(EMPTY_OUTPUT, "does-not-exist")
+
+    def test_invalid_validator_on_query_output(self):
+        with self.assertRaises(Exception):
+            check_format(IR_QUERY_OUTPUT, "does-not-exist")
+
+    def test_invalid_validator_on_valid_run_output_output(self):
+        with self.assertRaises(Exception):
+            check_format(VALID_RUN_OUTPUT, "does-not-exist")
+
+    def test_multiple_invalid_validators_on_empty_output(self):
+        with self.assertRaises(Exception):
+            check_format(EMPTY_OUTPUT, ["d1", "d2"])
+
+    def test_multiple_invalid_validators_on_query_output(self):
+        with self.assertRaises(Exception):
+            check_format(IR_QUERY_OUTPUT, ["d1", "d2"])
+
+    def test_multiple_invalid_validators_on_valid_run_output_output(self):
+        with self.assertRaises(Exception):
+            check_format(VALID_RUN_OUTPUT, ["d1", "d2"])

python-client/tests/format_check/test_check_format_for_run_file_format.py

@@ -0,0 +1,22 @@
+import unittest
+
+from tira.check_format import check_format
+
+from . import EMPTY_OUTPUT, IR_QUERY_OUTPUT, VALID_RUN_OUTPUT
+
+
+class TestCheckFormatForNonExistingFormats(unittest.TestCase):
+    def test_invalid_validator_on_empty_output(self):
+        expected = ["ERROR", "No file run.txt was found, only the files ['.gitkeep'] were available."]
+        actual = check_format(EMPTY_OUTPUT, "run.txt")
+        self.assertEqual(expected, actual)
+
+    def test_invalid_validator_on_query_output(self):
+        expected = ["ERROR", "No file run.txt was found, only the files ['queries.jsonl'] were available."]
+        actual = check_format(IR_QUERY_OUTPUT, "run.txt")
+        self.assertEqual(expected, actual)
+
+    def test_invalid_validator_on_valid_run_output_output(self):
+        expected = ["OK", "The run.txt file has the correct format."]
+        actual = check_format(VALID_RUN_OUTPUT, "run.txt")
+        self.assertEqual(expected, actual)

python-client/tira/check_format.py

@@ -0,0 +1,28 @@
+import os
+from pathlib import Path
+from typing import Sequence, Union
+
+
+class RunFormat:
+    """Checks if a given output is a valid run file."""
+
+    def check_format(self, run_output: Path):
+        if not (run_output / "run.txt").exists():
+            msg = "No file run.txt was found, only the files "
+            msg += str(os.listdir(run_output)) + " were available."
+            return ["ERROR", msg]
+        else:
+            return ["OK", "The run.txt file has the correct format."]
+
+
+def check_format(run_output: Path, format: Union[str, Sequence[str]]):
+    """Check if the provided run output is in the specified format. Provides debug messages intended for users.
+
+    Args:
+        format (Union[str, Sequence[str]]): The allowed format or a list of allowed formats.
+        run_output (Path): the output produced by some run that is to-be checked.
+    """
+    if format == "run.txt":
+        return RunFormat().check_format(run_output)
+
+    raise ValueError("Not yet implemented.", run_output, format)