Upgrade to version 2 spec

bin/parse_tests.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+
+import json
+import re
+import sys
+from enum import Enum
+
+
+class ExitCode(Enum):
+    PASS = 0
+    FAIL = 1
+    ERROR = 2
+
+
+class Status(Enum):
+    PASS = "pass"
+    FAIL = "fail"
+    ERROR = "error"
+
+
+TEST_FUNCTION = "ert-deftest"
+TEST_FAILED_FUNCTION = "ert-test-failed"
+
+
+def parse_test_functions(s: str):
+    """
+    Retrieve test function names and code from regions like
+        '(ert-deftest name-is-persistent
+            "Test that robot name is persistent."
+            (should (equal (robot-name *robbie*)
+                        (robot-name *robbie*))))'
+    in the test file
+    """
+    function_matches = re.finditer(
+        fr"\({TEST_FUNCTION}\s+(?P<name>[\w-]+)\s+\(\)\s*(?P<docstring>\".*\")?\s*(?P<code>(?:\n.+)+)\)",
+        s,
+    )
+    names = []
+    code_pieces = []
+    for m in function_matches:
+        names.append(m["name"])
+        code_pieces.append(m["code"].strip())
+    return names, code_pieces
+
+
+def parse_test_statuses(s: str):
+    """
+    Retrieve test statuses from lines like
+        'passed  3/4  name-is-persistent (0.000049 sec)'
+    in the test output
+    """
+    status_matches = re.finditer(
+        r"\s+(?P<status>passed|FAILED)\s+(?P<number>\d+)/\d+\s+(?P<name>[\w-]+)\s*\(.*\)",
+        s,
+    )
+    return {
+        m["name"]: (
+            Status.PASS if m["status"].strip() == "passed" else Status.FAIL
+        )
+        for m in status_matches
+    }
+
+
+def parse_test_message(name: str, s: str):
+    """
+    Retrieve test messages from regions like
+        'Test name-can-be-reset condition:
+            (wrong-type-argument hash-table-p nil)
+            FAILED  2/4  name-can-be-reset'
+    in the test output
+    """
+    condition_matches = re.finditer(
+        fr"Test\s{name}\scondition:\s+(?P<condition>\((?P<function>.+)(?:\n.+)+)FAILED\s+(?P<number>\d+)/\d+\s+{name}",
+        s,
+    )
+    try:
+        cond_match = next(condition_matches)
+    except StopIteration:
+        return None, None
+    message = cond_match["condition"].strip()
+    # status is 'fail' if test condition starts with the test failed function
+    # otherwise there is an error
+    status = (
+        Status.FAIL
+        if cond_match["function"] == TEST_FAILED_FUNCTION
+        else Status.ERROR
+    )
+    return message, status
+
+
+def parse_test_output(name: str, num: int, s: str):
+    """
+    Retrieve test outputs from regions like
+        'Running 4 tests (2022-01-04 17:06:51+0200, selector ‘t’)
+        "1DG190"
+            passed  1/4  different-robots-have-different-names (0.000075 sec)'
+    ,
+        '   passed  1/4  different-robots-have-different-names (0.000075 sec)
+        "1XW454"
+            passed  2/4  name-can-be-reset (0.000047 sec)'
+    and
+        '   passed  3/4  name-is-persistent (0.000049 sec)
+        "1DG190"
+        Test name-matches-expected-pattern backtrace:'
+    in the test output
+    """
+    status_line_regexp = (
+        fr"(?:\s+(?:passed|FAILED)\s+{num - 1}/\d+\s+(?:[\w-]+)\s*\(.*)"
+    )
+    output_regexp = fr"\)\n*(?P<output>(?:\n.*)+)\s*(?:passed\s+{num}|Test\s{name}\sbacktrace)"
+    output_matches = re.finditer(
+        ("" if num == 1 else status_line_regexp) + output_regexp, s
+    )
+    try:
+        output_match = next(output_matches)
+    except StopIteration:
+        return None, None
+    output = output_match["output"].strip()
+    message = None
+    # Output is limited to 500 chars
+    if len(output) > 500:
+        message = "Output was truncated. Please limit to 500 chars"
+        output = output[:500]
+    return output, message
+
+
+def run(test_file_path: str, test_output_file_path: str):
+    exit_code = ExitCode.PASS
+    with open(test_file_path, encoding="utf-8") as f:
+        test_file_content = f.read()
+    with open(test_output_file_path, encoding="utf-8") as f:
+        test_output_file_content = f.read()
+    names, code_pieces = parse_test_functions(test_file_content)
+    name_to_number = {name: i + 1 for i, name in enumerate(sorted(names))}
+    name_to_status = parse_test_statuses(test_output_file_content)
+    status_to_exit_code = {Status(ec.name.lower()): ec for ec in ExitCode}
+    tests = []
+    for name, code in zip(names, code_pieces):
+        test = {}
+        number = name_to_number[name]
+        test["name"] = name
+        test["test_code"] = code.strip()
+        # get status from status line or assume it is syntax error if there is no one
+        status = name_to_status.get(name, Status.ERROR)
+        exit_code = max(
+            exit_code, status_to_exit_code[status], key=lambda x: x.value
+        )
+        message = None
+        condition_message, message_status = parse_test_message(
+            name, test_output_file_content
+        )
+        if condition_message:
+            message, status = condition_message, message_status
+        output, output_message = parse_test_output(
+            name, int(number), test_output_file_content
+        )
+        if output_message and status != Status.PASS:
+            if message:
+                message += "\n" + output_message
+            else:
+                message = output_message
+        test["status"] = status.value
+        if message:
+            test["message"] = message
+        if output:
+            test["output"] = output
+        tests.append(test)
+    print(json.dumps(tests))
+    return exit_code
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        print("./parse-tests.py <test-file> <test-output>", file=sys.stderr)
+        sys.exit(ExitCode.ERROR.value)
+    else:
+        exit_code = run(*sys.argv[1:])
+        sys.exit(exit_code.value)

bin/run.sh

@@ -22,9 +22,11 @@ if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
 fi
 
 slug="$1"
+script_dir="$(dirname "$0")"
 input_dir="${2%/}"
 output_dir="${3%/}"
 test_file="${input_dir}/${slug}-test.el"
+test_output_file="$(mktemp --suffix ".out")"
 results_file="${output_dir}/results.json"
 
 # Create the output directory if it doesn't exist
@@ -34,24 +36,26 @@ echo "${slug}: testing..."
 
 pushd "${input_dir}" > /dev/null
 
-# Run the tests for the provided implementation file and redirect stdout and
-# stderr to capture it
-test_output=$(emacs -batch -l ert -l "${test_file}" -f ert-run-tests-batch-and-exit 2>&1)
-exit_code=$?
+# Run the tests for the provided implementation file and record all terminal
+# output to a temporary file to preserve output order
+script -c "emacs -batch -l ert -l \"${test_file}\" -f ert-run-tests-batch-and-exit" \
+    -O "$test_output_file" &> /dev/null
 
 popd > /dev/null
 
-# Write the results.json file based on the exit code of the command that was
-# just executed that tested the implementation file
-if [ $exit_code -eq 0 ]; then
-    jq -n '{version: 1, status: "pass"}' > ${results_file}
-else
-    # Manually add colors to the output to help scanning the output for errors
-    colorized_test_output=$(echo "${test_output}" \
-         | GREP_COLOR='01;31' grep --color=always -E -e 'FAILED.*$|$' \
-         | GREP_COLOR='01;32' grep --color=always -E -e 'passed.*$|$')
-
-    jq -n --arg output "${colorized_test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
-fi
+# Write the results.json file based on both the exit code of the command that
+# was just executed that tested the implementation file and per-test information
+tests=$("$script_dir/parse_tests.py" "$test_file" "$test_output_file")
+exit_code=$?
+case $exit_code in
+    0) status="pass" ;;
+    1) status="fail" ;;
+    2) status="error" ;;
+    *) echo "'parse_tests.py' script returned unknown exit code" 1>&2 && exit 1 ;;
+esac
+
+jq -n --arg status "$status" \
+    --argjson tests "$tests" \
+    '{version: 2, status: $status, message: null, tests: $tests}' > ${results_file}
 
 echo "${slug}: done"

tests/example-all-fail/example-all-fail.el

@@ -6,9 +6,9 @@
 
 (defun leap-year-p (year)
   "Determine if YEAR is a leap year."
-    (not ((and (= 0 (mod year 4))
+  (not (and (= 0 (mod year 4))
             (or (not (= 0 (mod year 100)))
-               (= 0 (mod year 401)))))))
+                (= 0 (mod year 400))))))
 
 (provide 'leap)
 ;;; leap.el ends here

tests/example-all-fail/expected_results.json

@@ -1,5 +1,37 @@
 {
-  "version": 1,
+  "version": 2,
   "status": "fail",
-  "message": "Loading /solution/example-all-fail.el (source)...\nRunning 5 tests \nTest any-old-year backtrace:\n  ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (mod ye\n  (not ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (m\n  leap-year-p(1997)\n  apply(leap-year-p 1997)\n  (setq value-7 (apply fn-5 args-6))\n  (unwind-protect (setq value-7 (apply fn-5 args-6)) (setq form-descri\n  (not (unwind-protect (setq value-7 (apply fn-5 args-6)) (setq form-d\n  (if (not (unwind-protect (setq value-7 (apply fn-5 args-6)) (setq fo\n  (let (form-description-9) (if (not (unwind-protect (setq value-7 (ap\n  (let ((value-7 (quote ert-form-evaluation-aborted-8))) (let (form-de\n  (let* ((fn-5 (function leap-year-p)) (args-6 (condition-case err (le\n  (lambda nil (let* ((fn-5 (function leap-year-p)) (args-6 (condition-\n  ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n  ert-run-test(#s(ert-test :name any-old-year :documentation nil :body\n  ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n  ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n  ert-run-tests-batch(nil)\n  ert-run-tests-batch-and-exit()\n  command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-all-\n  command-line()\n  normal-top-level()\nTest any-old-year condition:\n    (invalid-function\n     (and\n      (= 0\n\t (mod year 4))\n      (or\n       (not\n\t(= 0 ...))\n       (= 0\n\t  (mod year 401)))))\n   \u001b[01;31m\u001b[KFAILED  1/5  any-old-year\u001b[m\u001b[K\nTest century backtrace:\n  ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (mod ye\n  (not ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (m\n  leap-year-p(1900)\n  apply(leap-year-p 1900)\n  (setq value-17 (apply fn-15 args-16))\n  (unwind-protect (setq value-17 (apply fn-15 args-16)) (setq form-des\n  (not (unwind-protect (setq value-17 (apply fn-15 args-16)) (setq for\n  (if (not (unwind-protect (setq value-17 (apply fn-15 args-16)) (setq\n  (let (form-description-19) (if (not (unwind-protect (setq value-17 (\n  (let ((value-17 (quote ert-form-evaluation-aborted-18))) (let (form-\n  (let* ((fn-15 (function leap-year-p)) (args-16 (condition-case err (\n  (lambda nil (let* ((fn-15 (function leap-year-p)) (args-16 (conditio\n  ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n  ert-run-test(#s(ert-test :name century :documentation nil :body (lam\n  ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n  ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n  ert-run-tests-batch(nil)\n  ert-run-tests-batch-and-exit()\n  command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-all-\n  command-line()\n  normal-top-level()\nTest century condition:\n    (invalid-function\n     (and\n      (= 0\n\t (mod year 4))\n      (or\n       (not\n\t(= 0 ...))\n       (= 0\n\t  (mod year 401)))))\n   \u001b[01;31m\u001b[KFAILED  2/5  century\u001b[m\u001b[K\nTest exceptional-century backtrace:\n  ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (mod ye\n  (not ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (m\n  leap-year-p(2000)\n  apply(leap-year-p 2000)\n  (setq value-22 (apply fn-20 args-21))\n  (unwind-protect (setq value-22 (apply fn-20 args-21)) (setq form-des\n  (if (unwind-protect (setq value-22 (apply fn-20 args-21)) (setq form\n  (let (form-description-24) (if (unwind-protect (setq value-22 (apply\n  (let ((value-22 (quote ert-form-evaluation-aborted-23))) (let (form-\n  (let* ((fn-20 (function leap-year-p)) (args-21 (condition-case err (\n  (lambda nil (let* ((fn-20 (function leap-year-p)) (args-21 (conditio\n  ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n  ert-run-test(#s(ert-test :name exceptional-century :documentation ni\n  ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n  ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n  ert-run-tests-batch(nil)\n  ert-run-tests-batch-and-exit()\n  command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-all-\n  command-line()\n  normal-top-level()\nTest exceptional-century condition:\n    (invalid-function\n     (and\n      (= 0\n\t (mod year 4))\n      (or\n       (not\n\t(= 0 ...))\n       (= 0\n\t  (mod year 401)))))\n   \u001b[01;31m\u001b[KFAILED  3/5  exceptional-century\u001b[m\u001b[K\nTest non-leap-even-year backtrace:\n  ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (mod ye\n  (not ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (m\n  leap-year-p(1997)\n  apply(leap-year-p 1997)\n  (setq value-12 (apply fn-10 args-11))\n  (unwind-protect (setq value-12 (apply fn-10 args-11)) (setq form-des\n  (not (unwind-protect (setq value-12 (apply fn-10 args-11)) (setq for\n  (if (not (unwind-protect (setq value-12 (apply fn-10 args-11)) (setq\n  (let (form-description-14) (if (not (unwind-protect (setq value-12 (\n  (let ((value-12 (quote ert-form-evaluation-aborted-13))) (let (form-\n  (let* ((fn-10 (function leap-year-p)) (args-11 (condition-case err (\n  (lambda nil (let* ((fn-10 (function leap-year-p)) (args-11 (conditio\n  ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n  ert-run-test(#s(ert-test :name non-leap-even-year :documentation nil\n  ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n  ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n  ert-run-tests-batch(nil)\n  ert-run-tests-batch-and-exit()\n  command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-all-\n  command-line()\n  normal-top-level()\nTest non-leap-even-year condition:\n    (invalid-function\n     (and\n      (= 0\n\t (mod year 4))\n      (or\n       (not\n\t(= 0 ...))\n       (= 0\n\t  (mod year 401)))))\n   \u001b[01;31m\u001b[KFAILED  4/5  non-leap-even-year\u001b[m\u001b[K\nTest vanilla-leap-year backtrace:\n  ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (mod ye\n  (not ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (m\n  leap-year-p(1996)\n  apply(leap-year-p 1996)\n  (setq value-2 (apply fn-0 args-1))\n  (unwind-protect (setq value-2 (apply fn-0 args-1)) (setq form-descri\n  (if (unwind-protect (setq value-2 (apply fn-0 args-1)) (setq form-de\n  (let (form-description-4) (if (unwind-protect (setq value-2 (apply f\n  (let ((value-2 (quote ert-form-evaluation-aborted-3))) (let (form-de\n  (let* ((fn-0 (function leap-year-p)) (args-1 (condition-case err (le\n  (lambda nil (let* ((fn-0 (function leap-year-p)) (args-1 (condition-\n  ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n  ert-run-test(#s(ert-test :name vanilla-leap-year :documentation nil \n  ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n  ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n  ert-run-tests-batch(nil)\n  ert-run-tests-batch-and-exit()\n  command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-all-\n  command-line()\n  normal-top-level()\nTest vanilla-leap-year condition:\n    (invalid-function\n     (and\n      (= 0\n\t (mod year 4))\n      (or\n       (not\n\t(= 0 ...))\n       (= 0\n\t  (mod year 401)))))\n   \u001b[01;31m\u001b[KFAILED  5/5  vanilla-leap-year\u001b[m\u001b[K\n\nRan 5 tests, 0 results as expected, 5 unexpected \n\n5 unexpected results:\n   \u001b[01;31m\u001b[KFAILED  any-old-year\u001b[m\u001b[K\n   \u001b[01;31m\u001b[KFAILED  century\u001b[m\u001b[K\n   \u001b[01;31m\u001b[KFAILED  exceptional-century\u001b[m\u001b[K\n   \u001b[01;31m\u001b[KFAILED  non-leap-even-year\u001b[m\u001b[K\n   \u001b[01;31m\u001b[KFAILED  vanilla-leap-year\u001b[m\u001b[K"
+  "message": null,
+  "tests": [
+    {
+      "name": "vanilla-leap-year",
+      "test_code": "(should (leap-year-p 1996))",
+      "status": "fail",
+      "message": "(ert-test-failed\n     ((should\n       (leap-year-p 1996))\n      :form\n      (leap-year-p 1996)\n      :value nil))"
+    },
+    {
+      "name": "any-old-year",
+      "test_code": "(should-not (leap-year-p 1997))",
+      "status": "fail",
+      "message": "(ert-test-failed\n     ((should-not\n       (leap-year-p 1997))\n      :form\n      (leap-year-p 1997)\n      :value t))"
+    },
+    {
+      "name": "non-leap-even-year",
+      "test_code": "(should-not (leap-year-p 1997))",
+      "status": "fail",
+      "message": "(ert-test-failed\n     ((should-not\n       (leap-year-p 1997))\n      :form\n      (leap-year-p 1997)\n      :value t))"
+    },
+    {
+      "name": "century",
+      "test_code": "(should-not (leap-year-p 1900))",
+      "status": "fail",
+      "message": "(ert-test-failed\n     ((should-not\n       (leap-year-p 1900))\n      :form\n      (leap-year-p 1900)\n      :value t))"
+    },
+    {
+      "name": "exceptional-century",
+      "test_code": "(should (leap-year-p 2000))",
+      "status": "fail",
+      "message": "(ert-test-failed\n     ((should\n       (leap-year-p 2000))\n      :form\n      (leap-year-p 2000)\n      :value nil))"
+    }
+  ]
 }