From d85620a5308d950f250a45e8ba2bcfde554dd172 Mon Sep 17 00:00:00 2001 From: Yevhen Shymotiuk Date: Tue, 4 Jan 2022 00:59:09 +0200 Subject: [PATCH] Upgrade to version 2 spec --- bin/parse_tests.py | 178 ++++++++++++++++++ bin/run.sh | 36 ++-- tests/example-all-fail/example-all-fail.el | 4 +- tests/example-all-fail/expected_results.json | 36 +++- .../example-empty-file/expected_results.json | 36 +++- tests/example-output/example-output-test.el | 24 +++ tests/example-output/example-output.el | 23 +++ tests/example-output/expected_results.json | 37 ++++ .../expected_results.json | 32 +++- tests/example-success/expected_results.json | 32 +++- .../expected_results.json | 33 +++- 11 files changed, 442 insertions(+), 29 deletions(-) create mode 100755 bin/parse_tests.py create mode 100644 tests/example-output/example-output-test.el create mode 100644 tests/example-output/example-output.el create mode 100644 tests/example-output/expected_results.json diff --git a/bin/parse_tests.py b/bin/parse_tests.py new file mode 100755 index 0000000..d96181f --- /dev/null +++ b/bin/parse_tests.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 + +import json +import re +import sys +from enum import Enum + + +class ExitCode(Enum): + PASS = 0 + FAIL = 1 + ERROR = 2 + + +class Status(Enum): + PASS = "pass" + FAIL = "fail" + ERROR = "error" + + +TEST_FUNCTION = "ert-deftest" +TEST_FAILED_FUNCTION = "ert-test-failed" + + +def parse_test_functions(s: str): + """ + Retrieve test function names and code from regions like + '(ert-deftest name-is-persistent + "Test that robot name is persistent." + (should (equal (robot-name *robbie*) + (robot-name *robbie*))))' + in the test file + """ + function_matches = re.finditer( + fr"\({TEST_FUNCTION}\s+(?P[\w-]+)\s+\(\)\s*(?P\".*\")?\s*(?P(?:\n.+)+)\)", + s, + ) + names = [] + code_pieces = [] + for m in function_matches: + names.append(m["name"]) + code_pieces.append(m["code"].strip()) + return names, code_pieces + + +def parse_test_statuses(s: str): + """ + Retrieve test statuses from lines like + 'passed 3/4 name-is-persistent (0.000049 sec)' + in the test output + """ + status_matches = re.finditer( + r"\s+(?Ppassed|FAILED)\s+(?P\d+)/\d+\s+(?P[\w-]+)\s*\(.*\)", + s, + ) + return { + m["name"]: ( + Status.PASS if m["status"].strip() == "passed" else Status.FAIL + ) + for m in status_matches + } + + +def parse_test_message(name: str, s: str): + """ + Retrieve test messages from regions like + 'Test name-can-be-reset condition: + (wrong-type-argument hash-table-p nil) + FAILED 2/4 name-can-be-reset' + in the test output + """ + condition_matches = re.finditer( + fr"Test\s{name}\scondition:\s+(?P\((?P.+)(?:\n.+)+)FAILED\s+(?P\d+)/\d+\s+{name}", + s, + ) + try: + cond_match = next(condition_matches) + except StopIteration: + return None, None + message = cond_match["condition"].strip() + # status is 'fail' if test condition starts with the test failed function + # otherwise there is an error + status = ( + Status.FAIL + if cond_match["function"] == TEST_FAILED_FUNCTION + else Status.ERROR + ) + return message, status + + +def parse_test_output(name: str, num: int, s: str): + """ + Retrieve test outputs from regions like + 'Running 4 tests (2022-01-04 17:06:51+0200, selector ‘t’) + "1DG190" + passed 1/4 different-robots-have-different-names (0.000075 sec)' + , + ' passed 1/4 different-robots-have-different-names (0.000075 sec) + "1XW454" + passed 2/4 name-can-be-reset (0.000047 sec)' + and + ' passed 3/4 name-is-persistent (0.000049 sec) + "1DG190" + Test name-matches-expected-pattern backtrace:' + in the test output + """ + status_line_regexp = ( + fr"(?:\s+(?:passed|FAILED)\s+{num - 1}/\d+\s+(?:[\w-]+)\s*\(.*)" + ) + output_regexp = fr"\)\n*(?P(?:\n.*)+)\s*(?:passed\s+{num}|Test\s{name}\sbacktrace)" + output_matches = re.finditer( + ("" if num == 1 else status_line_regexp) + output_regexp, s + ) + try: + output_match = next(output_matches) + except StopIteration: + return None, None + output = output_match["output"].strip() + message = None + # Output is limited to 500 chars + if len(output) > 500: + message = "Output was truncated. Please limit to 500 chars" + output = output[:500] + return output, message + + +def run(test_file_path: str, test_output_file_path: str): + exit_code = ExitCode.PASS + with open(test_file_path, encoding="utf-8") as f: + test_file_content = f.read() + with open(test_output_file_path, encoding="utf-8") as f: + test_output_file_content = f.read() + names, code_pieces = parse_test_functions(test_file_content) + name_to_number = {name: i + 1 for i, name in enumerate(sorted(names))} + name_to_status = parse_test_statuses(test_output_file_content) + status_to_exit_code = {Status(ec.name.lower()): ec for ec in ExitCode} + tests = [] + for name, code in zip(names, code_pieces): + test = {} + number = name_to_number[name] + test["name"] = name + test["test_code"] = code.strip() + # get status from status line or assume it is syntax error if there is no one + status = name_to_status.get(name, Status.ERROR) + exit_code = max( + exit_code, status_to_exit_code[status], key=lambda x: x.value + ) + message = None + condition_message, message_status = parse_test_message( + name, test_output_file_content + ) + if condition_message: + message, status = condition_message, message_status + output, output_message = parse_test_output( + name, int(number), test_output_file_content + ) + if output_message and status != Status.PASS: + if message: + message += "\n" + output_message + else: + message = output_message + test["status"] = status.value + if message: + test["message"] = message + if output: + test["output"] = output + tests.append(test) + print(json.dumps(tests)) + return exit_code + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("./parse-tests.py ", file=sys.stderr) + sys.exit(ExitCode.ERROR.value) + else: + exit_code = run(*sys.argv[1:]) + sys.exit(exit_code.value) diff --git a/bin/run.sh b/bin/run.sh index 4a201ef..c84546f 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -22,9 +22,11 @@ if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then fi slug="$1" +script_dir="$(dirname "$0")" input_dir="${2%/}" output_dir="${3%/}" test_file="${input_dir}/${slug}-test.el" +test_output_file="$(mktemp --suffix ".out")" results_file="${output_dir}/results.json" # Create the output directory if it doesn't exist @@ -34,24 +36,26 @@ echo "${slug}: testing..." pushd "${input_dir}" > /dev/null -# Run the tests for the provided implementation file and redirect stdout and -# stderr to capture it -test_output=$(emacs -batch -l ert -l "${test_file}" -f ert-run-tests-batch-and-exit 2>&1) -exit_code=$? +# Run the tests for the provided implementation file and record all terminal +# output to a temporary file to preserve output order +script -c "emacs -batch -l ert -l \"${test_file}\" -f ert-run-tests-batch-and-exit" \ + -O "$test_output_file" &> /dev/null popd > /dev/null -# Write the results.json file based on the exit code of the command that was -# just executed that tested the implementation file -if [ $exit_code -eq 0 ]; then - jq -n '{version: 1, status: "pass"}' > ${results_file} -else - # Manually add colors to the output to help scanning the output for errors - colorized_test_output=$(echo "${test_output}" \ - | GREP_COLOR='01;31' grep --color=always -E -e 'FAILED.*$|$' \ - | GREP_COLOR='01;32' grep --color=always -E -e 'passed.*$|$') - - jq -n --arg output "${colorized_test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file} -fi +# Write the results.json file based on both the exit code of the command that +# was just executed that tested the implementation file and per-test information +tests=$("$script_dir/parse_tests.py" "$test_file" "$test_output_file") +exit_code=$? +case $exit_code in + 0) status="pass" ;; + 1) status="fail" ;; + 2) status="error" ;; + *) echo "'parse_tests.py' script returned unknown exit code" 1>&2 && exit 1 ;; +esac + +jq -n --arg status "$status" \ + --argjson tests "$tests" \ + '{version: 2, status: $status, message: null, tests: $tests}' > ${results_file} echo "${slug}: done" diff --git a/tests/example-all-fail/example-all-fail.el b/tests/example-all-fail/example-all-fail.el index 5a4a78a..90ad2ea 100644 --- a/tests/example-all-fail/example-all-fail.el +++ b/tests/example-all-fail/example-all-fail.el @@ -6,9 +6,9 @@ (defun leap-year-p (year) "Determine if YEAR is a leap year." - (not ((and (= 0 (mod year 4)) + (not (and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) - (= 0 (mod year 401))))))) + (= 0 (mod year 400)))))) (provide 'leap) ;;; leap.el ends here diff --git a/tests/example-all-fail/expected_results.json b/tests/example-all-fail/expected_results.json index ed2dd85..d3e1381 100644 --- a/tests/example-all-fail/expected_results.json +++ b/tests/example-all-fail/expected_results.json @@ -1,5 +1,37 @@ { - "version": 1, + "version": 2, "status": "fail", - "message": "Loading /solution/example-all-fail.el (source)...\nRunning 5 tests \nTest any-old-year backtrace:\n ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (mod ye\n (not ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (m\n leap-year-p(1997)\n apply(leap-year-p 1997)\n (setq value-7 (apply fn-5 args-6))\n (unwind-protect (setq value-7 (apply fn-5 args-6)) (setq form-descri\n (not (unwind-protect (setq value-7 (apply fn-5 args-6)) (setq form-d\n (if (not (unwind-protect (setq value-7 (apply fn-5 args-6)) (setq fo\n (let (form-description-9) (if (not (unwind-protect (setq value-7 (ap\n (let ((value-7 (quote ert-form-evaluation-aborted-8))) (let (form-de\n (let* ((fn-5 (function leap-year-p)) (args-6 (condition-case err (le\n (lambda nil (let* ((fn-5 (function leap-year-p)) (args-6 (condition-\n ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n ert-run-test(#s(ert-test :name any-old-year :documentation nil :body\n ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n ert-run-tests-batch(nil)\n ert-run-tests-batch-and-exit()\n command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-all-\n command-line()\n normal-top-level()\nTest any-old-year condition:\n (invalid-function\n (and\n (= 0\n\t (mod year 4))\n (or\n (not\n\t(= 0 ...))\n (= 0\n\t (mod year 401)))))\n \u001b[01;31m\u001b[KFAILED 1/5 any-old-year\u001b[m\u001b[K\nTest century backtrace:\n ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (mod ye\n (not ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (m\n leap-year-p(1900)\n apply(leap-year-p 1900)\n (setq value-17 (apply fn-15 args-16))\n (unwind-protect (setq value-17 (apply fn-15 args-16)) (setq form-des\n (not (unwind-protect (setq value-17 (apply fn-15 args-16)) (setq for\n (if (not (unwind-protect (setq value-17 (apply fn-15 args-16)) (setq\n (let (form-description-19) (if (not (unwind-protect (setq value-17 (\n (let ((value-17 (quote ert-form-evaluation-aborted-18))) (let (form-\n (let* ((fn-15 (function leap-year-p)) (args-16 (condition-case err (\n (lambda nil (let* ((fn-15 (function leap-year-p)) (args-16 (conditio\n ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n ert-run-test(#s(ert-test :name century :documentation nil :body (lam\n ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n ert-run-tests-batch(nil)\n ert-run-tests-batch-and-exit()\n command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-all-\n command-line()\n normal-top-level()\nTest century condition:\n (invalid-function\n (and\n (= 0\n\t (mod year 4))\n (or\n (not\n\t(= 0 ...))\n (= 0\n\t (mod year 401)))))\n \u001b[01;31m\u001b[KFAILED 2/5 century\u001b[m\u001b[K\nTest exceptional-century backtrace:\n ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (mod ye\n (not ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (m\n leap-year-p(2000)\n apply(leap-year-p 2000)\n (setq value-22 (apply fn-20 args-21))\n (unwind-protect (setq value-22 (apply fn-20 args-21)) (setq form-des\n (if (unwind-protect (setq value-22 (apply fn-20 args-21)) (setq form\n (let (form-description-24) (if (unwind-protect (setq value-22 (apply\n (let ((value-22 (quote ert-form-evaluation-aborted-23))) (let (form-\n (let* ((fn-20 (function leap-year-p)) (args-21 (condition-case err (\n (lambda nil (let* ((fn-20 (function leap-year-p)) (args-21 (conditio\n ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n ert-run-test(#s(ert-test :name exceptional-century :documentation ni\n ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n ert-run-tests-batch(nil)\n ert-run-tests-batch-and-exit()\n command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-all-\n command-line()\n normal-top-level()\nTest exceptional-century condition:\n (invalid-function\n (and\n (= 0\n\t (mod year 4))\n (or\n (not\n\t(= 0 ...))\n (= 0\n\t (mod year 401)))))\n \u001b[01;31m\u001b[KFAILED 3/5 exceptional-century\u001b[m\u001b[K\nTest non-leap-even-year backtrace:\n ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (mod ye\n (not ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (m\n leap-year-p(1997)\n apply(leap-year-p 1997)\n (setq value-12 (apply fn-10 args-11))\n (unwind-protect (setq value-12 (apply fn-10 args-11)) (setq form-des\n (not (unwind-protect (setq value-12 (apply fn-10 args-11)) (setq for\n (if (not (unwind-protect (setq value-12 (apply fn-10 args-11)) (setq\n (let (form-description-14) (if (not (unwind-protect (setq value-12 (\n (let ((value-12 (quote ert-form-evaluation-aborted-13))) (let (form-\n (let* ((fn-10 (function leap-year-p)) (args-11 (condition-case err (\n (lambda nil (let* ((fn-10 (function leap-year-p)) (args-11 (conditio\n ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n ert-run-test(#s(ert-test :name non-leap-even-year :documentation nil\n ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n ert-run-tests-batch(nil)\n ert-run-tests-batch-and-exit()\n command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-all-\n command-line()\n normal-top-level()\nTest non-leap-even-year condition:\n (invalid-function\n (and\n (= 0\n\t (mod year 4))\n (or\n (not\n\t(= 0 ...))\n (= 0\n\t (mod year 401)))))\n \u001b[01;31m\u001b[KFAILED 4/5 non-leap-even-year\u001b[m\u001b[K\nTest vanilla-leap-year backtrace:\n ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (mod ye\n (not ((and (= 0 (mod year 4)) (or (not (= 0 (mod year 100))) (= 0 (m\n leap-year-p(1996)\n apply(leap-year-p 1996)\n (setq value-2 (apply fn-0 args-1))\n (unwind-protect (setq value-2 (apply fn-0 args-1)) (setq form-descri\n (if (unwind-protect (setq value-2 (apply fn-0 args-1)) (setq form-de\n (let (form-description-4) (if (unwind-protect (setq value-2 (apply f\n (let ((value-2 (quote ert-form-evaluation-aborted-3))) (let (form-de\n (let* ((fn-0 (function leap-year-p)) (args-1 (condition-case err (le\n (lambda nil (let* ((fn-0 (function leap-year-p)) (args-1 (condition-\n ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n ert-run-test(#s(ert-test :name vanilla-leap-year :documentation nil \n ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n ert-run-tests-batch(nil)\n ert-run-tests-batch-and-exit()\n command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-all-\n command-line()\n normal-top-level()\nTest vanilla-leap-year condition:\n (invalid-function\n (and\n (= 0\n\t (mod year 4))\n (or\n (not\n\t(= 0 ...))\n (= 0\n\t (mod year 401)))))\n \u001b[01;31m\u001b[KFAILED 5/5 vanilla-leap-year\u001b[m\u001b[K\n\nRan 5 tests, 0 results as expected, 5 unexpected \n\n5 unexpected results:\n \u001b[01;31m\u001b[KFAILED any-old-year\u001b[m\u001b[K\n \u001b[01;31m\u001b[KFAILED century\u001b[m\u001b[K\n \u001b[01;31m\u001b[KFAILED exceptional-century\u001b[m\u001b[K\n \u001b[01;31m\u001b[KFAILED non-leap-even-year\u001b[m\u001b[K\n \u001b[01;31m\u001b[KFAILED vanilla-leap-year\u001b[m\u001b[K" + "message": null, + "tests": [ + { + "name": "vanilla-leap-year", + "test_code": "(should (leap-year-p 1996))", + "status": "fail", + "message": "(ert-test-failed\n ((should\n (leap-year-p 1996))\n :form\n (leap-year-p 1996)\n :value nil))" + }, + { + "name": "any-old-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "fail", + "message": "(ert-test-failed\n ((should-not\n (leap-year-p 1997))\n :form\n (leap-year-p 1997)\n :value t))" + }, + { + "name": "non-leap-even-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "fail", + "message": "(ert-test-failed\n ((should-not\n (leap-year-p 1997))\n :form\n (leap-year-p 1997)\n :value t))" + }, + { + "name": "century", + "test_code": "(should-not (leap-year-p 1900))", + "status": "fail", + "message": "(ert-test-failed\n ((should-not\n (leap-year-p 1900))\n :form\n (leap-year-p 1900)\n :value t))" + }, + { + "name": "exceptional-century", + "test_code": "(should (leap-year-p 2000))", + "status": "fail", + "message": "(ert-test-failed\n ((should\n (leap-year-p 2000))\n :form\n (leap-year-p 2000)\n :value nil))" + } + ] } diff --git a/tests/example-empty-file/expected_results.json b/tests/example-empty-file/expected_results.json index 94964f8..86bdd49 100644 --- a/tests/example-empty-file/expected_results.json +++ b/tests/example-empty-file/expected_results.json @@ -1,5 +1,37 @@ { - "version": 1, + "version": 2, "status": "fail", - "message": "Loading /solution/example-empty-file.el (source)...\nRunning 5 tests \nTest any-old-year backtrace:\n leap-year-p(1997)\n apply(leap-year-p 1997)\n (setq value-7 (apply fn-5 args-6))\n (unwind-protect (setq value-7 (apply fn-5 args-6)) (setq form-descri\n (not (unwind-protect (setq value-7 (apply fn-5 args-6)) (setq form-d\n (if (not (unwind-protect (setq value-7 (apply fn-5 args-6)) (setq fo\n (let (form-description-9) (if (not (unwind-protect (setq value-7 (ap\n (let ((value-7 (quote ert-form-evaluation-aborted-8))) (let (form-de\n (let* ((fn-5 (function leap-year-p)) (args-6 (condition-case err (le\n (lambda nil (let* ((fn-5 (function leap-year-p)) (args-6 (condition-\n ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n ert-run-test(#s(ert-test :name any-old-year :documentation nil :body\n ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n ert-run-tests-batch(nil)\n ert-run-tests-batch-and-exit()\n command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-empt\n command-line()\n normal-top-level()\nTest any-old-year condition:\n (void-function leap-year-p)\n \u001b[01;31m\u001b[KFAILED 1/5 any-old-year\u001b[m\u001b[K\nTest century backtrace:\n leap-year-p(1900)\n apply(leap-year-p 1900)\n (setq value-17 (apply fn-15 args-16))\n (unwind-protect (setq value-17 (apply fn-15 args-16)) (setq form-des\n (not (unwind-protect (setq value-17 (apply fn-15 args-16)) (setq for\n (if (not (unwind-protect (setq value-17 (apply fn-15 args-16)) (setq\n (let (form-description-19) (if (not (unwind-protect (setq value-17 (\n (let ((value-17 (quote ert-form-evaluation-aborted-18))) (let (form-\n (let* ((fn-15 (function leap-year-p)) (args-16 (condition-case err (\n (lambda nil (let* ((fn-15 (function leap-year-p)) (args-16 (conditio\n ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n ert-run-test(#s(ert-test :name century :documentation nil :body (lam\n ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n ert-run-tests-batch(nil)\n ert-run-tests-batch-and-exit()\n command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-empt\n command-line()\n normal-top-level()\nTest century condition:\n (void-function leap-year-p)\n \u001b[01;31m\u001b[KFAILED 2/5 century\u001b[m\u001b[K\nTest exceptional-century backtrace:\n leap-year-p(2000)\n apply(leap-year-p 2000)\n (setq value-22 (apply fn-20 args-21))\n (unwind-protect (setq value-22 (apply fn-20 args-21)) (setq form-des\n (if (unwind-protect (setq value-22 (apply fn-20 args-21)) (setq form\n (let (form-description-24) (if (unwind-protect (setq value-22 (apply\n (let ((value-22 (quote ert-form-evaluation-aborted-23))) (let (form-\n (let* ((fn-20 (function leap-year-p)) (args-21 (condition-case err (\n (lambda nil (let* ((fn-20 (function leap-year-p)) (args-21 (conditio\n ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n ert-run-test(#s(ert-test :name exceptional-century :documentation ni\n ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n ert-run-tests-batch(nil)\n ert-run-tests-batch-and-exit()\n command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-empt\n command-line()\n normal-top-level()\nTest exceptional-century condition:\n (void-function leap-year-p)\n \u001b[01;31m\u001b[KFAILED 3/5 exceptional-century\u001b[m\u001b[K\nTest non-leap-even-year backtrace:\n leap-year-p(1997)\n apply(leap-year-p 1997)\n (setq value-12 (apply fn-10 args-11))\n (unwind-protect (setq value-12 (apply fn-10 args-11)) (setq form-des\n (not (unwind-protect (setq value-12 (apply fn-10 args-11)) (setq for\n (if (not (unwind-protect (setq value-12 (apply fn-10 args-11)) (setq\n (let (form-description-14) (if (not (unwind-protect (setq value-12 (\n (let ((value-12 (quote ert-form-evaluation-aborted-13))) (let (form-\n (let* ((fn-10 (function leap-year-p)) (args-11 (condition-case err (\n (lambda nil (let* ((fn-10 (function leap-year-p)) (args-11 (conditio\n ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n ert-run-test(#s(ert-test :name non-leap-even-year :documentation nil\n ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n ert-run-tests-batch(nil)\n ert-run-tests-batch-and-exit()\n command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-empt\n command-line()\n normal-top-level()\nTest non-leap-even-year condition:\n (void-function leap-year-p)\n \u001b[01;31m\u001b[KFAILED 4/5 non-leap-even-year\u001b[m\u001b[K\nTest vanilla-leap-year backtrace:\n leap-year-p(1996)\n apply(leap-year-p 1996)\n (setq value-2 (apply fn-0 args-1))\n (unwind-protect (setq value-2 (apply fn-0 args-1)) (setq form-descri\n (if (unwind-protect (setq value-2 (apply fn-0 args-1)) (setq form-de\n (let (form-description-4) (if (unwind-protect (setq value-2 (apply f\n (let ((value-2 (quote ert-form-evaluation-aborted-3))) (let (form-de\n (let* ((fn-0 (function leap-year-p)) (args-1 (condition-case err (le\n (lambda nil (let* ((fn-0 (function leap-year-p)) (args-1 (condition-\n ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n ert-run-test(#s(ert-test :name vanilla-leap-year :documentation nil \n ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n ert-run-tests-batch(nil)\n ert-run-tests-batch-and-exit()\n command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-empt\n command-line()\n normal-top-level()\nTest vanilla-leap-year condition:\n (void-function leap-year-p)\n \u001b[01;31m\u001b[KFAILED 5/5 vanilla-leap-year\u001b[m\u001b[K\n\nRan 5 tests, 0 results as expected, 5 unexpected \n\n5 unexpected results:\n \u001b[01;31m\u001b[KFAILED any-old-year\u001b[m\u001b[K\n \u001b[01;31m\u001b[KFAILED century\u001b[m\u001b[K\n \u001b[01;31m\u001b[KFAILED exceptional-century\u001b[m\u001b[K\n \u001b[01;31m\u001b[KFAILED non-leap-even-year\u001b[m\u001b[K\n \u001b[01;31m\u001b[KFAILED vanilla-leap-year\u001b[m\u001b[K" + "message": null, + "tests": [ + { + "name": "vanilla-leap-year", + "test_code": "(should (leap-year-p 1996))", + "status": "error", + "message": "(void-function leap-year-p)" + }, + { + "name": "any-old-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "error", + "message": "(void-function leap-year-p)" + }, + { + "name": "non-leap-even-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "error", + "message": "(void-function leap-year-p)" + }, + { + "name": "century", + "test_code": "(should-not (leap-year-p 1900))", + "status": "error", + "message": "(void-function leap-year-p)" + }, + { + "name": "exceptional-century", + "test_code": "(should (leap-year-p 2000))", + "status": "error", + "message": "(void-function leap-year-p)" + } + ] } diff --git a/tests/example-output/example-output-test.el b/tests/example-output/example-output-test.el new file mode 100644 index 0000000..2f76cd1 --- /dev/null +++ b/tests/example-output/example-output-test.el @@ -0,0 +1,24 @@ +;;; leap-test.el --- Tests for Leap exercise (exercism) + +;;; Commentary: + +;;; Code: +(load-file "example-output.el") + +(ert-deftest vanilla-leap-year () + (should (leap-year-p 1996))) + +(ert-deftest any-old-year () + (should-not (leap-year-p 1997))) + +(ert-deftest non-leap-even-year () + (should-not (leap-year-p 1997))) + +(ert-deftest century () + (should-not (leap-year-p 1900))) + +(ert-deftest exceptional-century () + (should (leap-year-p 2000))) + +(provide 'leap-test) +;;; leap-test.el ends here diff --git a/tests/example-output/example-output.el b/tests/example-output/example-output.el new file mode 100644 index 0000000..8e90be0 --- /dev/null +++ b/tests/example-output/example-output.el @@ -0,0 +1,23 @@ +;;; leap.el --- Leap exercise (exercism) + +;;; Commentary: + +;;; Code: + +(defun leap-year-p (year) + "Determine if YEAR is a leap year." + (print (format "Hello from stdout. It is %s now" year)) + ; 'message' writes to the stderr in batch mode + (message "Hello from stderr") + (print " (__) + `------(oo) + || (__) + ||w--||") + (print + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris sit amet tellus lectus. Suspendisse dignissim molestie sagittis. Praesent egestas massa a est maximus iaculis. Morbi varius nulla in purus hendrerit, eget tincidunt diam malesuada. Morbi convallis non ex eget auctor. Aenean massa tellus, maximus a fermentum non, tempor quis sem. Integer finibus tincidunt convallis. Curabitur dapibus lorem vitae nunc luctus pretium. Proin at efficitur elit, in eleifend ligula. Aenean dapibus mattis augue, vel consectetur lorem.") + (and (= 0 (mod year 4)) + (or (not (= 0 (mod year 100))) + (= 0 (mod year 400))))) + +(provide 'leap) +;;; leap.el ends here diff --git a/tests/example-output/expected_results.json b/tests/example-output/expected_results.json new file mode 100644 index 0000000..7673cba --- /dev/null +++ b/tests/example-output/expected_results.json @@ -0,0 +1,37 @@ +{ + "version": 2, + "status": "pass", + "message": null, + "tests": [ + { + "name": "vanilla-leap-year", + "test_code": "(should (leap-year-p 1996))", + "status": "pass", + "output": "\"Hello from stdout. It is 1996 now\"\n\nHello from stderr\n\n\" (__)\n `------(oo)\n || (__)\n ||w--||\"\n\n\"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris sit amet tellus lectus. Suspendisse dignissim molestie sagittis. Praesent egestas massa a est maximus iaculis. Morbi varius nulla in purus hendrerit, eget tincidunt diam malesuada. Morbi convallis non ex eget auctor. Aenean massa tellus, maximus a fermentum non, tempor quis sem. Integer finibus tincidunt convallis. Curabitur d" + }, + { + "name": "any-old-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "pass", + "output": "\"Hello from stdout. It is 1997 now\"\n\nHello from stderr\n\n\" (__)\n `------(oo)\n || (__)\n ||w--||\"\n\n\"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris sit amet tellus lectus. Suspendisse dignissim molestie sagittis. Praesent egestas massa a est maximus iaculis. Morbi varius nulla in purus hendrerit, eget tincidunt diam malesuada. Morbi convallis non ex eget auctor. Aenean massa tellus, maximus a fermentum non, tempor quis sem. Integer finibus tincidunt convallis. Curabitur d" + }, + { + "name": "non-leap-even-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "pass", + "output": "\"Hello from stdout. It is 1997 now\"\n\nHello from stderr\n\n\" (__)\n `------(oo)\n || (__)\n ||w--||\"\n\n\"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris sit amet tellus lectus. Suspendisse dignissim molestie sagittis. Praesent egestas massa a est maximus iaculis. Morbi varius nulla in purus hendrerit, eget tincidunt diam malesuada. Morbi convallis non ex eget auctor. Aenean massa tellus, maximus a fermentum non, tempor quis sem. Integer finibus tincidunt convallis. Curabitur d" + }, + { + "name": "century", + "test_code": "(should-not (leap-year-p 1900))", + "status": "pass", + "output": "\"Hello from stdout. It is 1900 now\"\n\nHello from stderr\n\n\" (__)\n `------(oo)\n || (__)\n ||w--||\"\n\n\"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris sit amet tellus lectus. Suspendisse dignissim molestie sagittis. Praesent egestas massa a est maximus iaculis. Morbi varius nulla in purus hendrerit, eget tincidunt diam malesuada. Morbi convallis non ex eget auctor. Aenean massa tellus, maximus a fermentum non, tempor quis sem. Integer finibus tincidunt convallis. Curabitur d" + }, + { + "name": "exceptional-century", + "test_code": "(should (leap-year-p 2000))", + "status": "pass", + "output": "\"Hello from stdout. It is 2000 now\"\n\nHello from stderr\n\n\" (__)\n `------(oo)\n || (__)\n ||w--||\"\n\n\"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris sit amet tellus lectus. Suspendisse dignissim molestie sagittis. Praesent egestas massa a est maximus iaculis. Morbi varius nulla in purus hendrerit, eget tincidunt diam malesuada. Morbi convallis non ex eget auctor. Aenean massa tellus, maximus a fermentum non, tempor quis sem. Integer finibus tincidunt convallis. Curabitur d" + } + ] +} diff --git a/tests/example-partial-fail/expected_results.json b/tests/example-partial-fail/expected_results.json index 56ce685..4ac20e1 100644 --- a/tests/example-partial-fail/expected_results.json +++ b/tests/example-partial-fail/expected_results.json @@ -1,5 +1,33 @@ { - "version": 1, + "version": 2, "status": "fail", - "message": "Loading /solution/example-partial-fail.el (source)...\nRunning 5 tests \n \u001b[01;32m\u001b[Kpassed 1/5 any-old-year\u001b[m\u001b[K\n \u001b[01;32m\u001b[Kpassed 2/5 century\u001b[m\u001b[K\nTest exceptional-century backtrace:\n signal(ert-test-failed (((should (leap-year-p 2000)) :form (leap-yea\n ert-fail(((should (leap-year-p 2000)) :form (leap-year-p 2000) :valu\n (if (unwind-protect (setq value-22 (apply fn-20 args-21)) (setq form\n (let (form-description-24) (if (unwind-protect (setq value-22 (apply\n (let ((value-22 (quote ert-form-evaluation-aborted-23))) (let (form-\n (let* ((fn-20 (function leap-year-p)) (args-21 (condition-case err (\n (lambda nil (let* ((fn-20 (function leap-year-p)) (args-21 (conditio\n ert--run-test-internal(#s(ert--test-execution-info :test #s(ert-test\n ert-run-test(#s(ert-test :name exceptional-century :documentation ni\n ert-run-or-rerun-test(#s(ert--stats :selector t :tests [#s(ert-test \n ert-run-tests(t #f(compiled-function (event-type &rest event-args) #\n ert-run-tests-batch(nil)\n ert-run-tests-batch-and-exit()\n command-line-1((\"-l\" \"ert\" \"-l\" \"/opt/test-runner/tests/example-part\n command-line()\n normal-top-level()\nTest exceptional-century condition:\n (ert-test-failed\n ((should\n (leap-year-p 2000))\n :form\n (leap-year-p 2000)\n :value nil))\n \u001b[01;31m\u001b[KFAILED 3/5 exceptional-century\u001b[m\u001b[K\n \u001b[01;32m\u001b[Kpassed 4/5 non-leap-even-year\u001b[m\u001b[K\n \u001b[01;32m\u001b[Kpassed 5/5 vanilla-leap-year\u001b[m\u001b[K\n\nRan 5 tests, 4 results as expected, 1 unexpected \n\n1 unexpected results:\n \u001b[01;31m\u001b[KFAILED exceptional-century\u001b[m\u001b[K" + "message": null, + "tests": [ + { + "name": "vanilla-leap-year", + "test_code": "(should (leap-year-p 1996))", + "status": "pass" + }, + { + "name": "any-old-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "pass" + }, + { + "name": "non-leap-even-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "pass" + }, + { + "name": "century", + "test_code": "(should-not (leap-year-p 1900))", + "status": "pass" + }, + { + "name": "exceptional-century", + "test_code": "(should (leap-year-p 2000))", + "status": "fail", + "message": "(ert-test-failed\n ((should\n (leap-year-p 2000))\n :form\n (leap-year-p 2000)\n :value nil))" + } + ] } diff --git a/tests/example-success/expected_results.json b/tests/example-success/expected_results.json index 6c2223e..5f69941 100644 --- a/tests/example-success/expected_results.json +++ b/tests/example-success/expected_results.json @@ -1,4 +1,32 @@ { - "version": 1, - "status": "pass" + "version": 2, + "status": "pass", + "message": null, + "tests": [ + { + "name": "vanilla-leap-year", + "test_code": "(should (leap-year-p 1996))", + "status": "pass" + }, + { + "name": "any-old-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "pass" + }, + { + "name": "non-leap-even-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "pass" + }, + { + "name": "century", + "test_code": "(should-not (leap-year-p 1900))", + "status": "pass" + }, + { + "name": "exceptional-century", + "test_code": "(should (leap-year-p 2000))", + "status": "pass" + } + ] } diff --git a/tests/example-syntax-error/expected_results.json b/tests/example-syntax-error/expected_results.json index e464eec..e9ac341 100644 --- a/tests/example-syntax-error/expected_results.json +++ b/tests/example-syntax-error/expected_results.json @@ -1,5 +1,32 @@ { - "version": 1, - "status": "fail", - "message": "Loading /solution/example-syntax-error.el (source)...\nSymbol's value as variable is void: DEF&*@" + "version": 2, + "status": "error", + "message": null, + "tests": [ + { + "name": "vanilla-leap-year", + "test_code": "(should (leap-year-p 1996))", + "status": "error" + }, + { + "name": "any-old-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "error" + }, + { + "name": "non-leap-even-year", + "test_code": "(should-not (leap-year-p 1997))", + "status": "error" + }, + { + "name": "century", + "test_code": "(should-not (leap-year-p 1900))", + "status": "error" + }, + { + "name": "exceptional-century", + "test_code": "(should (leap-year-p 2000))", + "status": "error" + } + ] }