From 1ebe19730454291100603973cdf0b848d3a41ee0 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Wed, 20 Sep 2023 12:02:40 +0200 Subject: [PATCH 1/8] Support DSL values in oracles --- tested/internationalization/nl.yaml | 4 +- tested/judge/programmed.py | 12 +- tested/languages/bash/generators.py | 2 +- tested/languages/c/generators.py | 2 +- .../languages/c/templates/evaluation_result.h | 2 + tested/languages/csharp/generators.py | 2 +- .../csharp/templates/EvaluationResult.cs | 2 +- tested/languages/haskell/generators.py | 2 +- .../haskell/templates/EvaluationUtils.hs | 6 +- tested/languages/java/generators.py | 2 +- .../java/templates/EvaluationResult.java | 20 ++- tested/languages/java/templates/Values.java | 78 ++++++--- tested/languages/javascript/generators.py | 2 +- tested/languages/kotlin/generators.py | 2 +- .../kotlin/templates/EvaluationResult.kt | 18 +- tested/languages/kotlin/templates/Values.kt | 57 ++++--- tested/languages/python/generators.py | 2 +- .../python/templates/evaluation_utils.py | 2 + tested/oracles/common.py | 93 ++++++++--- tested/oracles/programmed.py | 46 +---- tested/oracles/specific.py | 18 +- tested/serialisation.py | 43 +---- .../echo-function/evaluation/Evaluator.cs | 5 + .../echo-function/evaluation/Evaluator.java | 8 + .../echo-function/evaluation/Evaluator.kt | 11 +- .../echo-function/evaluation/evaluator.js | 16 +- .../echo-function/evaluation/evaluator.py | 9 + .../evaluation/programmed-dsl-no-haskell.tson | 157 ++++++++++++++++++ tests/test_functionality.py | 19 +++ tests/test_serialisation.py | 2 +- 30 files changed, 446 insertions(+), 198 deletions(-) create mode 100644 tests/exercises/echo-function/evaluation/programmed-dsl-no-haskell.tson diff --git a/tested/internationalization/nl.yaml b/tested/internationalization/nl.yaml index f245a2dc..7caa5042 100644 --- a/tested/internationalization/nl.yaml +++ b/tested/internationalization/nl.yaml @@ -77,7 +77,7 @@ nl: Contacteer de lesgever. unsupported: "Evaluation in %{lang} wordt niet ondersteund." unknown: - compilation: "Ongekende compilatie fout" + compilation: "Ongekende compilatiefout" produced: stderr: "De evaluatiecode produceerde volgende uitvoer op stderr:" stdout: "De evaluatiecode produceerde volgende uitvoer op stdout:" @@ -86,7 +86,7 @@ nl: languages: config: unknown: - compilation: "Ongekende compilatie fout" + compilation: "Ongekende compilatiefout" generator: missing: input: "Geen invoer gevonden." diff --git a/tested/judge/programmed.py b/tested/judge/programmed.py index 415267fd..68d7acf4 100644 --- a/tested/judge/programmed.py +++ b/tested/judge/programmed.py @@ -22,7 +22,8 @@ generate_custom_evaluator, generate_statement, ) -from tested.serialisation import BooleanEvalResult, EvalResult, Value +from tested.oracles.common import BooleanEvalResult +from tested.serialisation import Value from tested.testsuite import CustomCheckOracle from tested.utils import get_identifier @@ -34,7 +35,7 @@ def evaluate_programmed( evaluator: CustomCheckOracle, expected: Value, actual: Value, -) -> BaseExecutionResult | EvalResult: +) -> BaseExecutionResult | BooleanEvalResult: """ Run the custom evaluation. Concerning structure and execution, the custom oracle is very similar to the execution of the whole evaluation. It a @@ -181,7 +182,7 @@ def _evaluate_python( oracle: CustomCheckOracle, expected: Value, actual: Value, -) -> EvalResult: +) -> BooleanEvalResult: """ Run an evaluation in Python. While the templates are still used to generate code, they are not executed in a separate process, but inside python itself. @@ -267,13 +268,12 @@ def _evaluate_python( permission=Permission.STAFF, ) ) - return EvalResult( + return BooleanEvalResult( result=Status.INTERNAL_ERROR, readable_expected=None, readable_actual=None, messages=messages, ) - assert isinstance(result_, BooleanEvalResult) result_.messages.extend(messages) - return result_.as_eval_result() + return result_ diff --git a/tested/languages/bash/generators.py b/tested/languages/bash/generators.py index 14d85318..f5a2c8f0 100644 --- a/tested/languages/bash/generators.py +++ b/tested/languages/bash/generators.py @@ -238,7 +238,7 @@ def convert_encoder(values: list[Value]) -> str: }} function send_value {{ - echo "{{\\"type\\": \\"text\\", \\"data\\": $(json_escape "$1")}}" + echo "{{\\"type\\": \\"text\\", \\"data\\": $(json_escape "$1")}}␞" }} """ for value in values: diff --git a/tested/languages/c/generators.py b/tested/languages/c/generators.py index 5b22f49d..0aae333d 100644 --- a/tested/languages/c/generators.py +++ b/tested/languages/c/generators.py @@ -322,6 +322,6 @@ def convert_encoder(values: list[Value]) -> str: """ for value in values: result += " " * 4 + f"write_value(stdout, {convert_value(value)});\n" - result += " " * 4 + 'printf("\\n");\n' + result += " " * 4 + 'printf("␞");\n' result += "}\n" return result diff --git a/tested/languages/c/templates/evaluation_result.h b/tested/languages/c/templates/evaluation_result.h index 0e76a2f0..56e39fcb 100644 --- a/tested/languages/c/templates/evaluation_result.h +++ b/tested/languages/c/templates/evaluation_result.h @@ -15,6 +15,8 @@ typedef struct EvaluationResult { char* readableActual; size_t nrOfMessages; Message** messages; + char* dslExpected; + char* dslActual; } EvaluationResult; diff --git a/tested/languages/csharp/generators.py b/tested/languages/csharp/generators.py index 1fc6f8f6..f31d0b7a 100644 --- a/tested/languages/csharp/generators.py +++ b/tested/languages/csharp/generators.py @@ -478,7 +478,7 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += " " * 6 + f"Values.WriteValue(stdout, {convert_value(value)});" - result += " " * 6 + 'stdout.Write("\\n");\n' + result += " " * 6 + 'stdout.Write("␞");\n' result += "}\n}\n}\n" return result diff --git a/tested/languages/csharp/templates/EvaluationResult.cs b/tested/languages/csharp/templates/EvaluationResult.cs index 35d4e353..b202d3a3 100644 --- a/tested/languages/csharp/templates/EvaluationResult.cs +++ b/tested/languages/csharp/templates/EvaluationResult.cs @@ -3,7 +3,7 @@ namespace Tested { public record Message(string Description, string Format = "text", string? Permission = null); - public record EvaluationResult(bool Result, string? ReadableExpected = null, string? ReadableActual = null, List? Messages = null) + public record EvaluationResult(bool Result, string? ReadableExpected = null, string? ReadableActual = null, List? Messages = null, string? DslExpected = null, string? DslActual = null) { public List Messages { get; init; } = Messages ?? new List(); } diff --git a/tested/languages/haskell/generators.py b/tested/languages/haskell/generators.py index 11e1956a..56349323 100644 --- a/tested/languages/haskell/generators.py +++ b/tested/languages/haskell/generators.py @@ -350,5 +350,5 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += indent + f"sendValueH stdout ({convert_value(value)})\n" - result += indent + 'putStr "\\n"\n' + result += indent + 'putStr "␞"\n' return result diff --git a/tested/languages/haskell/templates/EvaluationUtils.hs b/tested/languages/haskell/templates/EvaluationUtils.hs index c6eb54b6..6478c7a4 100644 --- a/tested/languages/haskell/templates/EvaluationUtils.hs +++ b/tested/languages/haskell/templates/EvaluationUtils.hs @@ -11,7 +11,9 @@ data EvaluationResult = EvaluationResult { result :: Bool, readableExpected :: Maybe (String), readableActual :: Maybe (String), - messages :: [Message] + messages :: [Message], + dslExpected :: Maybe (String), + dslActual :: Maybe (String) } deriving Show message description = Message { @@ -24,5 +26,7 @@ evaluationResult = EvaluationResult { result = False, readableExpected = Nothing, readableActual = Nothing, + dslExpected = Nothing, + dslActual = Nothing, messages = [] } diff --git a/tested/languages/java/generators.py b/tested/languages/java/generators.py index b5ef0d64..9cce196b 100644 --- a/tested/languages/java/generators.py +++ b/tested/languages/java/generators.py @@ -450,7 +450,7 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += " " * 8 + f"Values.send(writer, {convert_value(value)});\n" - result += " " * 8 + 'writer.write("\\n");\n' + result += " " * 8 + 'writer.write("␞");\n' result += " " * 8 + "writer.close();\n" result += " " * 4 + "}\n" diff --git a/tested/languages/java/templates/EvaluationResult.java b/tested/languages/java/templates/EvaluationResult.java index 87bef95d..9702110e 100644 --- a/tested/languages/java/templates/EvaluationResult.java +++ b/tested/languages/java/templates/EvaluationResult.java @@ -6,12 +6,16 @@ public class EvaluationResult { public final boolean result; public final String readableExpected; public final String readableActual; + public final String dslExpected; + public final String dslActual; public final List messages; - private EvaluationResult(boolean result, String readableExpected, String readableActual, List messages) { + private EvaluationResult(boolean result, String readableExpected, String readableActual, String dslExpected, String dslActual, List messages) { this.result = result; this.readableExpected = readableExpected; this.readableActual = readableActual; + this.dslExpected = dslExpected; + this.dslActual = dslActual; this.messages = messages; } @@ -44,6 +48,8 @@ public static class Builder { private final boolean result; private String readableExpected = null; private String readableActual = null; + private String dslExpected = null; + private String dslActual = null; private final List messages = new ArrayList<>(); private Builder(boolean result) { @@ -59,6 +65,16 @@ public Builder withReadableActual(String readableActual) { this.readableActual = readableActual; return this; } + + public Builder withDslExpected(String dslExpected) { + this.dslExpected = dslExpected; + return this; + } + + public Builder withDslActual(String dslActual) { + this.dslActual = dslActual; + return this; + } public Builder withMessages(List messages) { this.messages.addAll(messages); @@ -71,7 +87,7 @@ public Builder withMessage(Message message) { } public EvaluationResult build() { - return new EvaluationResult(result, readableExpected, readableActual, messages); + return new EvaluationResult(result, readableExpected, readableActual, dslExpected, dslActual, messages); } } } diff --git a/tested/languages/java/templates/Values.java b/tested/languages/java/templates/Values.java index 330b1337..bfcb54b0 100644 --- a/tested/languages/java/templates/Values.java +++ b/tested/languages/java/templates/Values.java @@ -128,9 +128,13 @@ private static List internalEncode(Object value) { private static String encode(Object value) { var typeAndData = internalEncode(value); - return "{ \"data\": " + typeAndData.get(1) + "," + - " \"type\": \"" + typeAndData.get(0) + "\", " + - " \"diagnostic\": " + typeAndData.get(2) + "}"; + return """ + { + "data": %s, + "type": "%s", + "diagnostic": %s + } + """.formatted(typeAndData.get(1), typeAndData.get(0), typeAndData.get(2)); } public static void send(PrintWriter writer, Object value) { @@ -145,37 +149,57 @@ public static void sendException(PrintWriter writer, Throwable exception) { exception.printStackTrace(new PrintWriter(sw)); var trace = sw.toString(); var msg = exception.getMessage(); - var result = "{ \"message\": \"" + escape(msg == null ? "" : msg) + - "\", \"stacktrace\": \"" + escape(trace) + - "\", \"type\": \"" + exception.getClass().getSimpleName() + "\"}"; + String result = """ + { + "message": %s, + "stacktrace": %s, + "type": "%s" + } + """.formatted(asJson(msg), asJson(trace), exception.getClass().getSimpleName()); writer.write(result); } private static String convertMessage(EvaluationResult.Message message) { - String result = "{" + - "\"description\": \"" + message.description + "\"," + - "\"format\": \"" + message.format + "\""; - if (message.permission instanceof String) { - result += ", \"permission\": \"" + message.permission + "\""; - } - return result + "}"; + var description = asJson(message.description); + var format = asJson(message.format); + var permission = asJson(message.permission); + + return """ + { + "description": %s, + "format": %s, + "permission": %s + } + """.formatted(description, format, permission); } - - public static void evaluated(PrintWriter writer, - boolean result, String expected, String actual, Collection messages) { - List converted = messages.stream().map(Values::convertMessage).collect(Collectors.toList()); - String builder = "{" + - "\"result\": " + - result + - ", \"readable_expected\": \"" + expected + "\"" + - ", \"readable_actual\": \"" + actual + "\"" + - ", \"messages\": [" + - String.join(", ", converted) + - "]}"; - writer.print(builder); + + private static String asJson(String value) { + if (value == null) { + return "null"; + } else { + return "\"%s\"".formatted(escape(value)); + } } public static void sendEvaluated(PrintWriter writer, EvaluationResult r) { - evaluated(writer, r.result, r.readableExpected, r.readableActual, r.messages); + List converted = r.messages.stream().map(Values::convertMessage).collect(Collectors.toList()); + var readableExpected = asJson(r.readableExpected); + var readableActual = asJson(r.readableActual); + var dslExpected = asJson(r.dslExpected); + var dslActual = asJson(r.dslActual); + var messages = String.join(", ", converted); + + String result = """ + { + "result": %b, + "readable_expected": %s, + "readable_actual": %s, + "dsl_expected": %s, + "dsl_actual": %s, + "messages": [%s] + } + """.formatted(r.result, readableExpected, readableActual, dslExpected, dslActual, messages); + + writer.print(result); } } diff --git a/tested/languages/javascript/generators.py b/tested/languages/javascript/generators.py index 5e9a94be..f2432934 100644 --- a/tested/languages/javascript/generators.py +++ b/tested/languages/javascript/generators.py @@ -275,6 +275,6 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += f"values.sendValue(process.stdout.fd, {convert_value(value)});\n" - result += 'fs.writeSync(process.stdout.fd, "\\n");\n' + result += 'fs.writeSync(process.stdout.fd, "␞");\n' return result diff --git a/tested/languages/kotlin/generators.py b/tested/languages/kotlin/generators.py index b73d7a44..6b3b1e68 100644 --- a/tested/languages/kotlin/generators.py +++ b/tested/languages/kotlin/generators.py @@ -422,7 +422,7 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += " " * 4 + f"valuesSend(writer, {convert_value(value)})\n" - result += " " * 4 + 'writer.write("\\n")\n' + result += " " * 4 + 'writer.write("␞")\n' result += " " * 4 + "writer.close()\n" result += "}\n" diff --git a/tested/languages/kotlin/templates/EvaluationResult.kt b/tested/languages/kotlin/templates/EvaluationResult.kt index 0c745ce2..db0b2c50 100644 --- a/tested/languages/kotlin/templates/EvaluationResult.kt +++ b/tested/languages/kotlin/templates/EvaluationResult.kt @@ -2,6 +2,8 @@ class EvaluationResult private constructor( val result: Boolean, val readableExpected: String?, val readableActual: String?, + val dslExpected: String?, + val dslActual: String?, val messages: List ) { class Message(val description: String, @@ -11,7 +13,9 @@ class EvaluationResult private constructor( class Builder( private val result: Boolean, private var readableActual: String? = null, - private var readableExpected: String? = null + private var readableExpected: String? = null, + private var dslActual: String? = null, + private var dslExpected: String? = null ) { private val messages: MutableList = ArrayList() @@ -25,6 +29,16 @@ class EvaluationResult private constructor( return this } + fun withDslActual(dslActual: String?): Builder { + this.dslActual = dslActual + return this + } + + fun withDslExpected(dslExpected: String?): Builder { + this.dslExpected = dslExpected + return this + } + fun withMessages(messages: List): Builder { this.messages += messages return this @@ -37,7 +51,7 @@ class EvaluationResult private constructor( fun build(): EvaluationResult { return EvaluationResult(result, readableExpected, - readableActual, messages) + readableActual, dslExpected, dslActual, messages) } } } diff --git a/tested/languages/kotlin/templates/Values.kt b/tested/languages/kotlin/templates/Values.kt index ffdb4bd0..aef14142 100644 --- a/tested/languages/kotlin/templates/Values.kt +++ b/tested/languages/kotlin/templates/Values.kt @@ -4,15 +4,13 @@ import java.math.BigDecimal import java.math.BigInteger private fun convertMessage(message: EvaluationResult.Message): String { - val builder = StringBuilder(64).append('{') - .append("\"description\": \"").append(message.description) - .append("\", ") - .append("\"format\": \"").append(message.format).append("\"") - if (message.permission != null) { - builder.append(", \"permission\": \"").append(message.permission) - .append('\"') - } - return builder.append('}').toString() + return """ + { + "description": ${asJson(message.description)}, + "format": ${asJson(message.format)}, + "permission": ${asJson(message.permission)} + } + """.trimIndent() } private fun escape(str: String): String { @@ -142,22 +140,29 @@ private fun internalEncode(value: Any?): Array { return arrayOf(type, data, diagnostic) } -fun evaluated(writer: PrintWriter, result: Boolean, expected: String?, - actual: String?, messages: Iterable) { - val builder = StringBuilder(64).append('{') - .append("\"result\": ").append(result).append(", ") - .append("\"readable_expected\": \"").append(expected).append("\", ") - .append("\"readable_actual\": \"").append(actual).append("\", ") - .append("\"messages\": [").append(messages.joinToString(separator = ", ", - transform = { m -> convertMessage(m) })).append("]}") - writer.print(builder.toString()) +fun asJson(value: String?): String { + if (value == null) { + return "null"; + } else { + return "\"${escape(value)}\""; + } } fun valuesSend(writer: PrintWriter, value: Any?): Unit = writer.print(encode(value)) fun valuesSendEvaluated(writer: PrintWriter, result: EvaluationResult) { - evaluated(writer, result.result, result.readableExpected, result.readableActual, - result.messages) + val messages = result.messages.joinToString(separator = ", ", transform = { m -> convertMessage(m) }) + val result = """ + { + "result": ${result.result}, + "readable_expected": ${asJson(result.readableExpected)}, + "readable_actual": ${asJson(result.readableActual)}, + "dsl_expected": ${asJson(result.dslExpected)}, + "dsl_actual": ${asJson(result.dslActual)}, + "messages": [${messages}] + } + """.trimIndent() + writer.print(result) } fun valuesSendException(writer: PrintWriter, throwable: Throwable?) { @@ -166,9 +171,11 @@ fun valuesSendException(writer: PrintWriter, throwable: Throwable?) { } val strStackTraceWriter = StringWriter() throwable.printStackTrace(PrintWriter(strStackTraceWriter)) - writer.printf("{ \"message\": \"%s\", \"stacktrace\": \"%s\", \"type\": \"%s\"}", - escape(throwable.message ?: ""), - escape(strStackTraceWriter.toString()), - throwable::class.simpleName - ) + writer.printf(""" + { + "message": "${escape(throwable.message ?: "")}", + "stacktrace": "${escape(strStackTraceWriter.toString())}", + "type": "${throwable::class.simpleName}" + } + """.trimIndent()) } diff --git a/tested/languages/python/generators.py b/tested/languages/python/generators.py index 2961275d..82965aaa 100644 --- a/tested/languages/python/generators.py +++ b/tested/languages/python/generators.py @@ -264,5 +264,5 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += f"values.send_value(sys.stdout, {convert_value(value)})\n" - result += "print()\n" + result += "print('␞')\n" return result diff --git a/tested/languages/python/templates/evaluation_utils.py b/tested/languages/python/templates/evaluation_utils.py index 74fc7f25..3f6d5601 100644 --- a/tested/languages/python/templates/evaluation_utils.py +++ b/tested/languages/python/templates/evaluation_utils.py @@ -15,3 +15,5 @@ class EvaluationResult: readable_expected: Optional[str] = None readable_actual: Optional[str] = None messages: List[Message] = field(default_factory=list) + dsl_expected: Optional[str] = None + dsl_actual: Optional[str] = None diff --git a/tested/oracles/common.py b/tested/oracles/common.py index ef3712c6..9d06dd58 100644 --- a/tested/oracles/common.py +++ b/tested/oracles/common.py @@ -31,8 +31,10 @@ def evaluate_text(configs, channel, actual): from tested.configs import Bundle from tested.dodona import Message, Status, StatusMessage +from tested.dsl import parse_string +from tested.languages.generation import generate_statement from tested.languages.utils import convert_stacktrace_to_clickable_feedback -from tested.serialisation import EvalResult +from tested.parsing import fallback_field, get_converter from tested.testsuite import ExceptionOutputChannel, NormalOutputChannel, OutputChannel @@ -51,6 +53,74 @@ class OracleResult: ) +@fallback_field( + get_converter(), + { + "readableExpected": "readable_expected", + "readableActual": "readable_actual", + "dslExpected": "dsl_expected", + "dslActual": "dsl_actual", + }, +) +@define +class BooleanEvalResult: + """ + Allows a boolean result. + + Note: this class is used directly in the Python oracle, so keep it backwards + compatible (also positional arguments) or make a new class for the oracle. + """ + + result: bool | Status + readable_expected: str | None = None + readable_actual: str | None = None + messages: list[Message] = field(factory=list) + dsl_expected: str | None = None + dsl_actual: str | None = None + + def to_oracle_result( + self, bundle: Bundle, channel: NormalOutputChannel + ) -> OracleResult: + if isinstance(self.result, Status): + status = self.result + else: + status = Status.CORRECT if self.result else Status.WRONG + + if self.readable_expected: + readable_expected = self.readable_expected + elif self.dsl_expected: + parsed_statement = parse_string(self.dsl_expected, True) + readable_expected = generate_statement(bundle, parsed_statement) + else: + readable_expected = "" + if self.readable_actual: + readable_actual = self.readable_actual + elif self.dsl_actual: + parsed_statement = parse_string(self.dsl_actual, True) + readable_actual = generate_statement(bundle, parsed_statement) + else: + readable_actual = "" + messages = self.messages + + if isinstance(channel, ExceptionOutputChannel): + readable_expected = bundle.language.cleanup_stacktrace(readable_expected) + message = convert_stacktrace_to_clickable_feedback( + bundle.language, readable_actual + ) + if message: + messages.append(message) + + if status == Status.CORRECT: + readable_actual = "" + + return OracleResult( + result=StatusMessage(enum=status), + readable_expected=readable_expected, + readable_actual=readable_actual, + messages=messages, + ) + + @define class OracleConfig: bundle: Bundle @@ -86,24 +156,3 @@ def try_outputs( if possible is not None: return possible, msg return actual, None - - -def cleanup_specific_programmed( - config: OracleConfig, channel: NormalOutputChannel, actual: EvalResult -) -> EvalResult: - if isinstance(channel, ExceptionOutputChannel): - lang_config = config.bundle.language - actual.readable_expected = lang_config.cleanup_stacktrace( - actual.readable_expected or "" - ) - message = convert_stacktrace_to_clickable_feedback( - lang_config, actual.readable_actual - ) - - if message: - actual.messages.append(message) - - if actual.result == Status.CORRECT: - actual.readable_actual = "" - - return actual diff --git a/tested/oracles/programmed.py b/tested/oracles/programmed.py index 7af28b5b..9fba6c5c 100644 --- a/tested/oracles/programmed.py +++ b/tested/oracles/programmed.py @@ -5,14 +5,9 @@ from tested.internationalization import get_i18n_string from tested.judge.programmed import evaluate_programmed from tested.judge.utils import BaseExecutionResult -from tested.oracles.common import ( - OracleConfig, - OracleResult, - cleanup_specific_programmed, -) +from tested.oracles.common import BooleanEvalResult, OracleConfig, OracleResult from tested.oracles.value import get_values from tested.parsing import get_converter -from tested.serialisation import BooleanEvalResult, EvalResult from tested.testsuite import CustomCheckOracle, OracleOutputChannel, OutputChannel _logger = logging.getLogger(__name__) @@ -37,7 +32,7 @@ def evaluate( # This is slightly tricky, since the actual value must also be converted # to a value, and we are not yet sure what the actual value is exactly result = get_values(config.bundle, channel, actual_str or "") - # TODO: why is this? + # If an error occurred, we get a result directly. if isinstance(result, OracleResult): return result else: @@ -61,6 +56,7 @@ def evaluate( ) if isinstance(result, BaseExecutionResult): + _logger.error(result.stderr) if result.timeout: return OracleResult( result=StatusMessage(enum=Status.TIME_LIMIT_EXCEEDED), @@ -86,9 +82,7 @@ def evaluate( messages=[stdout, stderr, DEFAULT_STUDENT], ) try: - evaluation_result = ( - get_converter().loads(result.stdout, BooleanEvalResult).as_eval_result() - ) + evaluation_result = get_converter().loads(result.stdout, BooleanEvalResult) except Exception as e: _logger.exception(e) messages: list[Message] = [ @@ -129,35 +123,7 @@ def evaluate( messages=messages, ) else: - assert isinstance(result, EvalResult) + assert isinstance(result, BooleanEvalResult) evaluation_result = result - if evaluation_result.readable_expected: - readable_expected = evaluation_result.readable_expected - if evaluation_result.readable_actual: - readable_actual = evaluation_result.readable_actual - - if isinstance(evaluation_result.result, Status): - result_status = StatusMessage(enum=evaluation_result.result) - else: - assert isinstance(evaluation_result.result, bool) - result_status = StatusMessage( - enum=Status.CORRECT if evaluation_result.result else Status.WRONG - ) - cleaned = cleanup_specific_programmed( - config, - channel, - EvalResult( - result=result_status.enum, - readable_expected=readable_expected, # type: ignore - readable_actual=readable_actual, # type: ignore - messages=evaluation_result.messages, - ), - ) - - return OracleResult( - result=result_status, - readable_expected=cleaned.readable_expected or "", - readable_actual=cleaned.readable_actual or "", - messages=cleaned.messages, - ) + return evaluation_result.to_oracle_result(config.bundle, channel) diff --git a/tested/oracles/specific.py b/tested/oracles/specific.py index 78f47709..4cd456c5 100644 --- a/tested/oracles/specific.py +++ b/tested/oracles/specific.py @@ -6,13 +6,8 @@ from tested.dodona import ExtendedMessage, Permission, Status, StatusMessage from tested.internationalization import get_i18n_string -from tested.oracles.common import ( - OracleConfig, - OracleResult, - cleanup_specific_programmed, -) +from tested.oracles.common import BooleanEvalResult, OracleConfig, OracleResult from tested.parsing import get_converter -from tested.serialisation import BooleanEvalResult from tested.testsuite import LanguageSpecificOracle, OracleOutputChannel, OutputChannel _logger = logging.getLogger(__name__) @@ -40,7 +35,7 @@ def evaluate( ) try: - actual = get_converter().loads(actual_str, BooleanEvalResult).as_eval_result() + actual = get_converter().loads(actual_str, BooleanEvalResult) except Exception as e: _logger.exception(e) staff_message = ExtendedMessage( @@ -61,11 +56,4 @@ def evaluate( messages=[staff_message, student_message], ) - actual = cleanup_specific_programmed(config, channel, actual) - - return OracleResult( - result=StatusMessage(enum=actual.result), - readable_expected=actual.readable_expected or "", - readable_actual=actual.readable_actual or "", - messages=actual.messages, - ) + return actual.to_oracle_result(config.bundle, channel) diff --git a/tested/serialisation.py b/tested/serialisation.py index 6f0a1020..26fc0ccb 100644 --- a/tested/serialisation.py +++ b/tested/serialisation.py @@ -50,9 +50,8 @@ StringTypes, resolve_to_basic, ) -from tested.dodona import Message, Status from tested.features import Construct, FeatureSet, WithFeatures, combine_features -from tested.parsing import fallback_field, get_converter, parse_json_value +from tested.parsing import parse_json_value from tested.utils import flatten, sorted_no_duplicates logger = logging.getLogger(__name__) @@ -688,46 +687,6 @@ def to_python_comparable(value: Value | None) -> Any: raise AssertionError(f"Unknown value type: {value}") -@fallback_field( - get_converter(), - {"readableExpected": "readable_expected", "readableActual": "readable_actual"}, -) -@define -class EvalResult: - result: Status - readable_expected: str | None = None - readable_actual: str | None = None - messages: list[Message] = field(factory=list) - - -@fallback_field( - get_converter(), - {"readableExpected": "readable_expected", "readableActual": "readable_actual"}, -) -@define -class BooleanEvalResult: - """ - Allows a boolean result. - """ - - result: bool | Status - readable_expected: str | None = None - readable_actual: str | None = None - messages: list[Message] = field(factory=list) - - def as_eval_result(self) -> EvalResult: - if isinstance(self.result, Status): - status = self.result - else: - status = Status.CORRECT if self.result else Status.WRONG - return EvalResult( - result=status, - readable_expected=self.readable_expected, - readable_actual=self.readable_actual, - messages=self.messages, - ) - - @define class ExceptionValue: """An exception that was thrown while executing the user context.""" diff --git a/tests/exercises/echo-function/evaluation/Evaluator.cs b/tests/exercises/echo-function/evaluation/Evaluator.cs index e16a280a..d56695a2 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.cs +++ b/tests/exercises/echo-function/evaluation/Evaluator.cs @@ -12,4 +12,9 @@ public static EvaluationResult EvaluateValue(Object expected, Object actual, ILi var messages = new List() {new Tested.Message("Hallo")}; return new EvaluationResult(expected == actual, expected.ToString(), actual != null ? actual.ToString() : "", messages); } + + public static EvaluationResult EvaluateValueDsl(Object expected, Object actual, IList arguments) { + var messages = new List() {new Tested.Message("Hallo")}; + return new EvaluationResult(expected == actual, null, null, messages, "{5, 5}", "{4, 4}"); + } } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.java b/tests/exercises/echo-function/evaluation/Evaluator.java index b7551587..ebae008a 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.java +++ b/tests/exercises/echo-function/evaluation/Evaluator.java @@ -18,4 +18,12 @@ public static EvaluationResult evaluateValue(Object expected, Object actual, Lis .withMessage(new EvaluationResult.Message("Hallo")) .build(); } + + public static EvaluationResult evaluateValueDsl(Object expected, Object actual, List arguments) { + return EvaluationResult.builder(expected.equals(actual)) + .withDslExpected("{5, 5}") + .withDslActual("{4, 4}") + .withMessage(new EvaluationResult.Message("Hallo")) + .build(); + } } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.kt b/tests/exercises/echo-function/evaluation/Evaluator.kt index 31cbe7dd..777a8721 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.kt +++ b/tests/exercises/echo-function/evaluation/Evaluator.kt @@ -17,5 +17,14 @@ class Evaluator { .withMessage(EvaluationResult.Message("Hallo")) .build() } + + @JvmStatic + fun evaluateValueDsl(expected: Any, actual: Any?, arguments: List?): EvaluationResult { + return EvaluationResult.Builder(result = expected == actual, + dslExpected = "{5, 5}", + dslActual = "{4, 4}") + .withMessage(EvaluationResult.Message("Hallo")) + .build() + } } -} \ No newline at end of file +} diff --git a/tests/exercises/echo-function/evaluation/evaluator.js b/tests/exercises/echo-function/evaluation/evaluator.js index 632c4e3b..2a5ffd93 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.js +++ b/tests/exercises/echo-function/evaluation/evaluator.js @@ -11,11 +11,21 @@ function evaluate(actual) { function evaluateValue(expected, actual, args) { return { "result": expected === actual, - "expected": expected, - "actual": actual, + "readable_expected": expected, + "readable_actual": actual, + "messages": [{"description": "Hallo", "format": "text"}] + } +} + +function evaluateValueDsl(expected, actual, args) { + return { + "result": expected === actual, + "dsl_expected": "{5, 5}", + "dsl_actual": "{4, 4}", "messages": [{"description": "Hallo", "format": "text"}] } } exports.evaluate = evaluate; -exports.evaluateValue = evaluateValue; \ No newline at end of file +exports.evaluateValue = evaluateValue; +exports.evaluateValueDsl = evaluateValueDsl; diff --git a/tests/exercises/echo-function/evaluation/evaluator.py b/tests/exercises/echo-function/evaluation/evaluator.py index 5cef55d6..6f177c1f 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.py +++ b/tests/exercises/echo-function/evaluation/evaluator.py @@ -8,3 +8,12 @@ def evaluate(actual): def evaluate_value(expected, actual, args): return EvaluationResult(expected == actual, expected, actual, [Message("Hallo")]) + + +def evaluate_value_dsl(expected, actual, args): + return EvaluationResult( + result=expected == actual, + messages=[Message("Hallo")], + dsl_expected="{5, 5}", + dsl_actual="{4, 4}" + ) diff --git a/tests/exercises/echo-function/evaluation/programmed-dsl-no-haskell.tson b/tests/exercises/echo-function/evaluation/programmed-dsl-no-haskell.tson new file mode 100644 index 00000000..65a6703f --- /dev/null +++ b/tests/exercises/echo-function/evaluation/programmed-dsl-no-haskell.tson @@ -0,0 +1,157 @@ +{ + "tabs": [ + { + "name": "Tab", + "runs": [ + { + "contexts": [ + { + "testcases": [ + { + "input": { + "type": "function", + "name": "echo", + "arguments": [ + { + "type": "text", + "data": "input-2" + } + ] + }, + "output": { + "result": { + "value": { + "type": "text", + "data": "input-2" + }, + "evaluator": { + "type": "programmed", + "function": { + "file": "Evaluator.java", + "name": "evaluateValueDsl" + }, + "language": "java" + } + } + } + }, + { + "input": { + "type": "function", + "name": "echo", + "arguments": [ + { + "type": "text", + "data": "input-3" + } + ] + }, + "output": { + "result": { + "value": { + "type": "text", + "data": "input-3" + }, + "evaluator": { + "type": "programmed", + "function": { + "file": "evaluator.py", + "name": "evaluate_value_dsl" + }, + "language": "python" + } + } + } + }, + { + "input": { + "type": "function", + "name": "echo", + "arguments": [ + { + "type": "text", + "data": "input-4" + } + ] + }, + "output": { + "result": { + "value": { + "type": "text", + "data": "input-4" + }, + "evaluator": { + "type": "programmed", + "function": { + "file": "evaluator.js", + "name": "evaluateValueDsl" + }, + "language": "javascript" + } + } + } + }, + { + "input": { + "type": "function", + "name": "echo", + "arguments": [ + { + "type": "text", + "data": "input-5" + } + ] + }, + "output": { + "result": { + "value": { + "type": "text", + "data": "input-5" + }, + "evaluator": { + "type": "programmed", + "function": { + "file": "Evaluator.kt", + "name": "evaluateValueDsl" + }, + "language": "kotlin" + } + } + } + }, + { + "input": { + "type": "function", + "name": "echo", + "arguments": [ + { + "type": "text", + "data": "input-89" + } + ] + }, + "output": { + "result": { + "value": { + "type": "text", + "data": "input-89" + }, + "evaluator": { + "type": "programmed", + "function": { + "file": "Evaluator.cs", + "name": "EvaluateValueDsl" + }, + "language": "csharp" + } + } + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/tests/test_functionality.py b/tests/test_functionality.py index ba16466c..38f265cd 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -293,6 +293,25 @@ def test_programmed_evaluation(language: str, tmp_path: Path, pytestconfig): assert len(updates.find_all("append-message")) == 5 +def test_programmed_evaluation_with_dsl(tmp_path: Path, pytestconfig): + conf = configuration( + pytestconfig, + "echo-function", + "javascript", + tmp_path, + "programmed-dsl-no-haskell.tson", + "correct", + ) + result = execute_config(conf) + updates = assert_valid_output(result, pytestconfig) + assert updates.find_status_enum() == ["correct"] * 5 + assert len(updates.find_all("append-message")) == 5 + all_expected = [x["expected"] for x in updates.find_all("start-test")] + assert all_expected == ["new Set([5, 5])"] * 5 + all_actual = [x["generated"] for x in updates.find_all("close-test")] + assert all_actual == ["new Set([4, 4])"] * 5 + + @pytest.mark.parametrize( "lang", [ diff --git a/tests/test_serialisation.py b/tests/test_serialisation.py index e740ac7d..8eee9b1c 100644 --- a/tests/test_serialisation.py +++ b/tests/test_serialisation.py @@ -194,7 +194,7 @@ def run_encoder(bundle: Bundle, values: list[Value]) -> list[str]: # Run the code. r = execute_file(bundle, executable.name, dest, None) print(r.stderr) - return r.stdout.splitlines(keepends=False) + return r.stdout.split(sep="␞")[:-1] def assert_serialisation(bundle: Bundle, expected: Value): From f545385684c9608bad3ddd5036704d6a657a8c13 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Wed, 20 Sep 2023 13:46:30 +0200 Subject: [PATCH 2/8] Make exit code ignored by default --- tested/testsuite.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/tested/testsuite.py b/tested/testsuite.py index 8ceda458..276fa204 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -412,8 +412,7 @@ class Output(WithFeatures): file: FileOutput = IgnoredChannel.IGNORED exception: ExceptionOutput = EmptyChannel.NONE result: ValueOutput = EmptyChannel.NONE - # This default value is adjusted later, based on the position of the output. - exit_code: ExitOutput = EmptyChannel.NONE + exit_code: ExitOutput = IgnoredChannel.IGNORED def get_used_features(self) -> FeatureSet: return combine_features( @@ -585,14 +584,6 @@ class Context(WithFeatures, WithFunctions): after: Code = field(factory=dict) description: str | None = None - def __attrs_post_init__(self): - # Fix the default value of the exit code outputs. - for non_last_testcase in self.testcases[:-1]: - if non_last_testcase.output.exit_code == EmptyChannel.NONE: - non_last_testcase.output.exit_code = IgnoredChannel.IGNORED - if (last_testcase := self.testcases[-1]).output.exit_code == EmptyChannel.NONE: - last_testcase.output.exit_code = ExitCodeOutputChannel() - @testcases.validator # type: ignore def check_testcases(self, _, value: list[Testcase]): # Check that only the first testcase has a main call. From 77afb438b7300b7577846800f1f5be6bdabd2d4f Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Wed, 20 Sep 2023 13:55:41 +0200 Subject: [PATCH 3/8] Fix bugs with exception handling --- tested/oracles/exception.py | 1 - tested/oracles/nothing.py | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tested/oracles/exception.py b/tested/oracles/exception.py index 9264b856..b8acb1af 100644 --- a/tested/oracles/exception.py +++ b/tested/oracles/exception.py @@ -24,7 +24,6 @@ def try_as_readable_exception( # noinspection PyBroadException try: actual = get_converter().loads(value, ExceptionValue) - actual.stacktrace = config.bundle.language.cleanup_stacktrace(actual.stacktrace) except Exception: return None, None else: diff --git a/tested/oracles/nothing.py b/tested/oracles/nothing.py index c0a4887c..bb5fce73 100644 --- a/tested/oracles/nothing.py +++ b/tested/oracles/nothing.py @@ -40,7 +40,11 @@ def evaluate_no_return( expected = NothingType() try: - value = parse_value(actual) + if actual == "": + # If there was an exception, we still get the empty string. + value = NothingType() + else: + value = parse_value(actual) except Exception as e: raw_message = f"Could not parse {actual}, which caused {e} for get_values." message = ExtendedMessage( From e470f3e82cefafef2f22aefa052cf0f658d71638 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Wed, 20 Sep 2023 14:01:27 +0200 Subject: [PATCH 4/8] Better fix for wrong line numbers --- tested/languages/javascript/config.py | 8 -------- tested/oracles/exception.py | 5 ++--- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/tested/languages/javascript/config.py b/tested/languages/javascript/config.py index 52d6f678..065753d9 100644 --- a/tested/languages/javascript/config.py +++ b/tested/languages/javascript/config.py @@ -165,11 +165,6 @@ def cleanup_stacktrace(self, traceback: str) -> str: compilation_submission_location = str(submission_location.resolve()) execution_location_regex = f"{self.config.dodona.workdir}/{EXECUTION_PREFIX}[_0-9]+/{EXECUTION_PREFIX}[_0-9]+.js" submission_namespace = f"{submission_name(self)}." - location_pattern = r"(\d+):(\d+)" - - def update_line_number(match: re.Match) -> str: - assert self.config - return f"{int(match.group(1)) + self.config.dodona.source_offset}:{match.group(2)}" resulting_lines = "" for line in traceback.splitlines(keepends=True): @@ -183,9 +178,6 @@ def update_line_number(match: re.Match) -> str: # Remove any references of the form "submission.SOMETHING" line = line.replace(submission_namespace, "") - # Adjust line numbers - line = re.sub(location_pattern, update_line_number, line) - resulting_lines += line return resulting_lines diff --git a/tested/oracles/exception.py b/tested/oracles/exception.py index b8acb1af..30f3aab8 100644 --- a/tested/oracles/exception.py +++ b/tested/oracles/exception.py @@ -12,9 +12,8 @@ _logger = logging.getLogger(__name__) -def try_as_exception(config: OracleConfig, value: str) -> ExceptionValue: +def try_as_exception(value: str) -> ExceptionValue: actual = get_converter().loads(value, ExceptionValue) - actual.stacktrace = config.bundle.language.cleanup_stacktrace(actual.stacktrace) return actual @@ -62,7 +61,7 @@ def evaluate( ) try: - actual = try_as_exception(config, actual_str) + actual = try_as_exception(actual_str) except Exception as e: _logger.exception(e) staff_message = ExtendedMessage( From 6e8eff1f4ca66ba9841b9a4e7da45f7164399ab9 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Wed, 20 Sep 2023 15:38:14 +0200 Subject: [PATCH 5/8] Support "!v" in more places in DSL --- tested/dsl/translate_parser.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index c68004bf..28acfc4a 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -303,14 +303,17 @@ def _convert_file(link_file: YamlDict) -> FileUrl: def _convert_custom_check_oracle(stream: dict) -> CustomCheckOracle: + converted_args = [] + for v in stream.get("arguments", []): + cv = _convert_yaml_value(v) + assert isinstance(cv, Value) + converted_args.append(cv) return CustomCheckOracle( language=stream["language"], function=EvaluationFunction( file=stream["file"], name=stream.get("name", "evaluate") ), - arguments=[ - parse_string(v, is_return=True) for v in stream.get("arguments", []) - ], + arguments=converted_args, ) @@ -338,27 +341,30 @@ def _convert_text_output_channel( raise TypeError(f"Unknown text oracle type: {stream['oracle']}") -def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel: +def _convert_yaml_value(stream: YamlObject) -> Value | None: if isinstance(stream, YamlValue): # A normal yaml type tagged explicitly. value = _convert_value(stream.value) - assert isinstance(value, Value) - return ValueOutputChannel(value=value) - if isinstance(stream, (int, float, bool, TestedType, list, set)): + elif isinstance(stream, (int, float, bool, TestedType, list, set)): # Simple values where no confusion is possible. value = _convert_value(stream) - assert isinstance(value, Value) - return ValueOutputChannel(value=value) elif isinstance(stream, str): # A normal YAML string is considered a "Python" string. value = parse_string(stream, is_return=True) - assert isinstance(value, Value) - return ValueOutputChannel(value=value) + else: + return None + assert isinstance(value, Value) + return value + + +def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel: + yaml_value = _convert_yaml_value(stream) + if yaml_value: + return ValueOutputChannel(value=yaml_value) else: # We have an object, which means we have an output channel. assert isinstance(stream, dict) - assert isinstance(stream["value"], str) - value = parse_string(stream["value"], is_return=True) + value = _convert_yaml_value(stream["value"]) assert isinstance(value, Value) if "oracle" not in stream or stream["oracle"] == "builtin": return ValueOutputChannel(value=value) From 1cdafd382d0c7a68587c8355b5e7ac706eefb409 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Wed, 20 Sep 2023 17:25:24 +0200 Subject: [PATCH 6/8] Better fallback of values for oracles --- tested/oracles/common.py | 10 +++++++--- tested/oracles/programmed.py | 4 +++- tested/oracles/specific.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tested/oracles/common.py b/tested/oracles/common.py index 9d06dd58..0859fb2a 100644 --- a/tested/oracles/common.py +++ b/tested/oracles/common.py @@ -79,7 +79,11 @@ class BooleanEvalResult: dsl_actual: str | None = None def to_oracle_result( - self, bundle: Bundle, channel: NormalOutputChannel + self, + bundle: Bundle, + channel: NormalOutputChannel, + fallback_actual: str, + fallback_expected: str, ) -> OracleResult: if isinstance(self.result, Status): status = self.result @@ -92,14 +96,14 @@ def to_oracle_result( parsed_statement = parse_string(self.dsl_expected, True) readable_expected = generate_statement(bundle, parsed_statement) else: - readable_expected = "" + readable_expected = fallback_expected if self.readable_actual: readable_actual = self.readable_actual elif self.dsl_actual: parsed_statement = parse_string(self.dsl_actual, True) readable_actual = generate_statement(bundle, parsed_statement) else: - readable_actual = "" + readable_actual = fallback_actual messages = self.messages if isinstance(channel, ExceptionOutputChannel): diff --git a/tested/oracles/programmed.py b/tested/oracles/programmed.py index 9fba6c5c..10be88f6 100644 --- a/tested/oracles/programmed.py +++ b/tested/oracles/programmed.py @@ -126,4 +126,6 @@ def evaluate( assert isinstance(result, BooleanEvalResult) evaluation_result = result - return evaluation_result.to_oracle_result(config.bundle, channel) + return evaluation_result.to_oracle_result( + config.bundle, channel, readable_actual, readable_expected + ) diff --git a/tested/oracles/specific.py b/tested/oracles/specific.py index 4cd456c5..d4f881b7 100644 --- a/tested/oracles/specific.py +++ b/tested/oracles/specific.py @@ -56,4 +56,4 @@ def evaluate( messages=[staff_message, student_message], ) - return actual.to_oracle_result(config.bundle, channel) + return actual.to_oracle_result(config.bundle, channel, "", "") From b85b4cfdcaa886362e4e3c9aa618e3f96ea63b13 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Wed, 20 Sep 2023 18:06:23 +0200 Subject: [PATCH 7/8] Don't show empty and correct return channels --- tested/judge/evaluation.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index caf1e3f7..bbbf27eb 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -33,8 +33,10 @@ get_readable_input, ) from tested.oracles import get_oracle +from tested.oracles.common import OracleResult from tested.testsuite import ( Context, + EmptyChannel, ExceptionOutput, ExceptionOutputChannel, ExitCodeOutputChannel, @@ -113,7 +115,7 @@ def _evaluate_channel( # Decide if we should show this channel or not. is_correct = status.enum == Status.CORRECT - should_report_case = should_show(output, channel) + should_report_case = should_show(output, channel, evaluation_result) if not should_report_case and is_correct: # We do report that a test is correct, to set the status. @@ -363,7 +365,9 @@ def _link_files_message(link_files: Collection[FileUrl]) -> AppendMessage: return AppendMessage(message=message) -def should_show(test: OutputChannel, channel: Channel) -> bool: +def should_show( + test: OutputChannel, channel: Channel, result: OracleResult | None = None +) -> bool: """ Determine if the channel should be shown, without accounting for the actual value. This function answers the question: "Assuming the actual value is @@ -371,6 +375,7 @@ def should_show(test: OutputChannel, channel: Channel) -> bool: :param test: The output for the channel from the test suite. :param channel: The channel. + :param result: The result of the evaluation. :return: True if the channel should be shown, false otherwise. """ @@ -390,7 +395,15 @@ def should_show(test: OutputChannel, channel: Channel) -> bool: elif channel == Channel.RETURN: assert isinstance(test, ValueOutput) # We don't show the channel if we ignore it. - return not isinstance(test, IgnoredChannel) + if isinstance(test, IgnoredChannel): + return False + if ( + isinstance(test, EmptyChannel) + and result + and result.result.enum == Status.CORRECT + ): + return False + return True elif channel == Channel.EXCEPTION: assert isinstance(test, ExceptionOutput) return not isinstance(test, SpecialOutputChannel) From e396a754b89c3cd7933df57e953c4535a1b25ba1 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Thu, 21 Sep 2023 10:43:07 +0200 Subject: [PATCH 8/8] Use individual arguments in custom check functions --- tested/judge/programmed.py | 24 +++++++------------ tested/languages/generation.py | 12 ++-------- .../echo-function/evaluation/Evaluator.cs | 4 ++-- .../echo-function/evaluation/Evaluator.hs | 6 ++--- .../echo-function/evaluation/Evaluator.java | 4 ++-- .../echo-function/evaluation/Evaluator.kt | 4 ++-- .../echo-function/evaluation/evaluator.js | 4 ++-- .../echo-function/evaluation/evaluator.py | 5 ++-- tests/exercises/echo/evaluation/evaluator.py | 5 ++-- tests/exercises/lotto/evaluation/evaluator.py | 4 ++-- tests/test_functionality.py | 2 +- tests/test_oracles.py | 2 +- 12 files changed, 32 insertions(+), 44 deletions(-) diff --git a/tested/judge/programmed.py b/tested/judge/programmed.py index 68d7acf4..379184e6 100644 --- a/tested/judge/programmed.py +++ b/tested/judge/programmed.py @@ -17,13 +17,9 @@ from tested.internationalization import get_i18n_string from tested.judge.execution import execute_file, filter_files from tested.judge.utils import BaseExecutionResult, copy_from_paths_to_path, run_command -from tested.languages.generation import ( - custom_oracle_arguments, - generate_custom_evaluator, - generate_statement, -) +from tested.languages.generation import generate_custom_evaluator, generate_statement from tested.oracles.common import BooleanEvalResult -from tested.serialisation import Value +from tested.serialisation import FunctionCall, FunctionType, Value from tested.testsuite import CustomCheckOracle from tested.utils import get_identifier @@ -214,17 +210,15 @@ def _evaluate_python( exec(evaluator_code, global_env) # Call the oracle. - literal_expected = generate_statement(eval_bundle, expected) - literal_actual = generate_statement(eval_bundle, actual) - arguments = custom_oracle_arguments(oracle) - literal_arguments = generate_statement(eval_bundle, arguments) + check_function_call = FunctionCall( + type=FunctionType.FUNCTION, + name=oracle.function.name, + arguments=[expected, actual, *oracle.arguments], + ) + literal_function_call = generate_statement(eval_bundle, check_function_call) with _catch_output() as (stdout_, stderr_): - exec( - f"__tested_test__result = {oracle.function.name}(" - f"{literal_expected}, {literal_actual}, {literal_arguments})", - global_env, - ) + exec(f"__tested_test__result = {literal_function_call}", global_env) stdout_ = stdout_.getvalue() stderr_ = stderr_.getvalue() diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 502d9066..3c66d8c0 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -17,7 +17,7 @@ from pygments.lexers import get_lexer_by_name from tested.configs import Bundle -from tested.datatypes import AllTypes, BasicObjectTypes, BasicSequenceTypes +from tested.datatypes import AllTypes, BasicObjectTypes from tested.dodona import ExtendedMessage from tested.internationalization import get_i18n_string from tested.judge.planning import PlannedExecutionUnit @@ -40,7 +40,6 @@ Expression, FunctionType, Identifier, - SequenceType, Statement, Value, VariableType, @@ -294,12 +293,6 @@ def generate_selector( return selector_filename -def custom_oracle_arguments(oracle: CustomCheckOracle) -> Value: - return SequenceType( - type=BasicSequenceTypes.SEQUENCE, data=oracle.arguments # pyright: ignore - ) - - def generate_custom_evaluator( bundle: Bundle, destination: Path, @@ -321,13 +314,12 @@ def generate_custom_evaluator( evaluator_name = conventionalize_namespace( bundle.language, evaluator.function.file.stem ) - arguments = custom_oracle_arguments(evaluator) function = PreparedFunctionCall( type=FunctionType.FUNCTION, namespace=Identifier(evaluator_name), name=evaluator.function.name, - arguments=[expected_value, actual_value, arguments], + arguments=[expected_value, actual_value, *evaluator.arguments], has_root_namespace=False, ) diff --git a/tests/exercises/echo-function/evaluation/Evaluator.cs b/tests/exercises/echo-function/evaluation/Evaluator.cs index d56695a2..17f74c73 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.cs +++ b/tests/exercises/echo-function/evaluation/Evaluator.cs @@ -8,12 +8,12 @@ public static EvaluationResult Evaluate(Object actual) { return new EvaluationResult(correct, "correct", actual != null ? actual.ToString() : "", messages); } - public static EvaluationResult EvaluateValue(Object expected, Object actual, IList arguments) { + public static EvaluationResult EvaluateValue(Object expected, Object actual) { var messages = new List() {new Tested.Message("Hallo")}; return new EvaluationResult(expected == actual, expected.ToString(), actual != null ? actual.ToString() : "", messages); } - public static EvaluationResult EvaluateValueDsl(Object expected, Object actual, IList arguments) { + public static EvaluationResult EvaluateValueDsl(Object expected, Object actual) { var messages = new List() {new Tested.Message("Hallo")}; return new EvaluationResult(expected == actual, null, null, messages, "{5, 5}", "{4, 4}"); } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.hs b/tests/exercises/echo-function/evaluation/Evaluator.hs index f3e76c89..586f84aa 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.hs +++ b/tests/exercises/echo-function/evaluation/Evaluator.hs @@ -15,12 +15,12 @@ evaluate value = } -evaluate_value :: String -> String -> [String] -> EvaluationResult -evaluate_value expected actual arguments = +evaluate_value :: String -> String -> EvaluationResult +evaluate_value expected actual = let correct = if actual == expected then True else False in evaluationResult { result = correct, readableExpected = Just expected, readableActual = Just actual, messages = [message "Hallo"] - } \ No newline at end of file + } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.java b/tests/exercises/echo-function/evaluation/Evaluator.java index ebae008a..f9e9a0a9 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.java +++ b/tests/exercises/echo-function/evaluation/Evaluator.java @@ -11,7 +11,7 @@ public static EvaluationResult evaluate(Object actual) { .build(); } - public static EvaluationResult evaluateValue(Object expected, Object actual, List arguments) { + public static EvaluationResult evaluateValue(Object expected, Object actual) { return EvaluationResult.builder(expected.equals(actual)) .withReadableExpected(expected.toString()) .withReadableActual(actual != null ? actual.toString() : "") @@ -19,7 +19,7 @@ public static EvaluationResult evaluateValue(Object expected, Object actual, Lis .build(); } - public static EvaluationResult evaluateValueDsl(Object expected, Object actual, List arguments) { + public static EvaluationResult evaluateValueDsl(Object expected, Object actual) { return EvaluationResult.builder(expected.equals(actual)) .withDslExpected("{5, 5}") .withDslActual("{4, 4}") diff --git a/tests/exercises/echo-function/evaluation/Evaluator.kt b/tests/exercises/echo-function/evaluation/Evaluator.kt index 777a8721..0f693aa6 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.kt +++ b/tests/exercises/echo-function/evaluation/Evaluator.kt @@ -10,7 +10,7 @@ class Evaluator { } @JvmStatic - fun evaluateValue(expected: Any, actual: Any?, arguments: List?): EvaluationResult { + fun evaluateValue(expected: Any, actual: Any?): EvaluationResult { return EvaluationResult.Builder(result = expected == actual, readableExpected = expected.toString(), readableActual = actual?.toString() ?: "") @@ -19,7 +19,7 @@ class Evaluator { } @JvmStatic - fun evaluateValueDsl(expected: Any, actual: Any?, arguments: List?): EvaluationResult { + fun evaluateValueDsl(expected: Any, actual: Any?): EvaluationResult { return EvaluationResult.Builder(result = expected == actual, dslExpected = "{5, 5}", dslActual = "{4, 4}") diff --git a/tests/exercises/echo-function/evaluation/evaluator.js b/tests/exercises/echo-function/evaluation/evaluator.js index 2a5ffd93..0b283517 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.js +++ b/tests/exercises/echo-function/evaluation/evaluator.js @@ -8,7 +8,7 @@ function evaluate(actual) { } } -function evaluateValue(expected, actual, args) { +function evaluateValue(expected, actual) { return { "result": expected === actual, "readable_expected": expected, @@ -17,7 +17,7 @@ function evaluateValue(expected, actual, args) { } } -function evaluateValueDsl(expected, actual, args) { +function evaluateValueDsl(expected, actual) { return { "result": expected === actual, "dsl_expected": "{5, 5}", diff --git a/tests/exercises/echo-function/evaluation/evaluator.py b/tests/exercises/echo-function/evaluation/evaluator.py index 6f177c1f..e4046066 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.py +++ b/tests/exercises/echo-function/evaluation/evaluator.py @@ -1,3 +1,4 @@ +# noinspection PyUnresolvedReferences from evaluation_utils import EvaluationResult, Message @@ -6,11 +7,11 @@ def evaluate(actual): return EvaluationResult(correct, "correct", actual, [Message("Hallo")]) -def evaluate_value(expected, actual, args): +def evaluate_value(expected, actual): return EvaluationResult(expected == actual, expected, actual, [Message("Hallo")]) -def evaluate_value_dsl(expected, actual, args): +def evaluate_value_dsl(expected, actual): return EvaluationResult( result=expected == actual, messages=[Message("Hallo")], diff --git a/tests/exercises/echo/evaluation/evaluator.py b/tests/exercises/echo/evaluation/evaluator.py index cdcf2dcc..0c99dcb1 100644 --- a/tests/exercises/echo/evaluation/evaluator.py +++ b/tests/exercises/echo/evaluation/evaluator.py @@ -1,9 +1,10 @@ +# noinspection PyUnresolvedReferences from evaluation_utils import EvaluationResult -def evaluate_correct(expected, actual, arguments): +def evaluate_correct(expected, actual): return EvaluationResult(expected.strip() == actual.strip()) -def evaluate_wrong(expected, actual, arguments): +def evaluate_wrong(_expected, _actual): return EvaluationResult(False) diff --git a/tests/exercises/lotto/evaluation/evaluator.py b/tests/exercises/lotto/evaluation/evaluator.py index dcec4923..a2c5c87e 100644 --- a/tests/exercises/lotto/evaluation/evaluator.py +++ b/tests/exercises/lotto/evaluation/evaluator.py @@ -1,4 +1,5 @@ import re +# noinspection PyUnresolvedReferences from evaluation_utils import EvaluationResult, Message @@ -39,8 +40,7 @@ def valid_lottery_numbers(number_str, count=6, maximum=42): return True, None -def evaluate(expected, actual, arguments): - count, maximum = arguments +def evaluate(expected, actual, count, maximum): valid, message = valid_lottery_numbers(actual, count, maximum) messages = [Message(message)] if message else [] if valid: diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 38f265cd..517e1de8 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -921,7 +921,7 @@ def test_expected_no_return_and_got_none(language: str, tmp_path: Path, pytestco result = execute_config(conf) updates = assert_valid_output(result, pytestconfig) - assert updates.find_status_enum() == ["correct"] + assert updates.find_status_enum() == [] @pytest.mark.parametrize("language", ALL_SPECIFIC_LANGUAGES) diff --git a/tests/test_oracles.py b/tests/test_oracles.py index 0d7df04a..db8888c5 100644 --- a/tests/test_oracles.py +++ b/tests/test_oracles.py @@ -6,7 +6,7 @@ from tested.configs import create_bundle from tested.datatypes import BasicObjectTypes, BasicSequenceTypes, BasicStringTypes from tested.dodona import Status -from tested.oracles.common import OracleConfig, OracleResult +from tested.oracles.common import OracleConfig from tested.oracles.exception import evaluate as evaluate_exception from tested.oracles.text import evaluate_file, evaluate_text from tested.oracles.value import evaluate as evaluate_value