diff --git a/tested/internationalization/nl.yaml b/tested/internationalization/nl.yaml index f245a2dc..7caa5042 100644 --- a/tested/internationalization/nl.yaml +++ b/tested/internationalization/nl.yaml @@ -77,7 +77,7 @@ nl: Contacteer de lesgever. unsupported: "Evaluation in %{lang} wordt niet ondersteund." unknown: - compilation: "Ongekende compilatie fout" + compilation: "Ongekende compilatiefout" produced: stderr: "De evaluatiecode produceerde volgende uitvoer op stderr:" stdout: "De evaluatiecode produceerde volgende uitvoer op stdout:" @@ -86,7 +86,7 @@ nl: languages: config: unknown: - compilation: "Ongekende compilatie fout" + compilation: "Ongekende compilatiefout" generator: missing: input: "Geen invoer gevonden." diff --git a/tested/judge/programmed.py b/tested/judge/programmed.py index 415267fd..68d7acf4 100644 --- a/tested/judge/programmed.py +++ b/tested/judge/programmed.py @@ -22,7 +22,8 @@ generate_custom_evaluator, generate_statement, ) -from tested.serialisation import BooleanEvalResult, EvalResult, Value +from tested.oracles.common import BooleanEvalResult +from tested.serialisation import Value from tested.testsuite import CustomCheckOracle from tested.utils import get_identifier @@ -34,7 +35,7 @@ def evaluate_programmed( evaluator: CustomCheckOracle, expected: Value, actual: Value, -) -> BaseExecutionResult | EvalResult: +) -> BaseExecutionResult | BooleanEvalResult: """ Run the custom evaluation. Concerning structure and execution, the custom oracle is very similar to the execution of the whole evaluation. It a @@ -181,7 +182,7 @@ def _evaluate_python( oracle: CustomCheckOracle, expected: Value, actual: Value, -) -> EvalResult: +) -> BooleanEvalResult: """ Run an evaluation in Python. While the templates are still used to generate code, they are not executed in a separate process, but inside python itself. @@ -267,13 +268,12 @@ def _evaluate_python( permission=Permission.STAFF, ) ) - return EvalResult( + return BooleanEvalResult( result=Status.INTERNAL_ERROR, readable_expected=None, readable_actual=None, messages=messages, ) - assert isinstance(result_, BooleanEvalResult) result_.messages.extend(messages) - return result_.as_eval_result() + return result_ diff --git a/tested/languages/bash/generators.py b/tested/languages/bash/generators.py index 14d85318..f5a2c8f0 100644 --- a/tested/languages/bash/generators.py +++ b/tested/languages/bash/generators.py @@ -238,7 +238,7 @@ def convert_encoder(values: list[Value]) -> str: }} function send_value {{ - echo "{{\\"type\\": \\"text\\", \\"data\\": $(json_escape "$1")}}" + echo "{{\\"type\\": \\"text\\", \\"data\\": $(json_escape "$1")}}␞" }} """ for value in values: diff --git a/tested/languages/c/generators.py b/tested/languages/c/generators.py index 5b22f49d..0aae333d 100644 --- a/tested/languages/c/generators.py +++ b/tested/languages/c/generators.py @@ -322,6 +322,6 @@ def convert_encoder(values: list[Value]) -> str: """ for value in values: result += " " * 4 + f"write_value(stdout, {convert_value(value)});\n" - result += " " * 4 + 'printf("\\n");\n' + result += " " * 4 + 'printf("␞");\n' result += "}\n" return result diff --git a/tested/languages/c/templates/evaluation_result.h b/tested/languages/c/templates/evaluation_result.h index 0e76a2f0..56e39fcb 100644 --- a/tested/languages/c/templates/evaluation_result.h +++ b/tested/languages/c/templates/evaluation_result.h @@ -15,6 +15,8 @@ typedef struct EvaluationResult { char* readableActual; size_t nrOfMessages; Message** messages; + char* dslExpected; + char* dslActual; } EvaluationResult; diff --git a/tested/languages/csharp/generators.py b/tested/languages/csharp/generators.py index 1fc6f8f6..f31d0b7a 100644 --- a/tested/languages/csharp/generators.py +++ b/tested/languages/csharp/generators.py @@ -478,7 +478,7 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += " " * 6 + f"Values.WriteValue(stdout, {convert_value(value)});" - result += " " * 6 + 'stdout.Write("\\n");\n' + result += " " * 6 + 'stdout.Write("␞");\n' result += "}\n}\n}\n" return result diff --git a/tested/languages/csharp/templates/EvaluationResult.cs b/tested/languages/csharp/templates/EvaluationResult.cs index 35d4e353..b202d3a3 100644 --- a/tested/languages/csharp/templates/EvaluationResult.cs +++ b/tested/languages/csharp/templates/EvaluationResult.cs @@ -3,7 +3,7 @@ namespace Tested { public record Message(string Description, string Format = "text", string? Permission = null); - public record EvaluationResult(bool Result, string? ReadableExpected = null, string? ReadableActual = null, List? Messages = null) + public record EvaluationResult(bool Result, string? ReadableExpected = null, string? ReadableActual = null, List? Messages = null, string? DslExpected = null, string? DslActual = null) { public List Messages { get; init; } = Messages ?? new List(); } diff --git a/tested/languages/haskell/generators.py b/tested/languages/haskell/generators.py index 11e1956a..56349323 100644 --- a/tested/languages/haskell/generators.py +++ b/tested/languages/haskell/generators.py @@ -350,5 +350,5 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += indent + f"sendValueH stdout ({convert_value(value)})\n" - result += indent + 'putStr "\\n"\n' + result += indent + 'putStr "␞"\n' return result diff --git a/tested/languages/haskell/templates/EvaluationUtils.hs b/tested/languages/haskell/templates/EvaluationUtils.hs index c6eb54b6..6478c7a4 100644 --- a/tested/languages/haskell/templates/EvaluationUtils.hs +++ b/tested/languages/haskell/templates/EvaluationUtils.hs @@ -11,7 +11,9 @@ data EvaluationResult = EvaluationResult { result :: Bool, readableExpected :: Maybe (String), readableActual :: Maybe (String), - messages :: [Message] + messages :: [Message], + dslExpected :: Maybe (String), + dslActual :: Maybe (String) } deriving Show message description = Message { @@ -24,5 +26,7 @@ evaluationResult = EvaluationResult { result = False, readableExpected = Nothing, readableActual = Nothing, + dslExpected = Nothing, + dslActual = Nothing, messages = [] } diff --git a/tested/languages/java/generators.py b/tested/languages/java/generators.py index b5ef0d64..9cce196b 100644 --- a/tested/languages/java/generators.py +++ b/tested/languages/java/generators.py @@ -450,7 +450,7 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += " " * 8 + f"Values.send(writer, {convert_value(value)});\n" - result += " " * 8 + 'writer.write("\\n");\n' + result += " " * 8 + 'writer.write("␞");\n' result += " " * 8 + "writer.close();\n" result += " " * 4 + "}\n" diff --git a/tested/languages/java/templates/EvaluationResult.java b/tested/languages/java/templates/EvaluationResult.java index 87bef95d..9702110e 100644 --- a/tested/languages/java/templates/EvaluationResult.java +++ b/tested/languages/java/templates/EvaluationResult.java @@ -6,12 +6,16 @@ public class EvaluationResult { public final boolean result; public final String readableExpected; public final String readableActual; + public final String dslExpected; + public final String dslActual; public final List messages; - private EvaluationResult(boolean result, String readableExpected, String readableActual, List messages) { + private EvaluationResult(boolean result, String readableExpected, String readableActual, String dslExpected, String dslActual, List messages) { this.result = result; this.readableExpected = readableExpected; this.readableActual = readableActual; + this.dslExpected = dslExpected; + this.dslActual = dslActual; this.messages = messages; } @@ -44,6 +48,8 @@ public static class Builder { private final boolean result; private String readableExpected = null; private String readableActual = null; + private String dslExpected = null; + private String dslActual = null; private final List messages = new ArrayList<>(); private Builder(boolean result) { @@ -59,6 +65,16 @@ public Builder withReadableActual(String readableActual) { this.readableActual = readableActual; return this; } + + public Builder withDslExpected(String dslExpected) { + this.dslExpected = dslExpected; + return this; + } + + public Builder withDslActual(String dslActual) { + this.dslActual = dslActual; + return this; + } public Builder withMessages(List messages) { this.messages.addAll(messages); @@ -71,7 +87,7 @@ public Builder withMessage(Message message) { } public EvaluationResult build() { - return new EvaluationResult(result, readableExpected, readableActual, messages); + return new EvaluationResult(result, readableExpected, readableActual, dslExpected, dslActual, messages); } } } diff --git a/tested/languages/java/templates/Values.java b/tested/languages/java/templates/Values.java index 330b1337..bfcb54b0 100644 --- a/tested/languages/java/templates/Values.java +++ b/tested/languages/java/templates/Values.java @@ -128,9 +128,13 @@ private static List internalEncode(Object value) { private static String encode(Object value) { var typeAndData = internalEncode(value); - return "{ \"data\": " + typeAndData.get(1) + "," + - " \"type\": \"" + typeAndData.get(0) + "\", " + - " \"diagnostic\": " + typeAndData.get(2) + "}"; + return """ + { + "data": %s, + "type": "%s", + "diagnostic": %s + } + """.formatted(typeAndData.get(1), typeAndData.get(0), typeAndData.get(2)); } public static void send(PrintWriter writer, Object value) { @@ -145,37 +149,57 @@ public static void sendException(PrintWriter writer, Throwable exception) { exception.printStackTrace(new PrintWriter(sw)); var trace = sw.toString(); var msg = exception.getMessage(); - var result = "{ \"message\": \"" + escape(msg == null ? "" : msg) + - "\", \"stacktrace\": \"" + escape(trace) + - "\", \"type\": \"" + exception.getClass().getSimpleName() + "\"}"; + String result = """ + { + "message": %s, + "stacktrace": %s, + "type": "%s" + } + """.formatted(asJson(msg), asJson(trace), exception.getClass().getSimpleName()); writer.write(result); } private static String convertMessage(EvaluationResult.Message message) { - String result = "{" + - "\"description\": \"" + message.description + "\"," + - "\"format\": \"" + message.format + "\""; - if (message.permission instanceof String) { - result += ", \"permission\": \"" + message.permission + "\""; - } - return result + "}"; + var description = asJson(message.description); + var format = asJson(message.format); + var permission = asJson(message.permission); + + return """ + { + "description": %s, + "format": %s, + "permission": %s + } + """.formatted(description, format, permission); } - - public static void evaluated(PrintWriter writer, - boolean result, String expected, String actual, Collection messages) { - List converted = messages.stream().map(Values::convertMessage).collect(Collectors.toList()); - String builder = "{" + - "\"result\": " + - result + - ", \"readable_expected\": \"" + expected + "\"" + - ", \"readable_actual\": \"" + actual + "\"" + - ", \"messages\": [" + - String.join(", ", converted) + - "]}"; - writer.print(builder); + + private static String asJson(String value) { + if (value == null) { + return "null"; + } else { + return "\"%s\"".formatted(escape(value)); + } } public static void sendEvaluated(PrintWriter writer, EvaluationResult r) { - evaluated(writer, r.result, r.readableExpected, r.readableActual, r.messages); + List converted = r.messages.stream().map(Values::convertMessage).collect(Collectors.toList()); + var readableExpected = asJson(r.readableExpected); + var readableActual = asJson(r.readableActual); + var dslExpected = asJson(r.dslExpected); + var dslActual = asJson(r.dslActual); + var messages = String.join(", ", converted); + + String result = """ + { + "result": %b, + "readable_expected": %s, + "readable_actual": %s, + "dsl_expected": %s, + "dsl_actual": %s, + "messages": [%s] + } + """.formatted(r.result, readableExpected, readableActual, dslExpected, dslActual, messages); + + writer.print(result); } } diff --git a/tested/languages/javascript/generators.py b/tested/languages/javascript/generators.py index 5e9a94be..f2432934 100644 --- a/tested/languages/javascript/generators.py +++ b/tested/languages/javascript/generators.py @@ -275,6 +275,6 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += f"values.sendValue(process.stdout.fd, {convert_value(value)});\n" - result += 'fs.writeSync(process.stdout.fd, "\\n");\n' + result += 'fs.writeSync(process.stdout.fd, "␞");\n' return result diff --git a/tested/languages/kotlin/generators.py b/tested/languages/kotlin/generators.py index b73d7a44..6b3b1e68 100644 --- a/tested/languages/kotlin/generators.py +++ b/tested/languages/kotlin/generators.py @@ -422,7 +422,7 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += " " * 4 + f"valuesSend(writer, {convert_value(value)})\n" - result += " " * 4 + 'writer.write("\\n")\n' + result += " " * 4 + 'writer.write("␞")\n' result += " " * 4 + "writer.close()\n" result += "}\n" diff --git a/tested/languages/kotlin/templates/EvaluationResult.kt b/tested/languages/kotlin/templates/EvaluationResult.kt index 0c745ce2..db0b2c50 100644 --- a/tested/languages/kotlin/templates/EvaluationResult.kt +++ b/tested/languages/kotlin/templates/EvaluationResult.kt @@ -2,6 +2,8 @@ class EvaluationResult private constructor( val result: Boolean, val readableExpected: String?, val readableActual: String?, + val dslExpected: String?, + val dslActual: String?, val messages: List ) { class Message(val description: String, @@ -11,7 +13,9 @@ class EvaluationResult private constructor( class Builder( private val result: Boolean, private var readableActual: String? = null, - private var readableExpected: String? = null + private var readableExpected: String? = null, + private var dslActual: String? = null, + private var dslExpected: String? = null ) { private val messages: MutableList = ArrayList() @@ -25,6 +29,16 @@ class EvaluationResult private constructor( return this } + fun withDslActual(dslActual: String?): Builder { + this.dslActual = dslActual + return this + } + + fun withDslExpected(dslExpected: String?): Builder { + this.dslExpected = dslExpected + return this + } + fun withMessages(messages: List): Builder { this.messages += messages return this @@ -37,7 +51,7 @@ class EvaluationResult private constructor( fun build(): EvaluationResult { return EvaluationResult(result, readableExpected, - readableActual, messages) + readableActual, dslExpected, dslActual, messages) } } } diff --git a/tested/languages/kotlin/templates/Values.kt b/tested/languages/kotlin/templates/Values.kt index ffdb4bd0..aef14142 100644 --- a/tested/languages/kotlin/templates/Values.kt +++ b/tested/languages/kotlin/templates/Values.kt @@ -4,15 +4,13 @@ import java.math.BigDecimal import java.math.BigInteger private fun convertMessage(message: EvaluationResult.Message): String { - val builder = StringBuilder(64).append('{') - .append("\"description\": \"").append(message.description) - .append("\", ") - .append("\"format\": \"").append(message.format).append("\"") - if (message.permission != null) { - builder.append(", \"permission\": \"").append(message.permission) - .append('\"') - } - return builder.append('}').toString() + return """ + { + "description": ${asJson(message.description)}, + "format": ${asJson(message.format)}, + "permission": ${asJson(message.permission)} + } + """.trimIndent() } private fun escape(str: String): String { @@ -142,22 +140,29 @@ private fun internalEncode(value: Any?): Array { return arrayOf(type, data, diagnostic) } -fun evaluated(writer: PrintWriter, result: Boolean, expected: String?, - actual: String?, messages: Iterable) { - val builder = StringBuilder(64).append('{') - .append("\"result\": ").append(result).append(", ") - .append("\"readable_expected\": \"").append(expected).append("\", ") - .append("\"readable_actual\": \"").append(actual).append("\", ") - .append("\"messages\": [").append(messages.joinToString(separator = ", ", - transform = { m -> convertMessage(m) })).append("]}") - writer.print(builder.toString()) +fun asJson(value: String?): String { + if (value == null) { + return "null"; + } else { + return "\"${escape(value)}\""; + } } fun valuesSend(writer: PrintWriter, value: Any?): Unit = writer.print(encode(value)) fun valuesSendEvaluated(writer: PrintWriter, result: EvaluationResult) { - evaluated(writer, result.result, result.readableExpected, result.readableActual, - result.messages) + val messages = result.messages.joinToString(separator = ", ", transform = { m -> convertMessage(m) }) + val result = """ + { + "result": ${result.result}, + "readable_expected": ${asJson(result.readableExpected)}, + "readable_actual": ${asJson(result.readableActual)}, + "dsl_expected": ${asJson(result.dslExpected)}, + "dsl_actual": ${asJson(result.dslActual)}, + "messages": [${messages}] + } + """.trimIndent() + writer.print(result) } fun valuesSendException(writer: PrintWriter, throwable: Throwable?) { @@ -166,9 +171,11 @@ fun valuesSendException(writer: PrintWriter, throwable: Throwable?) { } val strStackTraceWriter = StringWriter() throwable.printStackTrace(PrintWriter(strStackTraceWriter)) - writer.printf("{ \"message\": \"%s\", \"stacktrace\": \"%s\", \"type\": \"%s\"}", - escape(throwable.message ?: ""), - escape(strStackTraceWriter.toString()), - throwable::class.simpleName - ) + writer.printf(""" + { + "message": "${escape(throwable.message ?: "")}", + "stacktrace": "${escape(strStackTraceWriter.toString())}", + "type": "${throwable::class.simpleName}" + } + """.trimIndent()) } diff --git a/tested/languages/python/generators.py b/tested/languages/python/generators.py index 2961275d..82965aaa 100644 --- a/tested/languages/python/generators.py +++ b/tested/languages/python/generators.py @@ -264,5 +264,5 @@ def convert_encoder(values: list[Value]) -> str: for value in values: result += f"values.send_value(sys.stdout, {convert_value(value)})\n" - result += "print()\n" + result += "print('␞')\n" return result diff --git a/tested/languages/python/templates/evaluation_utils.py b/tested/languages/python/templates/evaluation_utils.py index 74fc7f25..3f6d5601 100644 --- a/tested/languages/python/templates/evaluation_utils.py +++ b/tested/languages/python/templates/evaluation_utils.py @@ -15,3 +15,5 @@ class EvaluationResult: readable_expected: Optional[str] = None readable_actual: Optional[str] = None messages: List[Message] = field(default_factory=list) + dsl_expected: Optional[str] = None + dsl_actual: Optional[str] = None diff --git a/tested/oracles/common.py b/tested/oracles/common.py index ef3712c6..9d06dd58 100644 --- a/tested/oracles/common.py +++ b/tested/oracles/common.py @@ -31,8 +31,10 @@ def evaluate_text(configs, channel, actual): from tested.configs import Bundle from tested.dodona import Message, Status, StatusMessage +from tested.dsl import parse_string +from tested.languages.generation import generate_statement from tested.languages.utils import convert_stacktrace_to_clickable_feedback -from tested.serialisation import EvalResult +from tested.parsing import fallback_field, get_converter from tested.testsuite import ExceptionOutputChannel, NormalOutputChannel, OutputChannel @@ -51,6 +53,74 @@ class OracleResult: ) +@fallback_field( + get_converter(), + { + "readableExpected": "readable_expected", + "readableActual": "readable_actual", + "dslExpected": "dsl_expected", + "dslActual": "dsl_actual", + }, +) +@define +class BooleanEvalResult: + """ + Allows a boolean result. + + Note: this class is used directly in the Python oracle, so keep it backwards + compatible (also positional arguments) or make a new class for the oracle. + """ + + result: bool | Status + readable_expected: str | None = None + readable_actual: str | None = None + messages: list[Message] = field(factory=list) + dsl_expected: str | None = None + dsl_actual: str | None = None + + def to_oracle_result( + self, bundle: Bundle, channel: NormalOutputChannel + ) -> OracleResult: + if isinstance(self.result, Status): + status = self.result + else: + status = Status.CORRECT if self.result else Status.WRONG + + if self.readable_expected: + readable_expected = self.readable_expected + elif self.dsl_expected: + parsed_statement = parse_string(self.dsl_expected, True) + readable_expected = generate_statement(bundle, parsed_statement) + else: + readable_expected = "" + if self.readable_actual: + readable_actual = self.readable_actual + elif self.dsl_actual: + parsed_statement = parse_string(self.dsl_actual, True) + readable_actual = generate_statement(bundle, parsed_statement) + else: + readable_actual = "" + messages = self.messages + + if isinstance(channel, ExceptionOutputChannel): + readable_expected = bundle.language.cleanup_stacktrace(readable_expected) + message = convert_stacktrace_to_clickable_feedback( + bundle.language, readable_actual + ) + if message: + messages.append(message) + + if status == Status.CORRECT: + readable_actual = "" + + return OracleResult( + result=StatusMessage(enum=status), + readable_expected=readable_expected, + readable_actual=readable_actual, + messages=messages, + ) + + @define class OracleConfig: bundle: Bundle @@ -86,24 +156,3 @@ def try_outputs( if possible is not None: return possible, msg return actual, None - - -def cleanup_specific_programmed( - config: OracleConfig, channel: NormalOutputChannel, actual: EvalResult -) -> EvalResult: - if isinstance(channel, ExceptionOutputChannel): - lang_config = config.bundle.language - actual.readable_expected = lang_config.cleanup_stacktrace( - actual.readable_expected or "" - ) - message = convert_stacktrace_to_clickable_feedback( - lang_config, actual.readable_actual - ) - - if message: - actual.messages.append(message) - - if actual.result == Status.CORRECT: - actual.readable_actual = "" - - return actual diff --git a/tested/oracles/programmed.py b/tested/oracles/programmed.py index 7af28b5b..9fba6c5c 100644 --- a/tested/oracles/programmed.py +++ b/tested/oracles/programmed.py @@ -5,14 +5,9 @@ from tested.internationalization import get_i18n_string from tested.judge.programmed import evaluate_programmed from tested.judge.utils import BaseExecutionResult -from tested.oracles.common import ( - OracleConfig, - OracleResult, - cleanup_specific_programmed, -) +from tested.oracles.common import BooleanEvalResult, OracleConfig, OracleResult from tested.oracles.value import get_values from tested.parsing import get_converter -from tested.serialisation import BooleanEvalResult, EvalResult from tested.testsuite import CustomCheckOracle, OracleOutputChannel, OutputChannel _logger = logging.getLogger(__name__) @@ -37,7 +32,7 @@ def evaluate( # This is slightly tricky, since the actual value must also be converted # to a value, and we are not yet sure what the actual value is exactly result = get_values(config.bundle, channel, actual_str or "") - # TODO: why is this? + # If an error occurred, we get a result directly. if isinstance(result, OracleResult): return result else: @@ -61,6 +56,7 @@ def evaluate( ) if isinstance(result, BaseExecutionResult): + _logger.error(result.stderr) if result.timeout: return OracleResult( result=StatusMessage(enum=Status.TIME_LIMIT_EXCEEDED), @@ -86,9 +82,7 @@ def evaluate( messages=[stdout, stderr, DEFAULT_STUDENT], ) try: - evaluation_result = ( - get_converter().loads(result.stdout, BooleanEvalResult).as_eval_result() - ) + evaluation_result = get_converter().loads(result.stdout, BooleanEvalResult) except Exception as e: _logger.exception(e) messages: list[Message] = [ @@ -129,35 +123,7 @@ def evaluate( messages=messages, ) else: - assert isinstance(result, EvalResult) + assert isinstance(result, BooleanEvalResult) evaluation_result = result - if evaluation_result.readable_expected: - readable_expected = evaluation_result.readable_expected - if evaluation_result.readable_actual: - readable_actual = evaluation_result.readable_actual - - if isinstance(evaluation_result.result, Status): - result_status = StatusMessage(enum=evaluation_result.result) - else: - assert isinstance(evaluation_result.result, bool) - result_status = StatusMessage( - enum=Status.CORRECT if evaluation_result.result else Status.WRONG - ) - cleaned = cleanup_specific_programmed( - config, - channel, - EvalResult( - result=result_status.enum, - readable_expected=readable_expected, # type: ignore - readable_actual=readable_actual, # type: ignore - messages=evaluation_result.messages, - ), - ) - - return OracleResult( - result=result_status, - readable_expected=cleaned.readable_expected or "", - readable_actual=cleaned.readable_actual or "", - messages=cleaned.messages, - ) + return evaluation_result.to_oracle_result(config.bundle, channel) diff --git a/tested/oracles/specific.py b/tested/oracles/specific.py index 78f47709..4cd456c5 100644 --- a/tested/oracles/specific.py +++ b/tested/oracles/specific.py @@ -6,13 +6,8 @@ from tested.dodona import ExtendedMessage, Permission, Status, StatusMessage from tested.internationalization import get_i18n_string -from tested.oracles.common import ( - OracleConfig, - OracleResult, - cleanup_specific_programmed, -) +from tested.oracles.common import BooleanEvalResult, OracleConfig, OracleResult from tested.parsing import get_converter -from tested.serialisation import BooleanEvalResult from tested.testsuite import LanguageSpecificOracle, OracleOutputChannel, OutputChannel _logger = logging.getLogger(__name__) @@ -40,7 +35,7 @@ def evaluate( ) try: - actual = get_converter().loads(actual_str, BooleanEvalResult).as_eval_result() + actual = get_converter().loads(actual_str, BooleanEvalResult) except Exception as e: _logger.exception(e) staff_message = ExtendedMessage( @@ -61,11 +56,4 @@ def evaluate( messages=[staff_message, student_message], ) - actual = cleanup_specific_programmed(config, channel, actual) - - return OracleResult( - result=StatusMessage(enum=actual.result), - readable_expected=actual.readable_expected or "", - readable_actual=actual.readable_actual or "", - messages=actual.messages, - ) + return actual.to_oracle_result(config.bundle, channel) diff --git a/tested/serialisation.py b/tested/serialisation.py index 6f0a1020..26fc0ccb 100644 --- a/tested/serialisation.py +++ b/tested/serialisation.py @@ -50,9 +50,8 @@ StringTypes, resolve_to_basic, ) -from tested.dodona import Message, Status from tested.features import Construct, FeatureSet, WithFeatures, combine_features -from tested.parsing import fallback_field, get_converter, parse_json_value +from tested.parsing import parse_json_value from tested.utils import flatten, sorted_no_duplicates logger = logging.getLogger(__name__) @@ -688,46 +687,6 @@ def to_python_comparable(value: Value | None) -> Any: raise AssertionError(f"Unknown value type: {value}") -@fallback_field( - get_converter(), - {"readableExpected": "readable_expected", "readableActual": "readable_actual"}, -) -@define -class EvalResult: - result: Status - readable_expected: str | None = None - readable_actual: str | None = None - messages: list[Message] = field(factory=list) - - -@fallback_field( - get_converter(), - {"readableExpected": "readable_expected", "readableActual": "readable_actual"}, -) -@define -class BooleanEvalResult: - """ - Allows a boolean result. - """ - - result: bool | Status - readable_expected: str | None = None - readable_actual: str | None = None - messages: list[Message] = field(factory=list) - - def as_eval_result(self) -> EvalResult: - if isinstance(self.result, Status): - status = self.result - else: - status = Status.CORRECT if self.result else Status.WRONG - return EvalResult( - result=status, - readable_expected=self.readable_expected, - readable_actual=self.readable_actual, - messages=self.messages, - ) - - @define class ExceptionValue: """An exception that was thrown while executing the user context.""" diff --git a/tests/exercises/echo-function/evaluation/Evaluator.cs b/tests/exercises/echo-function/evaluation/Evaluator.cs index e16a280a..d56695a2 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.cs +++ b/tests/exercises/echo-function/evaluation/Evaluator.cs @@ -12,4 +12,9 @@ public static EvaluationResult EvaluateValue(Object expected, Object actual, ILi var messages = new List() {new Tested.Message("Hallo")}; return new EvaluationResult(expected == actual, expected.ToString(), actual != null ? actual.ToString() : "", messages); } + + public static EvaluationResult EvaluateValueDsl(Object expected, Object actual, IList arguments) { + var messages = new List() {new Tested.Message("Hallo")}; + return new EvaluationResult(expected == actual, null, null, messages, "{5, 5}", "{4, 4}"); + } } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.java b/tests/exercises/echo-function/evaluation/Evaluator.java index b7551587..ebae008a 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.java +++ b/tests/exercises/echo-function/evaluation/Evaluator.java @@ -18,4 +18,12 @@ public static EvaluationResult evaluateValue(Object expected, Object actual, Lis .withMessage(new EvaluationResult.Message("Hallo")) .build(); } + + public static EvaluationResult evaluateValueDsl(Object expected, Object actual, List arguments) { + return EvaluationResult.builder(expected.equals(actual)) + .withDslExpected("{5, 5}") + .withDslActual("{4, 4}") + .withMessage(new EvaluationResult.Message("Hallo")) + .build(); + } } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.kt b/tests/exercises/echo-function/evaluation/Evaluator.kt index 31cbe7dd..777a8721 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.kt +++ b/tests/exercises/echo-function/evaluation/Evaluator.kt @@ -17,5 +17,14 @@ class Evaluator { .withMessage(EvaluationResult.Message("Hallo")) .build() } + + @JvmStatic + fun evaluateValueDsl(expected: Any, actual: Any?, arguments: List?): EvaluationResult { + return EvaluationResult.Builder(result = expected == actual, + dslExpected = "{5, 5}", + dslActual = "{4, 4}") + .withMessage(EvaluationResult.Message("Hallo")) + .build() + } } -} \ No newline at end of file +} diff --git a/tests/exercises/echo-function/evaluation/evaluator.js b/tests/exercises/echo-function/evaluation/evaluator.js index 632c4e3b..2a5ffd93 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.js +++ b/tests/exercises/echo-function/evaluation/evaluator.js @@ -11,11 +11,21 @@ function evaluate(actual) { function evaluateValue(expected, actual, args) { return { "result": expected === actual, - "expected": expected, - "actual": actual, + "readable_expected": expected, + "readable_actual": actual, + "messages": [{"description": "Hallo", "format": "text"}] + } +} + +function evaluateValueDsl(expected, actual, args) { + return { + "result": expected === actual, + "dsl_expected": "{5, 5}", + "dsl_actual": "{4, 4}", "messages": [{"description": "Hallo", "format": "text"}] } } exports.evaluate = evaluate; -exports.evaluateValue = evaluateValue; \ No newline at end of file +exports.evaluateValue = evaluateValue; +exports.evaluateValueDsl = evaluateValueDsl; diff --git a/tests/exercises/echo-function/evaluation/evaluator.py b/tests/exercises/echo-function/evaluation/evaluator.py index 5cef55d6..6f177c1f 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.py +++ b/tests/exercises/echo-function/evaluation/evaluator.py @@ -8,3 +8,12 @@ def evaluate(actual): def evaluate_value(expected, actual, args): return EvaluationResult(expected == actual, expected, actual, [Message("Hallo")]) + + +def evaluate_value_dsl(expected, actual, args): + return EvaluationResult( + result=expected == actual, + messages=[Message("Hallo")], + dsl_expected="{5, 5}", + dsl_actual="{4, 4}" + ) diff --git a/tests/exercises/echo-function/evaluation/programmed-dsl-no-haskell.tson b/tests/exercises/echo-function/evaluation/programmed-dsl-no-haskell.tson new file mode 100644 index 00000000..65a6703f --- /dev/null +++ b/tests/exercises/echo-function/evaluation/programmed-dsl-no-haskell.tson @@ -0,0 +1,157 @@ +{ + "tabs": [ + { + "name": "Tab", + "runs": [ + { + "contexts": [ + { + "testcases": [ + { + "input": { + "type": "function", + "name": "echo", + "arguments": [ + { + "type": "text", + "data": "input-2" + } + ] + }, + "output": { + "result": { + "value": { + "type": "text", + "data": "input-2" + }, + "evaluator": { + "type": "programmed", + "function": { + "file": "Evaluator.java", + "name": "evaluateValueDsl" + }, + "language": "java" + } + } + } + }, + { + "input": { + "type": "function", + "name": "echo", + "arguments": [ + { + "type": "text", + "data": "input-3" + } + ] + }, + "output": { + "result": { + "value": { + "type": "text", + "data": "input-3" + }, + "evaluator": { + "type": "programmed", + "function": { + "file": "evaluator.py", + "name": "evaluate_value_dsl" + }, + "language": "python" + } + } + } + }, + { + "input": { + "type": "function", + "name": "echo", + "arguments": [ + { + "type": "text", + "data": "input-4" + } + ] + }, + "output": { + "result": { + "value": { + "type": "text", + "data": "input-4" + }, + "evaluator": { + "type": "programmed", + "function": { + "file": "evaluator.js", + "name": "evaluateValueDsl" + }, + "language": "javascript" + } + } + } + }, + { + "input": { + "type": "function", + "name": "echo", + "arguments": [ + { + "type": "text", + "data": "input-5" + } + ] + }, + "output": { + "result": { + "value": { + "type": "text", + "data": "input-5" + }, + "evaluator": { + "type": "programmed", + "function": { + "file": "Evaluator.kt", + "name": "evaluateValueDsl" + }, + "language": "kotlin" + } + } + } + }, + { + "input": { + "type": "function", + "name": "echo", + "arguments": [ + { + "type": "text", + "data": "input-89" + } + ] + }, + "output": { + "result": { + "value": { + "type": "text", + "data": "input-89" + }, + "evaluator": { + "type": "programmed", + "function": { + "file": "Evaluator.cs", + "name": "EvaluateValueDsl" + }, + "language": "csharp" + } + } + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/tests/test_functionality.py b/tests/test_functionality.py index ba16466c..38f265cd 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -293,6 +293,25 @@ def test_programmed_evaluation(language: str, tmp_path: Path, pytestconfig): assert len(updates.find_all("append-message")) == 5 +def test_programmed_evaluation_with_dsl(tmp_path: Path, pytestconfig): + conf = configuration( + pytestconfig, + "echo-function", + "javascript", + tmp_path, + "programmed-dsl-no-haskell.tson", + "correct", + ) + result = execute_config(conf) + updates = assert_valid_output(result, pytestconfig) + assert updates.find_status_enum() == ["correct"] * 5 + assert len(updates.find_all("append-message")) == 5 + all_expected = [x["expected"] for x in updates.find_all("start-test")] + assert all_expected == ["new Set([5, 5])"] * 5 + all_actual = [x["generated"] for x in updates.find_all("close-test")] + assert all_actual == ["new Set([4, 4])"] * 5 + + @pytest.mark.parametrize( "lang", [ diff --git a/tests/test_serialisation.py b/tests/test_serialisation.py index e740ac7d..8eee9b1c 100644 --- a/tests/test_serialisation.py +++ b/tests/test_serialisation.py @@ -194,7 +194,7 @@ def run_encoder(bundle: Bundle, values: list[Value]) -> list[str]: # Run the code. r = execute_file(bundle, executable.name, dest, None) print(r.stderr) - return r.stdout.splitlines(keepends=False) + return r.stdout.split(sep="␞")[:-1] def assert_serialisation(bundle: Bundle, expected: Value):