From 762f5f8536c19949f445fd1041753cee473f8092 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Wed, 23 Aug 2023 17:20:57 +0200 Subject: [PATCH 1/2] Unify return values in DSL --- tested/dsl/schema.json | 8 ++- tested/dsl/schema_draft7.json | 8 ++- tested/dsl/translate_parser.py | 50 +++++++++++++------ .../expected_return_and_got_some.yaml | 2 +- .../expected_return_but_got_none.yaml | 2 +- .../evaluation/one-language-literals.yaml | 2 +- .../echo-function/evaluation/one-nested.yaml | 2 +- tests/exercises/global/evaluation/plan.yaml | 2 +- .../objects/evaluation/missing_key_types.yaml | 2 +- .../exercises/objects/evaluation/no-test.yaml | 4 +- tests/test_dsl_yaml.py | 33 ++++-------- 11 files changed, 60 insertions(+), 55 deletions(-) diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 13f59518..9b38deee 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -269,10 +269,7 @@ } }, "return" : { - "description" : "Expected return value" - }, - "return_raw" : { - "description" : "Value string to parse to the expected return value", + "description" : "Expected return value.", "$ref" : "#/$defs/advancedValueOutputChannel" }, "stderr" : { @@ -396,7 +393,8 @@ ] }, "advancedValueOutputChannel" : { - "oneOf" : [ + "anyOf" : [ + {}, { "type" : "string", "description" : "A 'Python' value to parse and use as the expected type." diff --git a/tested/dsl/schema_draft7.json b/tested/dsl/schema_draft7.json index 506e8c2b..16762de6 100644 --- a/tested/dsl/schema_draft7.json +++ b/tested/dsl/schema_draft7.json @@ -265,10 +265,7 @@ } }, "return" : { - "description" : "Expected return value" - }, - "return_raw" : { - "description" : "Value string to parse to the expected return value", + "description" : "Expected return value", "$ref" : "#/definitions/advancedValueOutputChannel" }, "stderr" : { @@ -391,7 +388,8 @@ ] }, "advancedValueOutputChannel" : { - "oneOf" : [ + "anyOf" : [ + {}, { "type" : "string", "description" : "A 'Python' value to parse and use as the expected type." diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 6a74df76..6b019ef8 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -86,18 +86,32 @@ class TestedType: type: str | AllTypes -def custom_type_constructors(loader: yaml.Loader, node: yaml.Node): - tested_tag = node.tag[1:] +@define +class YamlValue: + value: Any + + +def _parse_yaml_value(loader: yaml.Loader, node: yaml.Node) -> Any: if isinstance(node, yaml.MappingNode): - base_result = loader.construct_mapping(node) + result = loader.construct_mapping(node) elif isinstance(node, yaml.SequenceNode): - base_result = loader.construct_sequence(node) + result = loader.construct_sequence(node) else: assert isinstance(node, yaml.ScalarNode) - base_result = loader.construct_scalar(node) + result = loader.construct_scalar(node) + return result + + +def _custom_type_constructors(loader: yaml.Loader, node: yaml.Node) -> TestedType: + tested_tag = node.tag[1:] + base_result = _parse_yaml_value(loader, node) return TestedType(type=tested_tag, value=base_result) +def _yaml_value_constructor(loader: yaml.Loader, node: yaml.Node) -> YamlValue: + return YamlValue(value=_parse_yaml_value(loader, node)) + + def _parse_yaml(yaml_stream: Union[str, TextIO]) -> YamlObject: """ Parse a string or stream to YAML. @@ -105,7 +119,8 @@ def _parse_yaml(yaml_stream: Union[str, TextIO]) -> YamlObject: loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.CSafeLoader) for types in get_args(AllTypes): for actual_type in types: - yaml.add_constructor("!" + actual_type, custom_type_constructors, loader) + yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader) + yaml.add_constructor("!v", _yaml_value_constructor, loader) return yaml.load(yaml_stream, loader) @@ -336,11 +351,23 @@ def _convert_text_output_channel( def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel: - if isinstance(stream, str): + if isinstance(stream, YamlValue): + # A normal yaml type tagged explicitly. + value = _convert_value(stream.value) + assert isinstance(value, Value) + return ValueOutputChannel(value=value) + if isinstance(stream, (int, float, bool, TestedType, list, set)): + # Simple values where no confusion is possible. + value = _convert_value(stream) + assert isinstance(value, Value) + return ValueOutputChannel(value=value) + elif isinstance(stream, str): + # A normal YAML string is considered a "Python" string. value = parse_string(stream, is_return=True) assert isinstance(value, Value) return ValueOutputChannel(value=value) else: + # We have an object, which means we have an output channel. assert isinstance(stream, dict) assert isinstance(stream["value"], str) value = parse_string(stream["value"], is_return=True) @@ -362,10 +389,8 @@ def _validate_testcase_combinations(testcase: YamlDict): raise ValueError("A main call cannot contain an expression or a statement.") if "statement" in testcase and "expression" in testcase: raise ValueError("A statement and expression as input are mutually exclusive.") - if "statement" in testcase and ("return" in testcase or "return_raw" in testcase): + if "statement" in testcase and "return" in testcase: raise ValueError("A statement cannot have an expected return value.") - if "return" in testcase and "return_raw" in testcase: - raise ValueError("The outputs return and return_raw are mutually exclusive.") def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: @@ -373,7 +398,7 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: # This is backwards compatability to some extend. # TODO: remove this at some point. - if "statement" in testcase and ("return" in testcase or "return_raw" in testcase): + if "statement" in testcase and "return" in testcase: testcase["expression"] = testcase.pop("statement") _validate_testcase_combinations(testcase) @@ -430,9 +455,6 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: if (exit_code := testcase.get("exit_code")) is not None: output.exit_code = ExitCodeOutputChannel(value=cast(int, exit_code)) if (result := testcase.get("return")) is not None: - assert not return_channel - output.result = ValueOutputChannel(value=_convert_value(result)) - if (result := testcase.get("return_raw")) is not None: assert not return_channel output.result = _convert_advanced_value_output_channel(result) diff --git a/tests/exercises/echo-function/evaluation/expected_return_and_got_some.yaml b/tests/exercises/echo-function/evaluation/expected_return_and_got_some.yaml index f53ba87f..fa84fa48 100644 --- a/tests/exercises/echo-function/evaluation/expected_return_and_got_some.yaml +++ b/tests/exercises/echo-function/evaluation/expected_return_and_got_some.yaml @@ -1,4 +1,4 @@ - tab: "My tab" testcases: - expression: 'echo("input")' - return: "input" + return: !v "input" diff --git a/tests/exercises/echo-function/evaluation/expected_return_but_got_none.yaml b/tests/exercises/echo-function/evaluation/expected_return_but_got_none.yaml index d6cde337..7c687441 100644 --- a/tests/exercises/echo-function/evaluation/expected_return_but_got_none.yaml +++ b/tests/exercises/echo-function/evaluation/expected_return_but_got_none.yaml @@ -1,4 +1,4 @@ - tab: "My tab" testcases: - expression: 'no_echo("input")' - return: "input" + return: !v "input" diff --git a/tests/exercises/echo-function/evaluation/one-language-literals.yaml b/tests/exercises/echo-function/evaluation/one-language-literals.yaml index 20718509..7b5acfdb 100644 --- a/tests/exercises/echo-function/evaluation/one-language-literals.yaml +++ b/tests/exercises/echo-function/evaluation/one-language-literals.yaml @@ -9,4 +9,4 @@ kotlin: "toString(1+1)" python: "submission.to_string(1+1)" csharp: "Submission.toString(1+1)" - return: "2" + return: !v "2" diff --git a/tests/exercises/echo-function/evaluation/one-nested.yaml b/tests/exercises/echo-function/evaluation/one-nested.yaml index 126fe164..3ce7336b 100644 --- a/tests/exercises/echo-function/evaluation/one-nested.yaml +++ b/tests/exercises/echo-function/evaluation/one-nested.yaml @@ -1,4 +1,4 @@ - tab: "My tab" testcases: - expression: 'echo(echo("input"))' - return: "input" + return: !v "input" diff --git a/tests/exercises/global/evaluation/plan.yaml b/tests/exercises/global/evaluation/plan.yaml index e8f98e7c..f33e9441 100644 --- a/tests/exercises/global/evaluation/plan.yaml +++ b/tests/exercises/global/evaluation/plan.yaml @@ -1,4 +1,4 @@ - tab: "Global variable" testcases: - expression: "GLOBAL_VAR" - return: "GLOBAL" + return: !v "GLOBAL" diff --git a/tests/exercises/objects/evaluation/missing_key_types.yaml b/tests/exercises/objects/evaluation/missing_key_types.yaml index ccd82228..030651a3 100644 --- a/tests/exercises/objects/evaluation/missing_key_types.yaml +++ b/tests/exercises/objects/evaluation/missing_key_types.yaml @@ -1,4 +1,4 @@ - tab: "Feedback" testcases: - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' - return_raw: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' + return: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' diff --git a/tests/exercises/objects/evaluation/no-test.yaml b/tests/exercises/objects/evaluation/no-test.yaml index 585abb16..c92b18eb 100644 --- a/tests/exercises/objects/evaluation/no-test.yaml +++ b/tests/exercises/objects/evaluation/no-test.yaml @@ -1,11 +1,11 @@ - tab: "Feedback" testcases: - statement: '{["a", "b"], ["c"]}' - return_raw: '{["a", "b"], ["a"]}' + return: '{["a", "b"], ["a"]}' - statement: 'x = {{"a"}: [int16(1)], {"b"}: [int16(0)]}' - statement: 'x = {{"a"}: [int32(1)], {"b"}: "a"}' - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' - return_raw: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' + return: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' files: - name: "a.txt" url: "a.txt" diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 0fcbda01..bab7f013 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -255,7 +255,7 @@ def test_statements(): - statement: 'safe: Safe = Safe("Ignore whitespace")' stdout: "New safe" - expression: 'safe.content()' - return: "Ignore whitespace" + return: !v "Ignore whitespace" - testcases: - statement: 'safe: Safe = Safe(uint8(5))' stdout: @@ -263,7 +263,7 @@ def test_statements(): config: ignoreWhitespace: false - expression: 'safe.content()' - return_raw: 'uint8(5)' + return: 'uint8(5)' """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -378,25 +378,12 @@ def test_invalid_yaml(): stderr: [] testcases: - statement: 'data = () ()' - return_raw: '() {}' + return: '() {}' """ with pytest.raises(Exception): translate_to_test_suite(yaml_str) -def test_invalid_mutual_exclusive_return_yaml(): - yaml_str = """ -- tab: "Tab" - contexts: - - testcases: - - statement: "5" - return: 5 - return_raw: "5" - """ - with pytest.raises(ValueError): - translate_to_test_suite(yaml_str) - - def test_invalid_context_as_testcase(): yaml_str = """ - tab: "Tab" @@ -414,7 +401,7 @@ def test_statement_with_yaml_dict(): - tab: "Feedback" testcases: - expression: "get_dict()" - return: + return: !v alpha: 5 beta: 6 """ @@ -466,7 +453,7 @@ def test_expression_raw_return(): contexts: - testcases: - expression: 'test()' - return_raw: '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]' + return: '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]' """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -498,7 +485,7 @@ def test_empty_constructor(function_name, result): contexts: - testcases: - expression: 'test()' - return_raw: '{function_name}()' + return: '{function_name}()' """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -528,7 +515,7 @@ def test_empty_constructor_with_param(function_name, result): contexts: - testcases: - expression: 'test()' - return_raw: '{function_name}([])' + return: '{function_name}([])' """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -639,7 +626,7 @@ def test_value_built_in_checks_implied(): contexts: - testcases: - expression: 'test()' - return_raw: + return: value: "'hallo'" """ json_str = translate_to_test_suite(yaml_str) @@ -664,7 +651,7 @@ def test_value_built_in_checks_explicit(): contexts: - testcases: - expression: 'test()' - return_raw: + return: value: "'hallo'" oracle: "builtin" """ @@ -690,7 +677,7 @@ def test_value_custom_checks_correct(): contexts: - testcases: - expression: 'test()' - return_raw: + return: value: "'hallo'" oracle: "custom_check" language: "python" From 80dec7a09cd6744aa90fd55b0f1d3cfde61316c3 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Fri, 25 Aug 2023 13:31:32 +0200 Subject: [PATCH 2/2] Also support !value --- tested/dsl/translate_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 6b019ef8..e244a356 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -121,6 +121,7 @@ def _parse_yaml(yaml_stream: Union[str, TextIO]) -> YamlObject: for actual_type in types: yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader) yaml.add_constructor("!v", _yaml_value_constructor, loader) + yaml.add_constructor("!value", _yaml_value_constructor, loader) return yaml.load(yaml_stream, loader)