Skip to content

Commit

Permalink
Don't show empty and correct return channels
Browse files Browse the repository at this point in the history
  • Loading branch information
niknetniko committed Sep 20, 2023
1 parent 1cdafd3 commit b85b4cf
Showing 1 changed file with 16 additions and 3 deletions.
19 changes: 16 additions & 3 deletions tested/judge/evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,10 @@
get_readable_input,
)
from tested.oracles import get_oracle
from tested.oracles.common import OracleResult
from tested.testsuite import (
Context,
EmptyChannel,
ExceptionOutput,
ExceptionOutputChannel,
ExitCodeOutputChannel,
Expand Down Expand Up @@ -113,7 +115,7 @@ def _evaluate_channel(

# Decide if we should show this channel or not.
is_correct = status.enum == Status.CORRECT
should_report_case = should_show(output, channel)
should_report_case = should_show(output, channel, evaluation_result)

if not should_report_case and is_correct:
# We do report that a test is correct, to set the status.
Expand Down Expand Up @@ -363,14 +365,17 @@ def _link_files_message(link_files: Collection[FileUrl]) -> AppendMessage:
return AppendMessage(message=message)


def should_show(test: OutputChannel, channel: Channel) -> bool:
def should_show(
test: OutputChannel, channel: Channel, result: OracleResult | None = None
) -> bool:
"""
Determine if the channel should be shown, without accounting for the actual
value. This function answers the question: "Assuming the actual value is
correct, should we show this output channel?".
:param test: The output for the channel from the test suite.
:param channel: The channel.
:param result: The result of the evaluation.
:return: True if the channel should be shown, false otherwise.
"""
Expand All @@ -390,7 +395,15 @@ def should_show(test: OutputChannel, channel: Channel) -> bool:
elif channel == Channel.RETURN:
assert isinstance(test, ValueOutput)
# We don't show the channel if we ignore it.
return not isinstance(test, IgnoredChannel)
if isinstance(test, IgnoredChannel):
return False
if (
isinstance(test, EmptyChannel)
and result
and result.result.enum == Status.CORRECT
):
return False
return True
elif channel == Channel.EXCEPTION:
assert isinstance(test, ExceptionOutput)
return not isinstance(test, SpecialOutputChannel)
Expand Down

0 comments on commit b85b4cf

Please sign in to comment.