From f34cd1f8f41c1f16299dd793dc99c7c10a3acbab Mon Sep 17 00:00:00 2001 From: Nullpointer Date: Thu, 5 Sep 2024 15:30:24 +0200 Subject: [PATCH] append --- addons/gdUnit4/bin/GdUnitCmdTool.gd | 54 ++++++++----------- addons/gdUnit4/src/core/event/GdUnitEvent.gd | 3 +- .../core/execution/GdUnitExecutionContext.gd | 17 ++++-- .../stages/GdUnitTestCaseExecutionStage.gd | 2 +- addons/gdUnit4/src/report/GdUnitHtmlReport.gd | 26 ++++++--- .../gdUnit4/src/report/GdUnitReportSummary.gd | 3 +- .../src/report/GdUnitTestCaseReport.gd | 16 +++--- .../src/report/GdUnitTestSuiteReport.gd | 27 +++++++--- addons/gdUnit4/src/report/JUnitXmlReport.gd | 42 +++++++-------- .../asserts/GdUnitGodotErrorWithAssertTest.gd | 4 +- .../test/ui/parts/InspectorProgressBarTest.gd | 6 +-- 11 files changed, 109 insertions(+), 91 deletions(-) diff --git a/addons/gdUnit4/bin/GdUnitCmdTool.gd b/addons/gdUnit4/bin/GdUnitCmdTool.gd index c83beb1d..628cffa3 100644 --- a/addons/gdUnit4/bin/GdUnitCmdTool.gd +++ b/addons/gdUnit4/bin/GdUnitCmdTool.gd @@ -488,29 +488,14 @@ class CLIRunner: event.reports() ) GdUnitEvent.TESTCASE_BEFORE: - _report.add_testcase_report( - event.resource_path(), - GdUnitTestCaseReport.new( - event.resource_path(), - event.suite_name(), - event.test_name() - ) - ) + _report.add_testcase(event.resource_path(), event.suite_name(), event.test_name()) + GdUnitEvent.TESTCASE_AFTER: - var test_report := GdUnitTestCaseReport.new( - event.resource_path(), - event.suite_name(), - event.test_name(), - event.is_error(), - event.is_failed(), - event.failed_count(), - event.orphan_nodes(), - event.is_skipped(), - event.is_flaky(), - event.reports(), - event.elapsed_time() - ) - _report.update_testcase_report(event.resource_path(), test_report) + _report.add_testcase_reports(event.resource_path(), event.test_name(), event.reports()) + + GdUnitEvent.TESTCASE_STATISTICS: + _report.update_testsuite_counters(event.resource_path(), event.is_error(), event.failed_count(), event.orphan_nodes(),\ + event.is_skipped(), event.is_flaky(), event.elapsed_time()) print_status(event) @@ -564,11 +549,12 @@ class CLIRunner: _print_failure_report(event.reports()) _print_status(event) _console.prints_color( - "Statistics: | %d tests cases | %d error | %d failed | %d skipped | %d orphans |\n" + "Statistics: | %d tests cases | %d error | %d failed | %d flaky | %d skipped | %d orphans |\n" % [ _report.test_count(), _report.error_count(), _report.failure_count(), + _report.flaky_count(), _report.skipped_count(), _report.orphan_count() ], @@ -595,20 +581,22 @@ class CLIRunner: func _print_status(event: GdUnitEvent) -> void: - if event.is_skipped(): + if event.is_flaky() and event.is_success(): + var retries :int = event.statistic(GdUnitEvent.RETRY_COUNT) + _console.print_color("FLAKY (%d retries)" % retries, Color.GREEN_YELLOW, CmdConsole.BOLD | CmdConsole.ITALIC) + elif event.is_success(): + _console.print_color("PASSED", Color.FOREST_GREEN, CmdConsole.BOLD) + elif event.is_skipped(): _console.print_color("SKIPPED", Color.GOLDENROD, CmdConsole.BOLD | CmdConsole.ITALIC) elif event.is_failed() or event.is_error(): - _console.print_color("FAILED", Color.FIREBRICK, CmdConsole.BOLD) - elif event.orphan_nodes() > 0: - _console.print_color("PASSED", Color.GOLDENROD, CmdConsole.BOLD | CmdConsole.UNDERLINE) - elif event.is_flaky(): var retries :int = event.statistic(GdUnitEvent.RETRY_COUNT) - if retries > 0: - _console.print_color("FLAKY (%d retries)" % retries, Color.GREEN_YELLOW, CmdConsole.BOLD | CmdConsole.ITALIC) + if retries > 1: + _console.print_color("FAILED (retry %d)" % retries, Color.FIREBRICK, CmdConsole.BOLD) else: - _console.print_color("FLAKY", Color.GREEN_YELLOW, CmdConsole.BOLD | CmdConsole.ITALIC) - else: - _console.print_color("PASSED", Color.FOREST_GREEN, CmdConsole.BOLD) + _console.print_color("FAILED", Color.FIREBRICK, CmdConsole.BOLD) + elif event.is_warning(): + _console.print_color("WARNING", Color.GOLDENROD, CmdConsole.BOLD | CmdConsole.UNDERLINE) + _console.prints_color( " %s" % LocalTime.elapsed(event.elapsed_time()), Color.CORNFLOWER_BLUE ) diff --git a/addons/gdUnit4/src/core/event/GdUnitEvent.gd b/addons/gdUnit4/src/core/event/GdUnitEvent.gd index 0b29da7d..09a5b294 100644 --- a/addons/gdUnit4/src/core/event/GdUnitEvent.gd +++ b/addons/gdUnit4/src/core/event/GdUnitEvent.gd @@ -74,8 +74,9 @@ func test_after(p_resource_path :String, p_suite_name :String, p_test_name :Stri return self -func test_statistics(p_suite_name :String, p_test_name :String, p_statistics :Dictionary = {}) -> GdUnitEvent: +func test_statistics(p_resource_path :String, p_suite_name :String, p_test_name :String, p_statistics :Dictionary = {}) -> GdUnitEvent: _event_type = TESTCASE_STATISTICS + _resource_path = p_resource_path _suite_name = p_suite_name _test_name = p_test_name _statistics = p_statistics diff --git a/addons/gdUnit4/src/core/execution/GdUnitExecutionContext.gd b/addons/gdUnit4/src/core/execution/GdUnitExecutionContext.gd index bbe0fc88..9b4c3240 100644 --- a/addons/gdUnit4/src/core/execution/GdUnitExecutionContext.gd +++ b/addons/gdUnit4/src/core/execution/GdUnitExecutionContext.gd @@ -174,6 +174,7 @@ func add_orphan_report_teststage(p_reports :Array[GdUnitReport]) -> int: var calculated := false var _is_success: bool var _is_flaky: bool +var _is_skipped: bool var _has_warnings: bool var _has_failures: bool var _has_errors: bool @@ -192,6 +193,7 @@ func build_reports(recursive := true) -> Array[GdUnitReport]: if _orphan_count > 0: collected_reports.push_front(GdUnitReport.new() \ .create(GdUnitReport.WARN, 1, GdAssertMessages.orphan_detected_on_suite_setup(_orphan_count))) + _is_skipped = is_skipped() _is_success = is_success() _is_flaky = is_flaky() _has_warnings = has_warnings() @@ -206,9 +208,10 @@ func build_reports(recursive := true) -> Array[GdUnitReport]: func evaluate_test_retry_status() -> bool: # get last test retry status var last_test_status :GdUnitExecutionContext = _sub_context.back() + _is_skipped = last_test_status.is_skipped() _is_success = last_test_status.is_success() # if we have more than one sub contexts the test was rerurn and marked as flaky - _is_flaky = _sub_context.size() > 1 + _is_flaky = _is_success and _sub_context.size() > 1 _has_warnings = last_test_status.has_warnings() _has_errors = last_test_status.has_errors() _error_count = last_test_status.count_errors(false) @@ -228,6 +231,7 @@ func evaluate_test_retry_status() -> bool: func evaluate_test_case_status() -> bool: if calculated: return _is_success + _is_skipped = is_skipped() _is_success = is_success() _is_flaky = is_flaky() _has_warnings = has_warnings() @@ -255,7 +259,7 @@ func build_report_statistics(recursive := true) -> Dictionary: GdUnitEvent.ERRORS: _has_errors, GdUnitEvent.WARNINGS: _has_warnings, GdUnitEvent.FLAKY: _is_flaky, - GdUnitEvent.SKIPPED: has_skipped(), + GdUnitEvent.SKIPPED: _is_skipped, GdUnitEvent.FAILED_COUNT: _failure_count, GdUnitEvent.ERROR_COUNT: _error_count, GdUnitEvent.SKIPPED_COUNT: count_skipped(recursive) @@ -304,9 +308,12 @@ func is_success() -> bool: func is_skipped() -> bool: - if test_case != null: - return test_case.is_skipped() - return false + return ( + _sub_context.any(func(c :GdUnitExecutionContext) -> bool: + return c._is_skipped if c.calculated else c.is_skipped()) + or test_case.is_skipped() if test_case != null else false + ) + func is_interupted() -> bool: diff --git a/addons/gdUnit4/src/core/execution/stages/GdUnitTestCaseExecutionStage.gd b/addons/gdUnit4/src/core/execution/stages/GdUnitTestCaseExecutionStage.gd index 13a6dd82..8a55d5fa 100644 --- a/addons/gdUnit4/src/core/execution/stages/GdUnitTestCaseExecutionStage.gd +++ b/addons/gdUnit4/src/core/execution/stages/GdUnitTestCaseExecutionStage.gd @@ -30,7 +30,7 @@ func _execute(context :GdUnitExecutionContext) -> void: await context.error_monitor_stop() # finally fire test statistics report - fire_event(GdUnitEvent.new().test_statistics(context.get_test_suite_name(), context.get_test_case_name(), context.build_report_statistics(0))) + fire_event(GdUnitEvent.new().test_statistics(context.test_suite.get_script().resource_path, context.get_test_suite_name(), context.get_test_case_name(), context.build_report_statistics(0))) # finally free the test instance if is_instance_valid(context.test_case): diff --git a/addons/gdUnit4/src/report/GdUnitHtmlReport.gd b/addons/gdUnit4/src/report/GdUnitHtmlReport.gd index fda9833e..0deb712b 100644 --- a/addons/gdUnit4/src/report/GdUnitHtmlReport.gd +++ b/addons/gdUnit4/src/report/GdUnitHtmlReport.gd @@ -18,10 +18,11 @@ func add_testsuite_report(suite_report :GdUnitTestSuiteReport) -> void: @warning_ignore("shadowed_variable") -func add_testcase_report(resource_path :String, suite_report :GdUnitTestCaseReport) -> void: - for report in _reports: +func add_testcase(resource_path :String, suite_name :String, test_name: String) -> void: + for report:GdUnitTestSuiteReport in _reports: if report.resource_path() == resource_path: - report.add_report(suite_report) + var test_report := GdUnitTestCaseReport.new(resource_path, suite_name, test_name) + report.add_or_create_test_report(test_report) @warning_ignore("shadowed_variable") @@ -37,20 +38,29 @@ func update_test_suite_report( orphan_count :int, reports :Array = []) -> void: - for report in _reports: + for report:GdUnitTestSuiteReport in _reports: if report.resource_path() == resource_path: report.set_duration(duration) - report.set_failed(is_failed, failed_count) + if is_failed: + report.set_failed(failed_count) report.set_skipped(skipped_count) report.set_orphans(orphan_count) report.set_reports(reports) @warning_ignore("shadowed_variable") -func update_testcase_report(resource_path :String, test_report :GdUnitTestCaseReport) -> void: - for report in _reports: +func add_testcase_reports(resource_path: String, test_name: String, reports: Array[GdUnitReport]) -> void: + for report:GdUnitTestSuiteReport in _reports: if report.resource_path() == resource_path: - report.update(test_report) + report.add_testcase_reports(test_name, reports) + + +@warning_ignore("shadowed_variable") +func update_testsuite_counters(resource_path :String, error_count: int, failure_count: int, orphan_count: int, + is_skipped: bool, is_flaky: bool, duration: int) -> void: + for report:GdUnitTestSuiteReport in _reports: + if report.resource_path() == resource_path: + report.update_testsuite_counters(error_count, failure_count, orphan_count, is_skipped, is_flaky, duration) func write() -> String: diff --git a/addons/gdUnit4/src/report/GdUnitReportSummary.gd b/addons/gdUnit4/src/report/GdUnitReportSummary.gd index 4e095471..061bd1af 100644 --- a/addons/gdUnit4/src/report/GdUnitReportSummary.gd +++ b/addons/gdUnit4/src/report/GdUnitReportSummary.gd @@ -83,7 +83,6 @@ func flaky_count() -> int: return count - func orphan_count() -> int: var count := _orphan_count for report in _reports: @@ -98,7 +97,7 @@ func duration() -> int: return count -func reports() -> Array: +func get_reports() -> Array: return _reports diff --git a/addons/gdUnit4/src/report/GdUnitTestCaseReport.gd b/addons/gdUnit4/src/report/GdUnitTestCaseReport.gd index bcfe1b8c..b7dedece 100644 --- a/addons/gdUnit4/src/report/GdUnitTestCaseReport.gd +++ b/addons/gdUnit4/src/report/GdUnitTestCaseReport.gd @@ -36,7 +36,7 @@ func suite_name() -> String: func failure_report() -> String: var html_report := "" - for report in _failure_reports: + for report in get_test_reports(): html_report += convert_rtf_to_html(str(report)) return html_report @@ -51,11 +51,9 @@ func create_record(_report_dir :String) -> String: .replace(GdUnitHtmlPatterns.FAILURE_REPORT, failure_report()) -func update(report :GdUnitTestCaseReport) -> void: - _error_count += report.error_count() - _failure_count += report.failure_count() - _orphan_count += report.orphan_count() - _skipped_count += report.skipped_count() - _failure_reports += report._failure_reports - _flaky_count += report.flaky_count() - _duration += report.duration() +func add_testcase_reports(reports: Array[GdUnitReport]) -> void: + _failure_reports.append_array(reports) + + +func get_test_reports() -> Array[GdUnitReport]: + return _failure_reports diff --git a/addons/gdUnit4/src/report/GdUnitTestSuiteReport.gd b/addons/gdUnit4/src/report/GdUnitTestSuiteReport.gd index abe0a843..49de867a 100644 --- a/addons/gdUnit4/src/report/GdUnitTestSuiteReport.gd +++ b/addons/gdUnit4/src/report/GdUnitTestSuiteReport.gd @@ -74,25 +74,38 @@ func duration() -> int: func set_skipped(skipped :int) -> void: - _skipped_count = skipped + _skipped_count += skipped func set_orphans(orphans :int) -> void: _orphan_count = orphans -func set_failed(failed :bool, count :int) -> void: - if failed: - _failure_count += count +func set_failed(count :int) -> void: + _failure_count += count func set_reports(failure_reports :Array[GdUnitReport]) -> void: _failure_reports = failure_reports -func update(test_report :GdUnitTestCaseReport) -> void: +func add_or_create_test_report(test_report: GdUnitTestCaseReport) -> void: + _reports.append(test_report) + + +func update_testsuite_counters(p_error_count: int, p_failure_count: int, p_orphan_count: int, + p_is_skipped: bool, p_is_flaky: bool, p_duration: int) -> void: + _error_count += p_error_count + _failure_count += p_failure_count + _orphan_count += p_orphan_count + _skipped_count += p_is_skipped as int + _flaky_count += p_is_flaky as int + _duration += p_duration + + +func add_testcase_reports(test_name: String, reports: Array[GdUnitReport] ) -> void: # we lookup to latest matching report because of flaky tests could be retry the tests # and resultis in multipe report entries with the same name _reports.filter(func (report: GdUnitTestCaseReport) -> bool: - return report.name() == test_report.name() - ).back().update(test_report) + return report.name() == test_name + ).back().add_testcase_reports(reports) diff --git a/addons/gdUnit4/src/report/JUnitXmlReport.gd b/addons/gdUnit4/src/report/JUnitXmlReport.gd index f0eac35e..8921564b 100644 --- a/addons/gdUnit4/src/report/JUnitXmlReport.gd +++ b/addons/gdUnit4/src/report/JUnitXmlReport.gd @@ -58,8 +58,8 @@ func build_junit_report(report :GdUnitReportSummary) -> String: func build_test_suites(summary :GdUnitReportSummary) -> Array: var test_suites :Array[XmlElement] = [] - for index in summary.reports().size(): - var suite_report :GdUnitTestSuiteReport = summary.reports()[index] + for index in summary.get_reports().size(): + var suite_report :GdUnitTestSuiteReport = summary.get_reports()[index] var iso8601_datetime := Time.get_datetime_string_from_unix_time(suite_report.time_stamp()) test_suites.append(XmlElement.new("testsuite")\ .attribute(ATTR_ID, index)\ @@ -79,8 +79,8 @@ func build_test_suites(summary :GdUnitReportSummary) -> Array: func build_test_cases(suite_report :GdUnitTestSuiteReport) -> Array: var test_cases :Array[XmlElement] = [] - for index in suite_report.reports().size(): - var report :GdUnitTestCaseReport = suite_report.reports()[index] + for index in suite_report.get_reports().size(): + var report :GdUnitTestCaseReport = suite_report.get_reports()[index] test_cases.append( XmlElement.new("testcase")\ .attribute(ATTR_NAME, JUnitXmlReport.encode_xml(report.name()))\ .attribute(ATTR_CLASSNAME, report.suite_name())\ @@ -89,24 +89,24 @@ func build_test_cases(suite_report :GdUnitTestSuiteReport) -> Array: return test_cases -func build_reports(test_report :GdUnitTestCaseReport) -> Array: +func build_reports(test_report: GdUnitTestCaseReport) -> Array: var failure_reports :Array[XmlElement] = [] - if test_report.failure_count() or test_report.error_count(): - for report: GdUnitReport in test_report._failure_reports: - if report.is_failure(): - failure_reports.append( XmlElement.new("failure")\ - .attribute(ATTR_MESSAGE, "FAILED: %s:%d" % [test_report._resource_path, report.line_number()])\ - .attribute(ATTR_TYPE, JUnitXmlReport.to_type(report.type()))\ - .text(convert_rtf_to_text(report.message()))) - elif report.is_error(): - failure_reports.append( XmlElement.new("error")\ - .attribute(ATTR_MESSAGE, "ERROR: %s:%d" % [test_report._resource_path, report.line_number()])\ - .attribute(ATTR_TYPE, JUnitXmlReport.to_type(report.type()))\ - .text(convert_rtf_to_text(report.message()))) - if test_report.skipped_count(): - for report: GdUnitReport in test_report._failure_reports: - failure_reports.append( XmlElement.new("skipped")\ - .attribute(ATTR_MESSAGE, "SKIPPED: %s:%d" % [test_report._resource_path, report.line_number()])) + + for report: GdUnitReport in test_report.get_test_reports(): + if report.is_failure(): + failure_reports.append(XmlElement.new("failure")\ + .attribute(ATTR_MESSAGE, "FAILED: %s:%d" % [test_report._resource_path, report.line_number()])\ + .attribute(ATTR_TYPE, JUnitXmlReport.to_type(report.type()))\ + .text(convert_rtf_to_text(report.message()))) + elif report.is_error(): + failure_reports.append(XmlElement.new("error")\ + .attribute(ATTR_MESSAGE, "ERROR: %s:%d" % [test_report._resource_path, report.line_number()])\ + .attribute(ATTR_TYPE, JUnitXmlReport.to_type(report.type()))\ + .text(convert_rtf_to_text(report.message()))) + elif report.is_skipped(): + failure_reports.append(XmlElement.new("skipped")\ + .attribute(ATTR_MESSAGE, "SKIPPED: %s:%d" % [test_report._resource_path, report.line_number()])\ + .text(convert_rtf_to_text(report.message()))) return failure_reports diff --git a/addons/gdUnit4/test/asserts/GdUnitGodotErrorWithAssertTest.gd b/addons/gdUnit4/test/asserts/GdUnitGodotErrorWithAssertTest.gd index bef6bbe5..751ccdef 100644 --- a/addons/gdUnit4/test/asserts/GdUnitGodotErrorWithAssertTest.gd +++ b/addons/gdUnit4/test/asserts/GdUnitGodotErrorWithAssertTest.gd @@ -26,7 +26,9 @@ func do_a_fail() -> void: func catch_test_events(event :GdUnitEvent) -> void: - _catched_events.append(event) + # we not catch the statistics + if event.type() != GdUnitEvent.TESTCASE_STATISTICS: + _catched_events.append(event) func before() -> void: diff --git a/addons/gdUnit4/test/ui/parts/InspectorProgressBarTest.gd b/addons/gdUnit4/test/ui/parts/InspectorProgressBarTest.gd index 6dfced7e..1fc593e0 100644 --- a/addons/gdUnit4/test/ui/parts/InspectorProgressBarTest.gd +++ b/addons/gdUnit4/test/ui/parts/InspectorProgressBarTest.gd @@ -38,7 +38,7 @@ func test_progress_success() -> void: var expected_progess_index :float = 0 # simulate execution of 20 success test runs for index in 20: - _runner.invoke("_on_gdunit_event", GdUnitEvent.new().test_after("res://test/testA.gd", "TestSuiteA", "test_a%d" % index, {})) + _runner.invoke("_on_gdunit_event", GdUnitEvent.new().test_statistics("res://test/testA.gd", "TestSuiteA", "test_a%d" % index, {})) expected_progess_index += 1 assert_that(_progress.value).is_equal(expected_progess_index) assert_that(_status.text).is_equal("%d:42" % expected_progess_index) @@ -49,7 +49,7 @@ func test_progress_success() -> void: _runner.invoke("_on_gdunit_event", GdUnitEvent.new().test_after("res://test/testA.gd", "TestSuiteA", "test_parameterized:%d (params)" % index, {})) assert_that(_progress.value).is_equal(expected_progess_index) # final test end event - _runner.invoke("_on_gdunit_event", GdUnitEvent.new().test_after("res://test/testA.gd", "TestSuiteA", "test_parameterized", {})) + _runner.invoke("_on_gdunit_event", GdUnitEvent.new().test_statistics("res://test/testA.gd", "TestSuiteA", "test_parameterized", {})) # we expect only one progress step after a parameterized test has been executed, regardless of the iterations expected_progess_index += 1 assert_that(_progress.value).is_equal(expected_progess_index) @@ -80,5 +80,5 @@ func test_progress_failed(test_name :String, is_failed :bool, is_error :bool, ex GdUnitEvent.SKIPPED_COUNT: 0, } - _runner.invoke("_on_gdunit_event", GdUnitEvent.new().test_after("res://test/testA.gd", "TestSuiteA", test_name, statistics)) + _runner.invoke("_on_gdunit_event", GdUnitEvent.new().test_statistics("res://test/testA.gd", "TestSuiteA", test_name, statistics)) assert_that(_style.bg_color).is_equal(expected_color)