From 0b42753ba6fa292c7edea11ec45fa7ebcf7beb4c Mon Sep 17 00:00:00 2001 From: Alex Veden Date: Wed, 12 Feb 2025 11:20:07 +0400 Subject: [PATCH 1/5] test runner arguments fix + fixed memcheck for teardown function --- lib/std/core/runtime_test.c3 | 44 +++++++++++++++++++++++------- src/build/build.h | 1 + src/build/build_options.c | 8 +++++- src/build/builder.c | 11 ++++---- test/unit/stdlib/core/test_test.c3 | 6 ++++ 5 files changed, 54 insertions(+), 16 deletions(-) diff --git a/lib/std/core/runtime_test.c3 b/lib/std/core/runtime_test.c3 index 9bb110232..5320178c9 100644 --- a/lib/std/core/runtime_test.c3 +++ b/lib/std/core/runtime_test.c3 @@ -23,6 +23,7 @@ struct TestContext bool assert_print_backtrace; bool has_ansi_codes; bool is_in_panic; + bool leak_full_report; String current_test_name; TestFn setup_fn; TestFn teardown_fn; @@ -173,6 +174,17 @@ fn void unmute_output(bool has_error) @local (void)stdout.flush(); } +fn void test_runner_usage() @local +{ + io::printn("Test runner arguments help."); + io::printn("--test-filter , -f Set a filter when running tests, running only matching tests."); + io::printn("--test-breakpoint, -b When running tests, trigger a breakpoint on failure."); + io::printn("--test-nosort, -s Do not sort tests."); + io::printn("--test-leak-report, -l Display full memory leak report if any."); + io::printn("--test-noansi Disable test colour output"); + io::printn("--test-useansi Enable test colour output"); +} + fn bool run_tests(String[] args, TestUnit[] tests) @private { usz max_name; @@ -185,6 +197,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private { .assert_print_backtrace = true, .breakpoint_on_assert = false, + .leak_full_report = false, .test_filter = "", .has_ansi_codes = terminal_has_ansi_codes(), }; @@ -192,15 +205,25 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private { switch (args[i]) { - case "breakpoint": + case "--help": + case "-h": + test_runner_usage(); + return false; + case "--test-breakpoint": + case "-b": context.breakpoint_on_assert = true; - case "nosort": + case "--test-nosort": + case "-s": sort_tests = false; - case "noansi": + case "--test-noansi": context.has_ansi_codes = false; - case "useansi": + case "--test-useansi": context.has_ansi_codes = true; - case "filter": + case "--test-leak-report": + case "-l": + context.leak_full_report = true; + case "--test-filter": + case "-f": if (i == args.len - 1) { io::printn("Invalid arguments to test runner."); @@ -273,23 +296,24 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private continue; } $endif + // track cleanup that may take place in teardown_fn + if (test_context.teardown_fn) + { + test_context.teardown_fn(); + } }; unmute_output(false); // all good, discard output if (mem.has_leaks()) { io::print(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]"); io::printn(" LEAKS DETECTED!"); - mem.print_report(); + if (test_context.leak_full_report) mem.print_report(); } else { io::printfn(test_context.has_ansi_codes ? "[\e[0;32mPASS\e[0m]" : "[PASS]"); tests_passed++; } - if (test_context.teardown_fn) - { - test_context.teardown_fn(); - } } mem.free(); } diff --git a/src/build/build.h b/src/build/build.h index f774c74b6..268ce5069 100644 --- a/src/build/build.h +++ b/src/build/build.h @@ -504,6 +504,7 @@ typedef struct BuildOptions_ ValidationLevel validation_level; Ansi ansi; bool test_breakpoint; + bool test_leak_report; bool test_nosort; const char *custom_linker_path; uint32_t symtab_size; diff --git a/src/build/build_options.c b/src/build/build_options.c index 8b94105a6..cc4df7b13 100644 --- a/src/build/build_options.c +++ b/src/build/build_options.c @@ -135,7 +135,8 @@ static void usage(bool full) print_opt("--ansi=", "Set colour output using ansi on/off, default is to try to detect it."); print_opt("--test-filter ", "Set a filter when running tests, running only matching tests."); print_opt("--test-breakpoint", "When running tests, trigger a breakpoint on failure."); - print_opt("--test-disable-sort", "Do not sort tests."); + print_opt("--test-nosort", "Do not sort tests."); + print_opt("--test-leak-report", "Display full memory leak report if any."); } PRINTF(""); print_opt("-l ", "Link with the static or dynamic library provided."); @@ -722,6 +723,11 @@ static void parse_option(BuildOptions *options) options->test_breakpoint = true; return; } + if (match_longopt("test-leak-report")) + { + options->test_leak_report = true; + return; + } if (match_longopt("test-nosort")) { options->test_nosort = true; diff --git a/src/build/builder.c b/src/build/builder.c index d21ffc789..182def4b5 100644 --- a/src/build/builder.c +++ b/src/build/builder.c @@ -282,21 +282,22 @@ static void update_build_target_from_options(BuildTarget *target, BuildOptions * switch (options->ansi) { case ANSI_ON: - vec_add(target->args, "useansi"); + vec_add(target->args, "--test-useansi"); break; case ANSI_OFF: - vec_add(target->args, "noansi"); + vec_add(target->args, "--test-noansi"); break; default: break; } if (options->test_filter) { - vec_add(target->args, "filter"); + vec_add(target->args, "--test-filter"); vec_add(target->args, options->test_filter); } - if (options->test_breakpoint) vec_add(target->args, "breakpoint"); - if (options->test_nosort) vec_add(target->args, "nosort"); + if (options->test_breakpoint) vec_add(target->args, "--test-breakpoint"); + if (options->test_nosort) vec_add(target->args, "--test-nosort"); + if (options->test_leak_report) vec_add(target->args, "--test-leak-report"); break; case COMMAND_RUN: case COMMAND_COMPILE_RUN: diff --git a/test/unit/stdlib/core/test_test.c3 b/test/unit/stdlib/core/test_test.c3 index 36f44724b..e02daac39 100644 --- a/test/unit/stdlib/core/test_test.c3 +++ b/test/unit/stdlib/core/test_test.c3 @@ -8,6 +8,7 @@ struct TestState int n_runs; int n_fails; bool expected_fail; + void* some_mem; // NOTE: we must wrap setup/teardown functions to hide them from module @test runner TestFn setup_fn; @@ -22,6 +23,7 @@ TestState state = { state.n_runs++; state.n_fails = 0; + state.some_mem = mem::alloc(int); assert (runtime::test_context.assert_print_backtrace); assert (builtin::panic != state.panic_mock_fn, "missing finalization of panic"); @@ -42,6 +44,7 @@ TestState state = state.n_fails = 0; state.expected_fail = false; state.n_runs = 0; + mem::free(state.some_mem); }, .panic_mock_fn = fn void (String message, String file, String function, uint line) { @@ -139,6 +142,9 @@ fn void setup_no_teardown() test::eq(state.n_fails, 0); test::eq(state.expected_fail, false); + // this has to be freed in teardown + mem::free(state.some_mem); + // WARNING: reverting back original panic func builtin::panic = state.old_panic; } From 8e9448a2db89eb4f72ca9a9878b6778e979c92b5 Mon Sep 17 00:00:00 2001 From: Alex Veden Date: Wed, 12 Feb 2025 12:02:11 +0400 Subject: [PATCH 2/5] added io::stdout().flush() - to force printing test name before possible deadlock --- lib/std/core/runtime_test.c3 | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/core/runtime_test.c3 b/lib/std/core/runtime_test.c3 index 5320178c9..e172732a2 100644 --- a/lib/std/core/runtime_test.c3 +++ b/lib/std/core/runtime_test.c3 @@ -279,6 +279,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private name.appendf("Testing %s ", unit.name); name.append_repeat('.', max_name - unit.name.len + 2); io::printf("%s ", name.str_view()); + (void)io::stdout().flush(); TrackingAllocator mem; mem.init(allocator::heap()); if (libc::setjmp(&context.buf) == 0) From 8a1d17b0d40bde47058a96f72e9d654f74d33d0e Mon Sep 17 00:00:00 2001 From: Alex Veden Date: Wed, 12 Feb 2025 12:26:21 +0400 Subject: [PATCH 3/5] mem::scoped() and long jump resilience fixed #1963 --- lib/std/core/runtime_test.c3 | 36 +++++++++++++++++++++--------------- releasenotes.md | 5 +++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/lib/std/core/runtime_test.c3 b/lib/std/core/runtime_test.c3 index e172732a2..6d42f4269 100644 --- a/lib/std/core/runtime_test.c3 +++ b/lib/std/core/runtime_test.c3 @@ -3,6 +3,7 @@ // a copy of which can be found in the LICENSE_STDLIB file. module std::core::runtime; import std::core::test @public; +import std::core::mem::allocator @public; import libc, std::time, std::io, std::sort; import std::os::env; @@ -33,6 +34,7 @@ struct TestContext File fake_stdout; File orig_stdout; File orig_stderr; + Allocator orig_allocator; } struct TestUnit @@ -108,6 +110,8 @@ fn void test_panic(String message, String file, String function, uint line) @loc } test_context.is_in_panic = false; + allocator::thread_allocator = test_context.orig_allocator; + test_context.orig_allocator = null; libc::longjmp(&test_context.buf, 1); } @@ -286,23 +290,25 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private { mute_output(); mem.clear(); - mem::@scoped(&mem) - { - $if(!$$OLD_TEST): - unit.func(); - $else - if (catch err = unit.func()) - { - io::printf("[FAIL] Failed due to: %s", err); - continue; - } - $endif - // track cleanup that may take place in teardown_fn - if (test_context.teardown_fn) + // NOTE: longjmp resilient mem::@scoped(&mem) + context.orig_allocator = allocator::thread_allocator; + allocator::thread_allocator = &mem; + $if(!$$OLD_TEST): + unit.func(); + $else + if (catch err = unit.func()) { - test_context.teardown_fn(); + io::printf("[FAIL] Failed due to: %s", err); + continue; } - }; + $endif + // track cleanup that may take place in teardown_fn + if (test_context.teardown_fn) + { + test_context.teardown_fn(); + } + allocator::thread_allocator = context.orig_allocator; + unmute_output(false); // all good, discard output if (mem.has_leaks()) { diff --git a/releasenotes.md b/releasenotes.md index 08569fba5..4c9630b97 100644 --- a/releasenotes.md +++ b/releasenotes.md @@ -23,6 +23,7 @@ - Test runner will also check for leaks. - Improve inference on `??` #1943. - Detect unaligned loads #1951. +- c3c `--test-leak-report` flag for displaying full memory lead report if any ### Fixes - Fix issue requiring prefix on a generic interface declaration. @@ -63,6 +64,10 @@ - Fix issue where aligned bitstructs did not store/load with the given alignment. - Fix issue in GrowableBitSet with sanitizers. - Fix issue in List with sanitizers. +- Test runner --test-disable-sort fixed +- Test runner with tracking allocator assertion at failed test #1963 +- Test runner forcing `Testing test_name .........` stdout flush before deadlocking test + ### Stdlib changes - Added '%h' and '%H' for printing out binary data in hexadecimal using the formatter. From 48b00a09fe847715d8b3d1f02564ad8575461f0b Mon Sep 17 00:00:00 2001 From: Alex Veden Date: Wed, 12 Feb 2025 13:49:44 +0400 Subject: [PATCH 4/5] added --test-quiet flag --- lib/std/core/runtime_test.c3 | 26 +++++++++++++++++++++----- releasenotes.md | 1 + src/build/build.h | 1 + src/build/build_options.c | 6 ++++++ src/build/builder.c | 1 + 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/lib/std/core/runtime_test.c3 b/lib/std/core/runtime_test.c3 index 6d42f4269..f0b9a3cef 100644 --- a/lib/std/core/runtime_test.c3 +++ b/lib/std/core/runtime_test.c3 @@ -25,6 +25,7 @@ struct TestContext bool has_ansi_codes; bool is_in_panic; bool leak_full_report; + bool quiet_mode; String current_test_name; TestFn setup_fn; TestFn teardown_fn; @@ -89,6 +90,7 @@ fn void test_panic(String message, String file, String function, uint line) @loc test_context.is_in_panic = true; unmute_output(true); + (void)io::stdout().flush(); if (test_context.assert_print_backtrace) { @@ -153,6 +155,7 @@ fn void unmute_output(bool has_error) @local usz log_size = test_context.fake_stdout.seek(0, Seek.CURSOR)!!; if (has_error) { + io::printf("\nTesting %s ", test_context.current_test_name); io::printn(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]"); } @@ -185,8 +188,9 @@ fn void test_runner_usage() @local io::printn("--test-breakpoint, -b When running tests, trigger a breakpoint on failure."); io::printn("--test-nosort, -s Do not sort tests."); io::printn("--test-leak-report, -l Display full memory leak report if any."); - io::printn("--test-noansi Disable test colour output"); - io::printn("--test-useansi Enable test colour output"); + io::printn("--test-noansi Disable test colour output."); + io::printn("--test-useansi Enable test colour output."); + io::printn("--test-quiet, -q Replace full test names by '.' and print out only failed tests."); } fn bool run_tests(String[] args, TestUnit[] tests) @private @@ -202,6 +206,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private .assert_print_backtrace = true, .breakpoint_on_assert = false, .leak_full_report = false, + .quiet_mode = false, .test_filter = "", .has_ansi_codes = terminal_has_ansi_codes(), }; @@ -219,6 +224,9 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private case "--test-nosort": case "-s": sort_tests = false; + case "--test-quiet": + case "-q": + context.quiet_mode = true; case "--test-noansi": context.has_ansi_codes = false; case "--test-useansi": @@ -266,7 +274,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private name.append_repeat('-', len / 2); name.append(" TESTS "); name.append_repeat('-', len - len / 2); - io::printn(name); + if(!context.quiet_mode) io::printn(name); name.clear(); foreach(unit : tests) { @@ -282,7 +290,12 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private defer name.clear(); name.appendf("Testing %s ", unit.name); name.append_repeat('.', max_name - unit.name.len + 2); - io::printf("%s ", name.str_view()); + + if (context.quiet_mode) { + io::print("."); + } else { + io::printf("%s ", name.str_view()); + } (void)io::stdout().flush(); TrackingAllocator mem; mem.init(allocator::heap()); @@ -312,13 +325,16 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private unmute_output(false); // all good, discard output if (mem.has_leaks()) { + if(test_context.quiet_mode) io::printf("\n%s ", context.current_test_name); io::print(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]"); io::printn(" LEAKS DETECTED!"); if (test_context.leak_full_report) mem.print_report(); } else { - io::printfn(test_context.has_ansi_codes ? "[\e[0;32mPASS\e[0m]" : "[PASS]"); + if(!test_context.quiet_mode) { + io::printfn(test_context.has_ansi_codes ? "[\e[0;32mPASS\e[0m]" : "[PASS]"); + } tests_passed++; } } diff --git a/releasenotes.md b/releasenotes.md index 4c9630b97..dcd4be3ea 100644 --- a/releasenotes.md +++ b/releasenotes.md @@ -24,6 +24,7 @@ - Improve inference on `??` #1943. - Detect unaligned loads #1951. - c3c `--test-leak-report` flag for displaying full memory lead report if any +- c3c `--test-quiet` flag for minifying full test names to '.' and printing out only on errors ### Fixes - Fix issue requiring prefix on a generic interface declaration. diff --git a/src/build/build.h b/src/build/build.h index 268ce5069..2a2b1ec25 100644 --- a/src/build/build.h +++ b/src/build/build.h @@ -506,6 +506,7 @@ typedef struct BuildOptions_ bool test_breakpoint; bool test_leak_report; bool test_nosort; + bool test_quiet; const char *custom_linker_path; uint32_t symtab_size; unsigned version; diff --git a/src/build/build_options.c b/src/build/build_options.c index cc4df7b13..067f6a92a 100644 --- a/src/build/build_options.c +++ b/src/build/build_options.c @@ -135,6 +135,7 @@ static void usage(bool full) print_opt("--ansi=", "Set colour output using ansi on/off, default is to try to detect it."); print_opt("--test-filter ", "Set a filter when running tests, running only matching tests."); print_opt("--test-breakpoint", "When running tests, trigger a breakpoint on failure."); + print_opt("--test-quiet", "Replace full test names by '.' and print out only failed tests."); print_opt("--test-nosort", "Do not sort tests."); print_opt("--test-leak-report", "Display full memory leak report if any."); } @@ -728,6 +729,11 @@ static void parse_option(BuildOptions *options) options->test_leak_report = true; return; } + if (match_longopt("test-quiet")) + { + options->test_quiet = true; + return; + } if (match_longopt("test-nosort")) { options->test_nosort = true; diff --git a/src/build/builder.c b/src/build/builder.c index 182def4b5..8b978a082 100644 --- a/src/build/builder.c +++ b/src/build/builder.c @@ -295,6 +295,7 @@ static void update_build_target_from_options(BuildTarget *target, BuildOptions * vec_add(target->args, "--test-filter"); vec_add(target->args, options->test_filter); } + if (options->test_quiet) vec_add(target->args, "--test-quiet"); if (options->test_breakpoint) vec_add(target->args, "--test-breakpoint"); if (options->test_nosort) vec_add(target->args, "--test-nosort"); if (options->test_leak_report) vec_add(target->args, "--test-leak-report"); From ec3609fcc6b0532ca99ad55e481621c73ff88865 Mon Sep 17 00:00:00 2001 From: Alex Veden Date: Wed, 12 Feb 2025 14:01:15 +0400 Subject: [PATCH 5/5] redundant print on test failure in !quiet_mode --- lib/std/core/runtime_test.c3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/core/runtime_test.c3 b/lib/std/core/runtime_test.c3 index f0b9a3cef..b5707eb29 100644 --- a/lib/std/core/runtime_test.c3 +++ b/lib/std/core/runtime_test.c3 @@ -155,7 +155,7 @@ fn void unmute_output(bool has_error) @local usz log_size = test_context.fake_stdout.seek(0, Seek.CURSOR)!!; if (has_error) { - io::printf("\nTesting %s ", test_context.current_test_name); + if (test_context.quiet_mode) io::printf("\nTesting %s ", test_context.current_test_name); io::printn(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]"); }