test runner arguments fix + fixed memcheck for teardown function

lib/std/core/runtime_test.c3

@@ -3,6 +3,7 @@
 // a copy of which can be found in the LICENSE_STDLIB file.
 module std::core::runtime;
 import std::core::test @public;
+import std::core::mem::allocator @public;
 import libc, std::time, std::io, std::sort;
 import std::os::env;
 
@@ -23,6 +24,8 @@ struct TestContext
 	bool assert_print_backtrace;
 	bool has_ansi_codes;
 	bool is_in_panic;
+	bool leak_full_report;
+	bool quiet_mode;
 	String current_test_name;
 	TestFn setup_fn;
 	TestFn teardown_fn;
@@ -32,6 +35,7 @@ struct TestContext
 	File fake_stdout;
 	File orig_stdout;
 	File orig_stderr;
+	Allocator orig_allocator;
 }
 
 struct TestUnit
@@ -86,6 +90,7 @@ fn void test_panic(String message, String file, String function, uint line) @loc
 	test_context.is_in_panic = true;
 
 	unmute_output(true);
+
 	(void)io::stdout().flush();
 	if (test_context.assert_print_backtrace)
 	{
@@ -107,6 +112,8 @@ fn void test_panic(String message, String file, String function, uint line) @loc
 	}
 
 	test_context.is_in_panic = false;
+	allocator::thread_allocator = test_context.orig_allocator;
+	test_context.orig_allocator = null;
 	libc::longjmp(&test_context.buf, 1);
 }
 
@@ -148,6 +155,7 @@ fn void unmute_output(bool has_error) @local
 	usz log_size = test_context.fake_stdout.seek(0, Seek.CURSOR)!!;
 	if (has_error)
 	{
+		if (test_context.quiet_mode) io::printf("\nTesting %s ", test_context.current_test_name);
 		io::printn(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]");
 	}
 
@@ -173,6 +181,18 @@ fn void unmute_output(bool has_error) @local
 	(void)stdout.flush();
 }
 
+fn void test_runner_usage() @local
+{
+	io::printn("Test runner arguments help.");
+	io::printn("--test-filter <arg>, -f <arg>      Set a filter when running tests, running only matching tests.");
+	io::printn("--test-breakpoint, -b              When running tests, trigger a breakpoint on failure.");
+	io::printn("--test-nosort, -s                  Do not sort tests.");
+	io::printn("--test-leak-report, -l             Display full memory leak report if any.");
+	io::printn("--test-noansi                      Disable test colour output.");
+	io::printn("--test-useansi                     Enable test colour output.");
+	io::printn("--test-quiet, -q                   Replace full test names by '.' and print out only failed tests.");
+}
+
 fn bool run_tests(String[] args, TestUnit[] tests) @private
 {
 	usz max_name;
@@ -185,22 +205,37 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private
 	{
 		.assert_print_backtrace = true,
 		.breakpoint_on_assert = false,
+		.leak_full_report = false,
+		.quiet_mode = false,
 		.test_filter = "",
 		.has_ansi_codes = terminal_has_ansi_codes(),
 	};
 	for (int i = 1; i < args.len; i++)
 	{
 		switch (args[i])
 		{
-			case "breakpoint":
+			case "--help":
+			case "-h":
+				test_runner_usage();
+				return false;
+			case "--test-breakpoint":
+			case "-b":
 				context.breakpoint_on_assert = true;
-			case "nosort":
+			case "--test-nosort":
+			case "-s":
 				sort_tests = false;
-			case "noansi":
+			case "--test-quiet":
+			case "-q":
+				context.quiet_mode = true;
+			case "--test-noansi":
 				context.has_ansi_codes = false;
-			case "useansi":
+			case "--test-useansi":
 				context.has_ansi_codes = true;
-			case "filter":
+			case "--test-leak-report":
+			case "-l":
+				context.leak_full_report = true;
+			case "--test-filter":
+			case "-f":
 				if (i == args.len - 1)
 				{
 					io::printn("Invalid arguments to test runner.");
@@ -239,7 +274,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private
 	name.append_repeat('-', len / 2);
 	name.append(" TESTS ");
 	name.append_repeat('-', len - len / 2);
-	io::printn(name);
+	if(!context.quiet_mode) io::printn(name);
 	name.clear();
 	foreach(unit : tests)
 	{
@@ -255,41 +290,53 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private
 		defer name.clear();
 		name.appendf("Testing %s ", unit.name);
 		name.append_repeat('.', max_name - unit.name.len + 2);
-		io::printf("%s ", name.str_view());
+
+		if (context.quiet_mode) {
+			io::print(".");
+		} else {
+			io::printf("%s ", name.str_view());
+		}
+		(void)io::stdout().flush();
 		TrackingAllocator mem;
 		mem.init(allocator::heap());
 		if (libc::setjmp(&context.buf) == 0)
 		{
 			mute_output();
 			mem.clear();
-			mem::@scoped(&mem)
+			// NOTE: longjmp resilient mem::@scoped(&mem)
+			context.orig_allocator = allocator::thread_allocator;
+			allocator::thread_allocator = &mem;
+			$if(!$$OLD_TEST):
+				unit.func();
+			$else
+				if (catch err = unit.func())
+				{
+					io::printf("[FAIL] Failed due to: %s", err);
+					continue;
+				}
+			$endif
+			// track cleanup that may take place in teardown_fn
+			if (test_context.teardown_fn)
 			{
-				$if(!$$OLD_TEST):
-					unit.func();
-				$else
-					if (catch err = unit.func())
-					{
-						io::printf("[FAIL] Failed due to: %s", err);
-						continue;
-					}
-				$endif
-			};
+				test_context.teardown_fn();
+			}
+			allocator::thread_allocator = context.orig_allocator;
+
 			unmute_output(false); // all good, discard output
 			if (mem.has_leaks())
             {
+                if(test_context.quiet_mode) io::printf("\n%s ", context.current_test_name);
             	io::print(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]");
             	io::printn(" LEAKS DETECTED!");
-            	mem.print_report();
+            	if (test_context.leak_full_report) mem.print_report();
             }
             else
             {
-                io::printfn(test_context.has_ansi_codes ? "[\e[0;32mPASS\e[0m]" : "[PASS]");
+                if(!test_context.quiet_mode) {
+                	io::printfn(test_context.has_ansi_codes ? "[\e[0;32mPASS\e[0m]" : "[PASS]");
+                }
 				tests_passed++;
 			}
-			if (test_context.teardown_fn)
-			{
-				test_context.teardown_fn();
-			}
 		}
 		mem.free();
 	}

releasenotes.md

@@ -23,6 +23,8 @@
 - Test runner will also check for leaks.
 - Improve inference on `??` #1943.
 - Detect unaligned loads #1951.
+- c3c `--test-leak-report` flag for displaying full memory lead report if any
+- c3c `--test-quiet` flag for minifying full test names to '.' and printing out only on errors
 
 ### Fixes
 - Fix issue requiring prefix on a generic interface declaration.
@@ -63,6 +65,9 @@
 - Fix issue where aligned bitstructs did not store/load with the given alignment.
 - Fix issue in GrowableBitSet with sanitizers.
 - Fix issue in List with sanitizers.
+- Test runner --test-disable-sort fixed 
+- Test runner with tracking allocator assertion at failed test #1963
+- Test runner forcing `Testing test_name .........` stdout flush before deadlocking test
 - Circumvent Aarch64 miscompilations of atomics.
 - Fixes to ByteBuffer allocation/free.
 - Fix issue where compiling both for asm and object file would corrupt the obj file output.

src/build/build.h

@@ -504,7 +504,9 @@ typedef struct BuildOptions_
 	ValidationLevel validation_level;
 	Ansi ansi;
 	bool test_breakpoint;
+	bool test_leak_report;
 	bool test_nosort;
+	bool test_quiet;
 	const char *custom_linker_path;
 	uint32_t symtab_size;
 	unsigned version;

src/build/build_options.c

@@ -135,7 +135,9 @@ static void usage(bool full)
 		print_opt("--ansi=<yes|no>", "Set colour output using ansi on/off, default is to try to detect it.");
 		print_opt("--test-filter <arg>", "Set a filter when running tests, running only matching tests.");
 		print_opt("--test-breakpoint", "When running tests, trigger a breakpoint on failure.");
-		print_opt("--test-disable-sort", "Do not sort tests.");
+		print_opt("--test-quiet", "Replace full test names by '.' and print out only failed tests.");
+		print_opt("--test-nosort", "Do not sort tests.");
+		print_opt("--test-leak-report", "Display full memory leak report if any.");
 	}
 	PRINTF("");
 	print_opt("-l <library>", "Link with the static or dynamic library provided.");
@@ -722,6 +724,16 @@ static void parse_option(BuildOptions *options)
 				options->test_breakpoint = true;
 				return;
 			}
+			if (match_longopt("test-leak-report"))
+			{
+				options->test_leak_report = true;
+				return;
+			}
+			if (match_longopt("test-quiet"))
+			{
+				options->test_quiet = true;
+				return;
+			}
 			if (match_longopt("test-nosort"))
 			{
 				options->test_nosort = true;

src/build/builder.c

@@ -282,21 +282,23 @@ static void update_build_target_from_options(BuildTarget *target, BuildOptions *
 			switch (options->ansi)
 			{
 				case ANSI_ON:
-					vec_add(target->args, "useansi");
+					vec_add(target->args, "--test-useansi");
 					break;
 				case ANSI_OFF:
-					vec_add(target->args, "noansi");
+					vec_add(target->args, "--test-noansi");
 					break;
 				default:
 					break;
 			}
 			if (options->test_filter)
 			{
-				vec_add(target->args, "filter");
+				vec_add(target->args, "--test-filter");
 				vec_add(target->args, options->test_filter);
 			}
-			if (options->test_breakpoint) vec_add(target->args, "breakpoint");
-			if (options->test_nosort) vec_add(target->args, "nosort");
+			if (options->test_quiet) vec_add(target->args, "--test-quiet");
+			if (options->test_breakpoint) vec_add(target->args, "--test-breakpoint");
+			if (options->test_nosort) vec_add(target->args, "--test-nosort");
+			if (options->test_leak_report) vec_add(target->args, "--test-leak-report");
 			break;
 		case COMMAND_RUN:
 		case COMMAND_COMPILE_RUN:

test/unit/stdlib/core/test_test.c3

@@ -8,6 +8,7 @@ struct TestState
 	int n_runs;
 	int n_fails;
 	bool expected_fail;
+	void* some_mem;
 
 	// NOTE: we must wrap setup/teardown functions to hide them from module @test runner
 	TestFn setup_fn;
@@ -22,6 +23,7 @@ TestState state =
 	{
 		state.n_runs++;
 		state.n_fails = 0;
+		state.some_mem = mem::alloc(int);
 
 		assert (runtime::test_context.assert_print_backtrace);
 		assert (builtin::panic != state.panic_mock_fn, "missing finalization of panic");
@@ -42,6 +44,7 @@ TestState state =
 		state.n_fails = 0;
 		state.expected_fail = false;
 		state.n_runs = 0;
+		mem::free(state.some_mem);
 	},
 	.panic_mock_fn = fn void (String message, String file, String function, uint line)
 	{
@@ -139,6 +142,9 @@ fn void setup_no_teardown()
 	test::eq(state.n_fails, 0);
 	test::eq(state.expected_fail, false);
 
+	// this has to be freed in teardown
+	mem::free(state.some_mem);
+
 	// WARNING: reverting back original panic func
 	builtin::panic = state.old_panic;
 }