Tests - update test command to show failures

src/main/java/ch/njol/skript/Skript.java

@@ -669,7 +669,10 @@ protected void afterErrors() {
 				debug("Early init done");
 
 				if (TestMode.ENABLED) {
-					Bukkit.getWorlds().get(0).getChunkAtAsync(100, 100).thenRun(() -> runTests());
+					if (TestMode.DEV_MODE)
+						runTests(); // Dev mode doesn't need a delay
+					else
+						Bukkit.getWorlds().get(0).getChunkAtAsync(100, 100).thenRun(() -> runTests());
 				}
 
 				Skript.metrics = new Metrics(Skript.getInstance(), 722); // 722 is our bStats plugin ID
@@ -807,7 +810,7 @@ private void runTests() {
 			info("Loading all tests from " + TestMode.TEST_DIR);
 
 			// Treat parse errors as fatal testing failure
-			CountingLogHandler errorCounter = new CountingLogHandler(Level.SEVERE);
+			TestingLogHandler errorCounter = new TestingLogHandler(Level.SEVERE);
 			try {
 				errorCounter.start();
 				File testDir = TestMode.TEST_DIR.toFile();

src/main/java/ch/njol/skript/SkriptCommand.java

@@ -10,6 +10,7 @@
 import ch.njol.skript.localization.PluralizingArgsMessage;
 import ch.njol.skript.log.LogEntry;
 import ch.njol.skript.log.RedirectingLogHandler;
+import ch.njol.skript.log.TestingLogHandler;
 import ch.njol.skript.log.TimingLogHandler;
 import ch.njol.skript.test.runner.SkriptTestEvent;
 import ch.njol.skript.test.runner.TestMode;
@@ -416,7 +417,13 @@ public boolean onCommand(CommandSender sender, Command command, String label, St
 					return true;
 				}
 
-				ScriptLoader.loadScripts(scriptFile, logHandler)
+				// Close previous loggers before we create a new one
+				// This prevents closing logger errors
+				timingLogHandler.close();
+				logHandler.close();
+
+				TestingLogHandler errorCounter = new TestingLogHandler(Level.SEVERE).start();
+				ScriptLoader.loadScripts(scriptFile, errorCounter)
 					.thenAccept(scriptInfo ->
 						// Code should run on server thread
 						Bukkit.getScheduler().scheduleSyncDelayedTask(Skript.getInstance(), () -> {

src/main/java/ch/njol/skript/log/CountingLogHandler.java

@@ -16,7 +16,7 @@ public class CountingLogHandler extends LogHandler {
 	public CountingLogHandler(Level minimum) {
 		this.minimum = minimum.intValue();
 	}
-	
+
 	@Override
 	public LogResult log(LogEntry entry) {
 		if (entry.level.intValue() >= minimum)

src/main/java/ch/njol/skript/log/TestingLogHandler.java

@@ -0,0 +1,54 @@
+package ch.njol.skript.log;
+
+import ch.njol.skript.config.Node;
+import ch.njol.skript.lang.parser.ParserInstance;
+import ch.njol.skript.test.runner.EvtTestCase;
+import ch.njol.skript.test.runner.TestTracker;
+import org.skriptlang.skript.lang.structure.Structure;
+
+import java.util.logging.Level;
+
+/**
+ * Counts logged messages of a certain type and logs parse errors to {@link TestTracker}
+ */
+public class TestingLogHandler extends LogHandler {
+
+	private final int minimum;
+	private int count;
+	private final ParserInstance parser;
+
+	public TestingLogHandler(Level minimum) {
+		this.minimum = minimum.intValue();
+		this.parser = ParserInstance.get();
+	}
+
+	@Override
+	public LogResult log(LogEntry entry) {
+		if (entry.level.intValue() >= minimum) {
+			count++;
+			Structure struct = parser.getCurrentStructure();
+			Node node = parser.getNode();
+
+			String name = struct instanceof EvtTestCase test ? test.getTestName() : struct != null ? struct.getSyntaxTypeName() : null;
+			TestTracker.parsingStarted(name);
+
+			if (node != null) {
+				TestTracker.testFailed(entry.getMessage(), parser.getCurrentScript(), node.getLine());
+			} else {
+				TestTracker.testFailed(entry.getMessage());
+			}
+		}
+		return LogResult.LOG;
+	}
+
+	@Override
+	public TestingLogHandler start() {
+		SkriptLogger.startLogHandler(this);
+		return this;
+	}
+
+	public int getCount() {
+		return count;
+	}
+
+}

src/main/java/ch/njol/skript/test/runner/EvtTestCase.java

@@ -26,15 +26,15 @@ public class EvtTestCase extends SkriptEvent {
 		}
 	}
 
-	private Expression<String> name;
+	private Literal<String> name;
 
 	@Nullable
 	private Condition condition;
 
 	@Override
 	@SuppressWarnings("unchecked")
 	public boolean init(Literal<?>[] args, int matchedPattern, SkriptParser.ParseResult parseResult) {
-		name = (Expression<String>) args[0];
+		name = (Literal<String>) args[0];
 		if (!parseResult.regexes.isEmpty()) { // Do not parse or run unless condition is met
 			String cond = parseResult.regexes.get(0).group();
 			condition = Condition.parse(cond, "Can't understand this condition: " + cond);
@@ -44,7 +44,7 @@ public boolean init(Literal<?>[] args, int matchedPattern, SkriptParser.ParseRes
 
 	@Override
 	public boolean check(Event event) {
-		String n = name.getSingle(event);
+		String n = name.getSingle();
 		if (n == null)
 			return false;
 		Skript.info("Running test case " + n);
@@ -57,11 +57,13 @@ public boolean shouldLoadEvent() {
 		return condition != null ? condition.check(new SkriptTestEvent()) : true;
 	}
 
+	public String getTestName() {
+		return name.getSingle();
+	}
+
 	@Override
 	public String toString(@Nullable Event event, boolean debug) {
-		if (event != null)
-			return "test " + name.getSingle(event);
-		return "test case";
+		return "test " + name.getSingle();
 	}
 
 }

src/main/java/ch/njol/skript/test/runner/TestTracker.java

@@ -34,6 +34,10 @@ public static void testStarted(String name) {
 		currentTest = name;
 	}
 
+	public static void parsingStarted(String name) {
+		currentTest = name + " (parsing)";
+	}
+
 	public static void JUnitTestFailed(String currentTest, String msg) {
 		failedTests.put(currentTest, msg);
 	}

src/main/java/ch/njol/skript/test/utils/TestResults.java

@@ -44,11 +44,15 @@ public boolean docsFailed() {
 
 	public String createReport() {
 		StringBuilder sb = new StringBuilder("Succeeded:\n");
+		if (succeeded.isEmpty())
+			sb.append("<reset> - none\n");
 		for (String test : succeeded)
-			sb.append("<light green>").append(test).append('\n');
+			sb.append("<reset> - <light green>").append(test).append('\n');
 		sb.append("<reset>Failed:\n");
+		if (failed.isEmpty())
+			sb.append("<reset> - none");
 		for (Map.Entry<String, String> entry : failed.entrySet())
-			sb.append("<light red>").append(entry.getKey()).append("<reset>: <gray>").append(entry.getValue()).append('\n');
+			sb.append("<reset> - <light red>").append(entry.getKey()).append("<reset>: <gray>").append(entry.getValue()).append('\n');
 		return sb.toString();
 	}