Improve test failure messages (#89967)

dotnet · Aug 7, 2023 · 4415472 · 4415472
1 parent 26103db
commit 4415472
Show file tree

Hide file tree

Showing 12 changed files with 52 additions and 3 deletions.
diff --git a/...nk/test/Mono.Linker.Tests.Cases.Expectations/Assertions/ExpectNonZeroExitCodeAttribute.cs b/...nk/test/Mono.Linker.Tests.Cases.Expectations/Assertions/ExpectNonZeroExitCodeAttribute.cs
@@ -0,0 +1,14 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+
+namespace Mono.Linker.Tests.Cases.Expectations.Assertions;
+
+[AttributeUsage (AttributeTargets.Class)]
+public class ExpectNonZeroExitCodeAttribute : BaseExpectedLinkedBehaviorAttribute
+{
+	public ExpectNonZeroExitCodeAttribute (int value)
+	{
+	}
+}
diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/CommandLine/InvalidArguments.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/CommandLine/InvalidArguments.cs
@@ -3,6 +3,7 @@
 
 namespace Mono.Linker.Tests.Cases.CommandLine
 {
+	[ExpectNonZeroExitCode (1)]
 	[SetupLinkerArgument ("--verbose", "--invalidArgument")]
 	[LogContains ("Unrecognized command-line option")]
 	[NoLinkedOutput]

diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/FeatureSettings/FeatureSubstitutionsInvalid.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/FeatureSettings/FeatureSubstitutionsInvalid.cs
@@ -3,6 +3,7 @@
 
 namespace Mono.Linker.Tests.Cases.FeatureSettings
 {
+	[ExpectNonZeroExitCode (1)]
 	[SetupLinkerSubstitutionFile ("FeatureSubstitutionsInvalid.xml")]
 	[SetupLinkerArgument ("--feature", "NoValueFeature", "true")]
 	[LogContains ("FeatureSubstitutionsInvalid.xml'. Feature 'NoValueFeature' does not specify a 'featurevalue' attribute")]

diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/LinkAttributes/TypedArgumentsErrors.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/LinkAttributes/TypedArgumentsErrors.cs
@@ -4,6 +4,7 @@
 
 namespace Mono.Linker.Tests.Cases.LinkAttributes
 {
+	[ExpectNonZeroExitCode (1)]
 	[SetupLinkAttributesFile ("TypedArgumentsErrors.xml")]
 	[IgnoreLinkAttributes (false)]
 	[NoLinkedOutput]

diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Logging/CommonLogs.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Logging/CommonLogs.cs
@@ -3,6 +3,7 @@
 
 namespace Mono.Linker.Tests.Cases.Logging
 {
+	[ExpectNonZeroExitCode (1)]
 	[SetupCompileBefore ("LogStep.dll", new[] { "Dependencies/LogStep.cs" }, new[] { "illink.dll", "Mono.Cecil.dll" })]
 	[SetupLinkerArgument ("--custom-step", "Log.LogStep,LogStep.dll")]
 	[SetupLinkerArgument ("--verbose")]

diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanDisableWarnAsError.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanDisableWarnAsError.cs
@@ -3,6 +3,7 @@
 
 namespace Mono.Linker.Tests.Cases.Warnings
 {
+	[ExpectNonZeroExitCode (1)]
 	[IgnoreTestCase ("Ignore in NativeAOT, see https://github.com/dotnet/runtime/issues/82447", IgnoredBy = Tool.NativeAot)]
 	[SkipKeptItemsValidation]
 	[SkipRemainingErrorsValidation]

diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanSingleWarnWithWarnAsError.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanSingleWarnWithWarnAsError.cs
@@ -5,6 +5,7 @@
 
 namespace Mono.Linker.Tests.Cases.Warnings
 {
+	[ExpectNonZeroExitCode (1)]
 	[IgnoreTestCase ("Ignore in NativeAOT, see https://github.com/dotnet/runtime/issues/82447", IgnoredBy = Tool.NativeAot)]
 	[SkipKeptItemsValidation]
 	[SetupCompileBefore ("library.dll", new[] { typeof (TriggerWarnings_Lib) })]

diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanWarnAsError.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanWarnAsError.cs
@@ -3,6 +3,7 @@
 
 namespace Mono.Linker.Tests.Cases.Warnings
 {
+	[ExpectNonZeroExitCode (1)]
 	[IgnoreTestCase ("Ignore in NativeAOT, see https://github.com/dotnet/runtime/issues/82447", IgnoredBy = Tool.NativeAot)]
 	[SkipKeptItemsValidation]
 	[SetupLinkerSubstitutionFile ("CanWarnAsErrorSubstitutions.xml")]

diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/InvalidWarningVersion.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/InvalidWarningVersion.cs
@@ -3,6 +3,7 @@
 
 namespace Mono.Linker.Tests.Cases.Warnings
 {
+	[ExpectNonZeroExitCode (1)]
 	[IgnoreTestCase ("Ignore in NativeAOT, see https://github.com/dotnet/runtime/issues/82447", IgnoredBy = Tool.NativeAot)]
 	[SetupLinkerArgument ("--verbose")]
 	[SetupLinkerArgument ("--warn", "invalid")]

diff --git a/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/LinkedTestCaseResult.cs b/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/LinkedTestCaseResult.cs
@@ -17,8 +17,9 @@ public class LinkedTestCaseResult
 		public readonly ManagedCompilationResult CompilationResult;
 		public readonly LinkerTestLogger Logger;
 		public readonly LinkerCustomizations Customizations;
+		public readonly int ExitCode;
 
-		public LinkedTestCaseResult (TestCase testCase, NPath inputAssemblyPath, NPath outputAssemblyPath, NPath expectationsAssemblyPath, TestCaseSandbox sandbox, TestCaseMetadataProvider metadataProvider, ManagedCompilationResult compilationResult, LinkerTestLogger logger, LinkerCustomizations customizations)
+		public LinkedTestCaseResult (TestCase testCase, NPath inputAssemblyPath, NPath outputAssemblyPath, NPath expectationsAssemblyPath, TestCaseSandbox sandbox, TestCaseMetadataProvider metadataProvider, ManagedCompilationResult compilationResult, LinkerTestLogger logger, LinkerCustomizations customizations, int exitCode)
 		{
 			TestCase = testCase;
 			InputAssemblyPath = inputAssemblyPath;
@@ -29,6 +30,7 @@ public LinkedTestCaseResult (TestCase testCase, NPath inputAssemblyPath, NPath o
 			CompilationResult = compilationResult;
 			Logger = logger;
 			Customizations = customizations;
+			ExitCode = exitCode;
 		}
 	}
 }
diff --git a/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/ResultChecker.cs b/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/ResultChecker.cs
@@ -7,6 +7,7 @@
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Text.RegularExpressions;
 using Mono.Cecil;
 using Mono.Cecil.Cil;
@@ -110,8 +111,12 @@ public virtual void Check (LinkedTestCaseResult linkResult)
 
 			try {
 				var original = ResolveOriginalsAssembly (linkResult.ExpectationsAssemblyPath.FileNameWithoutExtension);
+
+				VerifyExitCode (linkResult, original);
+
 				if (!HasAttribute (original, nameof (NoLinkedOutputAttribute))) {
 					Assert.IsTrue (linkResult.OutputAssemblyPath.FileExists (), $"The linked output assembly was not found.  Expected at {linkResult.OutputAssemblyPath}");
+
 					var linked = ResolveLinkedAssembly (linkResult.OutputAssemblyPath.FileNameWithoutExtension);
 
 					if (ShouldValidateIL (original)) {
@@ -239,6 +244,26 @@ void PerformOutputSymbolChecks (AssemblyDefinition original, NPath outputDirecto
 			}
 		}
 
+		void VerifyExitCode (LinkedTestCaseResult linkResult, AssemblyDefinition original)
+		{
+			if (TryGetCustomAttribute (original, nameof(ExpectNonZeroExitCodeAttribute), out var attr)) {
+				var expectedExitCode = (int) attr.ConstructorArguments[0].Value;
+				Assert.AreEqual (expectedExitCode, linkResult.ExitCode, $"Expected exit code {expectedExitCode} but got {linkResult.ExitCode}.  Output was:\n{FormatLinkerOutput()}");
+			} else {
+				if (linkResult.ExitCode != 0) {
+					Assert.Fail($"Linker exited with an unexpected non-zero exit code of {linkResult.ExitCode} and output:\n{FormatLinkerOutput()}");
+				}
+			}
+
+			string FormatLinkerOutput ()
+			{
+				var sb = new StringBuilder ();
+				foreach (var message in linkResult.Logger.GetLoggedMessages ())
+					sb.AppendLine (message.ToString ());
+				return sb.ToString ();
+			}
+		}
+
 		void VerifyKeptSymbols (CustomAttribute symbolsAttribute)
 		{
 			var assemblyName = (string) symbolsAttribute.ConstructorArguments[0].Value;

diff --git a/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/TestRunner.cs b/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/TestRunner.cs
@@ -109,9 +109,9 @@ private LinkedTestCaseResult Link (TestCase testCase, TestCaseSandbox sandbox, M
 			AddLinkOptions (sandbox, compilationResult, builder, metadataProvider);
 
 			LinkerTestLogger logger = new LinkerTestLogger ();
-			linker.Link (builder.ToArgs (), linkerCustomizations, logger);
+			var exitCode = linker.Link (builder.ToArgs (), linkerCustomizations, logger);
 
-			return new LinkedTestCaseResult (testCase, compilationResult.InputAssemblyPath, sandbox.OutputDirectory.Combine (compilationResult.InputAssemblyPath.FileName), compilationResult.ExpectationsAssemblyPath, sandbox, metadataProvider, compilationResult, logger, linkerCustomizations);
+			return new LinkedTestCaseResult (testCase, compilationResult.InputAssemblyPath, sandbox.OutputDirectory.Combine (compilationResult.InputAssemblyPath.FileName), compilationResult.ExpectationsAssemblyPath, sandbox, metadataProvider, compilationResult, logger, linkerCustomizations, exitCode);
 		}
 
 		protected virtual void AddLinkOptions (TestCaseSandbox sandbox, ManagedCompilationResult compilationResult, LinkerArgumentBuilder builder, TestCaseMetadataProvider metadataProvider)