Skip to content

Commit

Permalink
[tests] Improve the CustomizedCodeSigning tests to not depend on file…
Browse files Browse the repository at this point in the history 
… order. (xamarin#20889)

The CustomizedCodeSigning test asserts that a certain condition shows a
particular error message from codesign.

There are multiple files in the app bundle that can trigger this particular
message, so change the logic to not assert on a particular file, instead use
assert on the remained of the error message.

Fixes this random test failure:

    Xamarin.Tests.DotNetProjectTest.CustomizedCodeSigning(iOS,"ios-arm64"): Failure when comparing error messages:
        Unexpected error message #0:
            Expected: /usr/bin/codesign exited with code 1:\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all\nIn subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Diagnostics.DiagnosticSource.dll
            Actual: /usr/bin/codesign exited with code 1:\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all\nIn subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Collections.NonGeneric.aotdata.arm64
        Unexpected error message #1:
            Expected: Failed to codesign '/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app': /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all\nIn subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Diagnostics.DiagnosticSource.dll
            Actual: Failed to codesign '/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app': /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all\nIn subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Collections.NonGeneric.aotdata.arm64
        All errors:
            /usr/bin/codesign exited with code 1:
                /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature
                /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all
                In subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Collections.NonGeneric.aotdata.arm64
                        Failed to codesign '/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app': /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature
                /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all
                In subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Collections.NonGeneric.aotdata.arm64

---------

Co-authored-by: Alex Soto <alex@alexsoto.me>
Co-authored-by: Manuel de la Pena <mandel@microsoft.com>
  • Loading branch information
@rolfbjarne @dalexsoto @mandel-macaque
3 people authored Aug 12, 2024
1 parent f790ba0 commit 7da0418
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 13 deletions.
14 changes: 9 additions & 5 deletions tests/dotnet/UnitTests/ProjectTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1508,15 +1508,19 @@ public void CustomizedCodeSigning (ApplePlatform platform, string runtimeIdentif
properties ["CodesignDisallowResourcesSubdirectoryInAppBundle"] = "false";
buildFailure = DotNet.AssertBuildFailure (project_path, properties);
errors = BinLog.GetBuildLogErrors (buildFailure.BinLogPath).ToArray ();
AssertErrorMessages (errors,
var errorMessagePrefixes = new string []
{
$"/usr/bin/codesign exited with code 1:\n" +
$"{appPath}: replacing existing signature\n" +
$"{appPath}: code object is not signed at all\n" +
$"In subcomponent: {appPath}/System.Diagnostics.DiagnosticSource.dll",
$"{appPath}: code object is not signed at all\n",

$"Failed to codesign '{appPath}': {appPath}: replacing existing signature\n" +
$"{appPath}: code object is not signed at all\n" +
$"In subcomponent: {appPath}/System.Diagnostics.DiagnosticSource.dll"
$"{appPath}: code object is not signed at all\n",
};

AssertErrorMessages (errors,
errorMessagePrefixes.Select (prefix => new Func<string, bool> ((msg) => msg.StartsWith (prefix))).ToArray (),
errorMessagePrefixes.Select (prefix => new Func<string> (() => prefix)).ToArray ()
);

// Remove the dir, and now the build should succeed again.
Expand Down
44 changes: 36 additions & 8 deletions tests/dotnet/UnitTests/TestBaseClass.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,20 +445,48 @@ public static void AssertErrorMessages (IList<BuildLogEvent> actualErrors, param
AssertBuildMessages ("error", actualErrors, expectedErrorMessages);
}

public static void AssertErrorMessages (IList<BuildLogEvent> actualErrors, Func<string, bool> [] matchesExpectedErrorMessage, Func<string> [] rendersExpectedErrorMessage)
{
AssertBuildMessages ("error", actualErrors, matchesExpectedErrorMessage, rendersExpectedErrorMessage);
}

public static void AssertBuildMessages (string type, IList<BuildLogEvent> actualMessages, params string [] expectedMessages)
{
if (actualMessages.Count != expectedMessages.Length) {
Assert.Fail ($"Expected {expectedMessages.Length} {type}s, got {actualMessages.Count} {type}s:\n\t{string.Join ("\n\t", actualMessages.Select (v => v.Message?.TrimEnd ()))}");
AssertBuildMessages (type, actualMessages,
expectedMessages.Select (v => new Func<string, bool> ((msg) => msg == v)).ToArray (),
expectedMessages.Select (v => new Func<string> (() => v)).ToArray ()
);
}

static string makeSingleLine (string? msg)
{
if (msg is null)
return "";
return msg.TrimEnd ().Replace ("\n", "\\n").Replace ("\r", "\\r");
}

public static void AssertBuildMessages (string type, IList<BuildLogEvent> actualMessages, Func<string, bool> [] matchesExpectedMessage, Func<string> [] rendersExpectedMessage)
{
var expectedCount = matchesExpectedMessage.Length;
if (expectedCount != rendersExpectedMessage.Length)
throw new InvalidOperationException ($"Mismatched function count");

if (actualMessages.Count != expectedCount) {
Assert.Fail ($"Expected {expectedCount} {type}(s), got {actualMessages.Count} {type}(s)\n" +
$"\tExpected:\n" +
$"\t\t{string.Join ("\n\t\t", rendersExpectedMessage.Select (v => makeSingleLine (v ())))}" +
$"\tActual:\n" +
$"\t\t{string.Join ("\n\t\t", actualMessages.Select (v => makeSingleLine (v.Message)))}");
return;
}

var failures = new List<string> ();
for (var i = 0; i < expectedMessages.Length; i++) {
var actual = (actualMessages [i].Message ?? string.Empty).Trim ('\n', '\r', ' ');
var expected = expectedMessages [i].Trim ('\n', '\r', ' ');
if (actual != expected) {
actual = actual.Replace ("\n", "\\n").Replace ("\r", "\\r");
expected = expected.Replace ("\n", "\\n").Replace ("\r", "\\r");
for (var i = 0; i < expectedCount; i++) {
var actual = actualMessages [i].Message ?? string.Empty;
var isExpected = matchesExpectedMessage [i];
if (!isExpected (actual)) {
actual = makeSingleLine (actual);
var expected = makeSingleLine (rendersExpectedMessage [i] ());
failures.Add ($"\tUnexpected {type} message #{i}:\n\t\tExpected: {expected}\n\t\tActual: {actual}");
}
}
Expand Down

0 comments on commit 7da0418

Please sign in to comment.