From f31211ad4a4eefe2cb71908e7fca5e34eb3f1e1e Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Fri, 2 Oct 2020 10:19:04 -0500 Subject: [PATCH] Manifest publishing support for duplicated platforms (#666) --- .../src/Commands/BuildCommand.cs | 3 +- .../src/Commands/PublishManifestCommand.cs | 40 ++++- .../src/McrTagsMetadataGenerator.cs | 5 +- .../src/Models/Image/PlatformData.cs | 22 +-- .../src/ViewModel/PlatformInfo.cs | 6 + .../tests/ImageInfoHelperTests.cs | 105 ++++++++++++- .../tests/MergeImageInfoFilesCommandTests.cs | 16 +- .../tests/PublishImageInfoCommandTests.cs | 4 +- .../tests/PublishManifestCommandTests.cs | 147 ++++++++++++++++++ 9 files changed, 316 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.DotNet.ImageBuilder/src/Commands/BuildCommand.cs b/src/Microsoft.DotNet.ImageBuilder/src/Commands/BuildCommand.cs index 61c49723..b7beaca0 100644 --- a/src/Microsoft.DotNet.ImageBuilder/src/Commands/BuildCommand.cs +++ b/src/Microsoft.DotNet.ImageBuilder/src/Commands/BuildCommand.cs @@ -101,8 +101,9 @@ private void PublishImageInfo() } SetPlatformDataCreatedDate(platform, tag.FullyQualifiedName); - platform.CommitUrl = _gitService.GetDockerfileCommitUrl(platform.PlatformInfo, Options.SourceRepoUrl); } + + platform.CommitUrl = _gitService.GetDockerfileCommitUrl(platform.PlatformInfo, Options.SourceRepoUrl); } string imageInfoString = JsonHelper.SerializeObject(_imageArtifactDetails); diff --git a/src/Microsoft.DotNet.ImageBuilder/src/Commands/PublishManifestCommand.cs b/src/Microsoft.DotNet.ImageBuilder/src/Commands/PublishManifestCommand.cs index 9d884c54..df18a728 100644 --- a/src/Microsoft.DotNet.ImageBuilder/src/Commands/PublishManifestCommand.cs +++ b/src/Microsoft.DotNet.ImageBuilder/src/Commands/PublishManifestCommand.cs @@ -37,14 +37,17 @@ public override Task ExecuteAsync() ExecuteWithUser(() => { - IEnumerable multiArchImages = Manifest.GetFilteredImages() - .Where(image => image.SharedTags.Any()) + IEnumerable<(RepoInfo Repo, ImageInfo Image)> multiArchRepoImages = Manifest.FilteredRepos + .SelectMany(repo => + repo.FilteredImages + .Where(image => image.SharedTags.Any()) + .Select(image => (repo, image))) .ToList(); DateTime createdDate = DateTime.Now.ToUniversalTime(); - Parallel.ForEach(multiArchImages, image => + Parallel.ForEach(multiArchRepoImages, repoImage => { - string manifest = GenerateManifest(image); + string manifest = GenerateManifest(repoImage.Repo, repoImage.Image); string manifestFilename = $"manifest.{Guid.NewGuid()}.yml"; _loggerService.WriteSubheading($"PUBLISHING MANIFEST: '{manifestFilename}'{Environment.NewLine}{manifest}"); @@ -91,7 +94,7 @@ private void SaveTagInfoToImageInfoFile(DateTime createdDate, ImageArtifactDetai File.WriteAllText(Options.ImageInfoPath, imageInfoString); } - private string GenerateManifest(ImageInfo image) + private string GenerateManifest(RepoInfo repo, ImageInfo image) { StringBuilder manifestYml = new StringBuilder(); manifestYml.AppendLine($"image: {image.SharedTags.First().FullyQualifiedName}"); @@ -107,7 +110,32 @@ private string GenerateManifest(ImageInfo image) manifestYml.AppendLine("manifests:"); foreach (PlatformInfo platform in image.AllPlatforms) { - manifestYml.AppendLine($"- image: {platform.Tags.First().FullyQualifiedName}"); + string imageTag; + if (platform.Tags.Any()) + { + imageTag = platform.Tags.First().FullyQualifiedName; + } + else + { + (ImageInfo Image, PlatformInfo Platform) matchingImagePlatform = repo.AllImages + .SelectMany(image => + image.AllPlatforms + .Select(p => (Image: image, Platform: p)) + .Where(imagePlatform => platform != imagePlatform.Platform && + PlatformInfo.AreMatchingPlatforms(image, platform, imagePlatform.Image, imagePlatform.Platform) && + imagePlatform.Platform.Tags.Any())) + .FirstOrDefault(); + + if (matchingImagePlatform.Platform is null) + { + throw new InvalidOperationException( + $"Could not find a platform with concrete tags for '{platform.DockerfilePathRelativeToManifest}'."); + } + + imageTag = matchingImagePlatform.Platform.Tags.First().FullyQualifiedName; + } + + manifestYml.AppendLine($"- image: {imageTag}"); manifestYml.AppendLine($" platform:"); manifestYml.AppendLine($" architecture: {platform.Model.Architecture.GetDockerName()}"); manifestYml.AppendLine($" os: {platform.Model.OS.GetDockerName()}"); diff --git a/src/Microsoft.DotNet.ImageBuilder/src/McrTagsMetadataGenerator.cs b/src/Microsoft.DotNet.ImageBuilder/src/McrTagsMetadataGenerator.cs index d087a0ab..8b4a0dd1 100644 --- a/src/Microsoft.DotNet.ImageBuilder/src/McrTagsMetadataGenerator.cs +++ b/src/Microsoft.DotNet.ImageBuilder/src/McrTagsMetadataGenerator.cs @@ -140,10 +140,7 @@ private string GetVariableValue(string variableType, string variableName) // duplicated in another image in order to associate it within a distinct set of shared tags. IEnumerable matchingDocInfos = _imageDocInfos .Where(docInfo => docInfo != info && - docInfo.Platform.DockerfilePath == info.Platform.DockerfilePath && - docInfo.Platform.Model.OsVersion == info.Platform.Model.OsVersion && - docInfo.Platform.Model.Architecture == info.Platform.Model.Architecture && - docInfo.Image.ProductVersion == info.Image.ProductVersion) + PlatformInfo.AreMatchingPlatforms(docInfo.Image, docInfo.Platform, info.Image, info.Platform)) .Prepend(info) .ToArray(); diff --git a/src/Microsoft.DotNet.ImageBuilder/src/Models/Image/PlatformData.cs b/src/Microsoft.DotNet.ImageBuilder/src/Models/Image/PlatformData.cs index ea1235c2..d4ce3c32 100644 --- a/src/Microsoft.DotNet.ImageBuilder/src/Models/Image/PlatformData.cs +++ b/src/Microsoft.DotNet.ImageBuilder/src/Models/Image/PlatformData.cs @@ -69,25 +69,29 @@ public int CompareTo([AllowNull] PlatformData other) return 1; } + // If either of the platforms has no simple tags while the other does have simple tags, they are not equal + if ((SimpleTags?.Count == 0 && other.SimpleTags?.Count > 0) || + (SimpleTags?.Count > 0 && other.SimpleTags?.Count == 0)) + { + return 1; + } + return GetIdentifier().CompareTo(other.GetIdentifier()); } - public bool Equals(PlatformInfo platformInfo) - { - return GetIdentifier() == FromPlatformInfo(platformInfo, null).GetIdentifier(); - } + public bool Equals(PlatformInfo platformInfo) => + CompareTo(FromPlatformInfo(platformInfo, null)) == 0; public string GetIdentifier() => $"{Dockerfile}-{Architecture}-{OsType}-{OsVersion}"; - public static PlatformData FromPlatformInfo(PlatformInfo platform, ImageInfo image) - { - return new PlatformData(image, platform) + public static PlatformData FromPlatformInfo(PlatformInfo platform, ImageInfo image) => + new PlatformData(image, platform) { Dockerfile = platform.DockerfilePathRelativeToManifest, Architecture = platform.Model.Architecture.GetDisplayName(), OsType = platform.Model.OS.ToString(), - OsVersion = platform.GetOSDisplayName() + OsVersion = platform.GetOSDisplayName(), + SimpleTags = platform.Tags.Select(tag => tag.Name).ToList() }; - } } } diff --git a/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs b/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs index 59bba15e..20db9901 100644 --- a/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs +++ b/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs @@ -210,6 +210,12 @@ public string GetOSDisplayName() return displayName; } + public static bool AreMatchingPlatforms(ImageInfo image1, PlatformInfo platform1, ImageInfo image2, PlatformInfo platform2) => + platform1.DockerfilePath == platform2.DockerfilePath && + platform1.Model.OsVersion == platform2.Model.OsVersion && + platform1.Model.Architecture == platform2.Model.Architecture && + image1.ProductVersion == image2.ProductVersion; + private static bool IsStageReference(string fromImage, IList fromMatches) { bool isStageReference = false; diff --git a/src/Microsoft.DotNet.ImageBuilder/tests/ImageInfoHelperTests.cs b/src/Microsoft.DotNet.ImageBuilder/tests/ImageInfoHelperTests.cs index 55ba5140..d77b73cd 100644 --- a/src/Microsoft.DotNet.ImageBuilder/tests/ImageInfoHelperTests.cs +++ b/src/Microsoft.DotNet.ImageBuilder/tests/ImageInfoHelperTests.cs @@ -41,14 +41,16 @@ public void LoadFromContent() Dockerfile = "1.0/runtime/linux/Dockerfile", OsType = "Linux", OsVersion = "Ubuntu 20.04", - Architecture = "amd64" + Architecture = "amd64", + SimpleTags = new List { "linux" } }, new PlatformData { Dockerfile = "1.0/runtime/windows/Dockerfile", OsType = "Windows", OsVersion = "Windows Server, version 2004", - Architecture = "amd64" + Architecture = "amd64", + SimpleTags = new List { "windows" } } }, Manifest = new ManifestData @@ -668,6 +670,105 @@ public void ImageInfoHelper_MergeRepos_RemoveTag() CompareImageArtifactDetails(expected, targetImageArtifactDetails); } + [Fact] + public void Merge_DuplicatedPlatforms() + { + ImageArtifactDetails imageArtifactDetails = new ImageArtifactDetails + { + Repos = + { + new RepoData + { + Repo = "repo", + Images = + { + new ImageData + { + ManifestImage = CreateImageInfo(), + Platforms = + { + new PlatformData + { + Dockerfile = "image1" + } + } + } + } + } + } + }; + + ImageArtifactDetails targetImageArtifactDetails = new ImageArtifactDetails + { + Repos = + { + new RepoData + { + Repo = "repo", + Images = + { + new ImageData + { + ManifestImage = CreateImageInfo(), + Platforms = + { + new PlatformData + { + Dockerfile = "image1", + SimpleTags = new List + { + "tag1" + } + } + } + } + } + } + } + }; + + ImageArtifactDetails expectedImageArtifactDetails = new ImageArtifactDetails + { + Repos = + { + new RepoData + { + Repo = "repo", + Images = + { + new ImageData + { + Platforms = + { + new PlatformData + { + Dockerfile = "image1" + } + } + }, + new ImageData + { + Platforms = + { + new PlatformData + { + Dockerfile = "image1", + SimpleTags = new List + { + "tag1" + } + } + } + }, + } + } + } + }; + + ImageInfoHelper.MergeImageArtifactDetails(imageArtifactDetails, targetImageArtifactDetails); + CompareImageArtifactDetails(expectedImageArtifactDetails, targetImageArtifactDetails); + } + public static void CompareImageArtifactDetails(ImageArtifactDetails expected, ImageArtifactDetails actual) { Assert.Equal(JsonHelper.SerializeObject(expected), JsonHelper.SerializeObject(actual)); diff --git a/src/Microsoft.DotNet.ImageBuilder/tests/MergeImageInfoFilesCommandTests.cs b/src/Microsoft.DotNet.ImageBuilder/tests/MergeImageInfoFilesCommandTests.cs index 21febff9..e7d32c8f 100644 --- a/src/Microsoft.DotNet.ImageBuilder/tests/MergeImageInfoFilesCommandTests.cs +++ b/src/Microsoft.DotNet.ImageBuilder/tests/MergeImageInfoFilesCommandTests.cs @@ -178,15 +178,15 @@ public async Task MergeImageInfoFilesCommand_HappyPath() CreateRepo("repo1"), CreateRepo("repo2", CreateImage( - CreatePlatform(repo2Image1Dockerfile, new string[0])), + CreatePlatform(repo2Image1Dockerfile, new string[] { "tag1" })), CreateImage( - CreatePlatform(repo2Image2Dockerfile, new string[0]))), + CreatePlatform(repo2Image2Dockerfile, new string[] { "tag2" }))), CreateRepo("repo3"), CreateRepo("repo4", CreateImage( - CreatePlatform(repo4Image2Dockerfile, new string[0])), + CreatePlatform(repo4Image2Dockerfile, new string[] { "tag1" })), CreateImage( - CreatePlatform(repo4Image3Dockerfile, new string[0]))) + CreatePlatform(repo4Image3Dockerfile, new string[] { "tag2" }))) ); File.WriteAllText(Path.Combine(context.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest)); @@ -456,12 +456,12 @@ public async Task MergeImageInfoFilesCommand_DuplicateDockerfilePaths() Manifest manifest = CreateManifest( CreateRepo("repo1", CreateImage( - CreatePlatform(dockerfile1, new string[0], osVersion: Os1, architecture: Architecture.ARM), - CreatePlatform(dockerfile1, new string[0], osVersion: Os1, architecture: Architecture.ARM64)), + CreatePlatform(dockerfile1, new string[] { "tag1" }, osVersion: Os1, architecture: Architecture.ARM), + CreatePlatform(dockerfile1, new string[] { "tag2" }, osVersion: Os1, architecture: Architecture.ARM64)), CreateImage( - CreatePlatform(dockerfile1, new string[0], osVersion: Os2, architecture: Architecture.ARM)), + CreatePlatform(dockerfile1, new string[] { "tag3" }, osVersion: Os2, architecture: Architecture.ARM)), CreateImage( - CreatePlatform(dockerfile2, new string[0], osVersion: Os1))) + CreatePlatform(dockerfile2, new string[] { "tag4" }, osVersion: Os1))) ); File.WriteAllText(Path.Combine(context.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest)); diff --git a/src/Microsoft.DotNet.ImageBuilder/tests/PublishImageInfoCommandTests.cs b/src/Microsoft.DotNet.ImageBuilder/tests/PublishImageInfoCommandTests.cs index b2ee8393..753fc8ee 100644 --- a/src/Microsoft.DotNet.ImageBuilder/tests/PublishImageInfoCommandTests.cs +++ b/src/Microsoft.DotNet.ImageBuilder/tests/PublishImageInfoCommandTests.cs @@ -47,10 +47,10 @@ public async Task PublishImageInfoCommand_ReplaceTags() Manifest manifest = CreateManifest( CreateRepo("repo1", CreateImage( - CreatePlatform(repo1Image1DockerfilePath, new string[0]))), + CreatePlatform(repo1Image1DockerfilePath, new string[] { "tag1" }))), CreateRepo("repo2", CreateImage( - CreatePlatform(repo2Image2DockerfilePath, new string[0]))) + CreatePlatform(repo2Image2DockerfilePath, new string[] { "tag1" }))) ); RepoData repo2; diff --git a/src/Microsoft.DotNet.ImageBuilder/tests/PublishManifestCommandTests.cs b/src/Microsoft.DotNet.ImageBuilder/tests/PublishManifestCommandTests.cs index 6cb30866..f0af0335 100644 --- a/src/Microsoft.DotNet.ImageBuilder/tests/PublishManifestCommandTests.cs +++ b/src/Microsoft.DotNet.ImageBuilder/tests/PublishManifestCommandTests.cs @@ -183,5 +183,152 @@ public async Task ImageInfoTagOutput() Assert.Equal(expectedOutput, actualOutput); } + + /// + /// Verifies a correct manifest is generated when there are duplicate platforms defined, with some not containing + /// concrete tags. + /// + [Fact] + public async Task DuplicatePlatform() + { + string expectedManifest1 = +@"image: repo1:sharedtag2 +tags: [sharedtag1] +manifests: +- image: repo1:tag1 + platform: + architecture: amd64 + os: linux +"; + + string expectedManifest2 = +@"image: repo1:sharedtag3 +manifests: +- image: repo1:tag1 + platform: + architecture: amd64 + os: linux +"; + + bool manifest1Found = false; + bool manifest2Found = false; + + Mock manifestToolService = new Mock(); + manifestToolService + .Setup(o => o.PushFromSpec(It.IsAny(), false)) + .Callback((string manifestFile, bool isDryRun) => + { + string manifestContents = File.ReadAllText(manifestFile); + + if (manifestContents == expectedManifest1) + { + manifest1Found = true; + } + else if (manifestContents == expectedManifest2) + { + manifest2Found = true; + } + }); + + manifestToolService + .Setup(o => o.Inspect(It.IsAny(), false)) + .Returns(ManifestToolServiceHelper.CreateTagManifest(ManifestToolService.ManifestListMediaType, "digest")); + + PublishManifestCommand command = new PublishManifestCommand( + manifestToolService.Object, Mock.Of()); + + using TempFolderContext tempFolderContext = new TempFolderContext(); + + command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); + command.Options.ImageInfoPath = Path.Combine(tempFolderContext.Path, "image-info.json"); + + string dockerfile = CreateDockerfile("1.0/repo1/os", tempFolderContext); + + ImageArtifactDetails imageArtifactDetails = new ImageArtifactDetails + { + Repos = + { + new RepoData + { + Repo = "repo1", + Images = + { + new ImageData + { + Platforms = + { + CreatePlatform(dockerfile, + simpleTags: new List + { + "tag1", + "tag2" + }) + }, + Manifest = new ManifestData + { + SharedTags = + { + "sharedtag1", + "sharedtag2" + } + } + }, + new ImageData + { + Platforms = + { + CreatePlatform(dockerfile) + }, + Manifest = new ManifestData + { + SharedTags = + { + "sharedtag3" + } + } + } + } + } + } + }; + File.WriteAllText(command.Options.ImageInfoPath, JsonHelper.SerializeObject(imageArtifactDetails)); + + Manifest manifest = CreateManifest( + CreateRepo("repo1", + CreateImage( + new Platform[] + { + CreatePlatform(dockerfile, + new string[] + { + "tag1", + "tag2" + }) + }, + new Dictionary + { + { "sharedtag2", new Tag() }, + { "sharedtag1", new Tag() } + }), + CreateImage( + new Platform[] + { + CreatePlatform(dockerfile, Array.Empty()) + }, + new Dictionary + { + { "sharedtag3", new Tag() } + })) + ); + File.WriteAllText(command.Options.Manifest, JsonHelper.SerializeObject(manifest)); + + command.LoadManifest(); + await command.ExecuteAsync(); + + Assert.True(manifest1Found); + Assert.True(manifest2Found); + manifestToolService + .Verify(o => o.PushFromSpec(It.IsAny(), false), Times.Exactly(2)); + } } }