From 16f48b8aa7a7b9ab8068fc2eb9ce24bf50e5a58b Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Tue, 6 Oct 2020 10:58:25 -0500 Subject: [PATCH] Fix matrix generation for duplicated platforms (#667) --- .../Commands/GenerateBuildMatrixCommand.cs | 27 +++--- .../src/ViewModel/PlatformInfo.cs | 8 +- .../tests/GenerateBuildMatrixCommandTests.cs | 84 +++++++++++++++++-- 3 files changed, 100 insertions(+), 19 deletions(-) diff --git a/src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateBuildMatrixCommand.cs b/src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateBuildMatrixCommand.cs index 90e69b04..dda04cf3 100644 --- a/src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateBuildMatrixCommand.cs +++ b/src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateBuildMatrixCommand.cs @@ -47,8 +47,8 @@ public override Task ExecuteAsync() return Task.CompletedTask; } - private static IEnumerable> ConsolidateSubgraphsWithCommonRootDockerfile( - IEnumerable> subgraphs) + private static IEnumerable> ConsolidateSubgraphsWithCommonRoot( + IEnumerable> subgraphs, Func getKey) { List> subGraphsList = subgraphs.Select(subgraph => subgraph.ToList()).ToList(); Dictionary> subgraphsByRootDockerfilePath = new Dictionary>(); @@ -57,15 +57,16 @@ private static IEnumerable> ConsolidateSubgraphsWithCo foreach (List subgraph in subGraphsList) { PlatformInfo rootPlatform = subgraph.First(); + string key = getKey(rootPlatform); - if (subgraphsByRootDockerfilePath.TryGetValue(rootPlatform.DockerfilePath, out List commonSubgraph)) + if (subgraphsByRootDockerfilePath.TryGetValue(key, out List commonSubgraph)) { commonSubgraph.AddRange(subgraph); subgraphsToDelete.Add(subgraph); } else { - subgraphsByRootDockerfilePath.Add(rootPlatform.DockerfilePath, subgraph); + subgraphsByRootDockerfilePath.Add(key, subgraph); } } @@ -82,7 +83,7 @@ private void AddDockerfilePathLegs( platform => GetPlatformDependencies(platform, platformGrouping)); // Pass 2: Combine subgraphs that have a common Dockerfile path for the root image - subgraphs = ConsolidateSubgraphsWithCommonRootDockerfile(subgraphs); + subgraphs = ConsolidateSubgraphsWithCommonRoot(subgraphs, platform => platform.DockerfilePath); // Pass 3: Find dependencies amongst the subgraphs that result from custom leg groups // to produce a new set of subgraphs. @@ -223,19 +224,24 @@ private static string GetNormalizedOsVersion(string osVersion) => .Replace("windowsservercore", "windows") .Replace("ltsc2019", "1809"); - private void AddVersionedOsLegs(BuildMatrixInfo matrix, IGrouping platformGrouping) + private void AddVersionedOsLegs(BuildMatrixInfo matrix, + IGrouping platformGrouping) { - // Get the set of subgraphs grouped by their FROM dependencies as well as any integral custom leg dependencies. + // Pass 1: Get the set of subgraphs grouped by their FROM dependencies as well as any integral custom leg dependencies. IEnumerable> subgraphs = platformGrouping .GetCompleteSubgraphs(platform => GetPlatformDependencies(platform, platformGrouping) .Union(GetCustomLegGroupPlatforms(platform, CustomBuildLegDependencyType.Integral))); - // Append any supplemental custom leg dependencies to each subgraph + // Pass 2: Combine subgraphs that have matching roots. This combines any duplicated platforms into a single subgraph. + subgraphs = ConsolidateSubgraphsWithCommonRoot(subgraphs, + platform => platform.GetUniqueKey(Manifest.GetImageByPlatform(platform))); + + // Pass 3: Append any supplemental custom leg dependencies to each subgraph subgraphs = subgraphs .Select(subgraph => subgraph.Union(subgraph.SelectMany(platform => GetCustomLegGroupPlatforms(platform, CustomBuildLegDependencyType.Supplemental)))); - // Append the parent graph of each platform to each respective subgraph + // Pass 4: Append the parent graph of each platform to each respective subgraph subgraphs = subgraphs.GetCompleteSubgraphs( subgraph => subgraph.Select(platform => GetParents(platform, platformGrouping))) .Select(set => set @@ -331,7 +337,8 @@ public IEnumerable GenerateMatrixInfo() .GroupBy(platform => new PlatformId() { OS = platform.Model.OS, - OsVersion = platform.Model.OS == OS.Linux ? null : GetNormalizedOsVersion(platform.BaseOsVersion), + OsVersion = platform.Model.OS == OS.Linux ? + null : GetNormalizedOsVersion(platform.BaseOsVersion), Architecture = platform.Model.Architecture, Variant = platform.Model.Variant }) diff --git a/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs b/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs index 20db9901..1267eda7 100644 --- a/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs +++ b/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs @@ -211,10 +211,10 @@ public string GetOSDisplayName() } public static bool AreMatchingPlatforms(ImageInfo image1, PlatformInfo platform1, ImageInfo image2, PlatformInfo platform2) => - platform1.DockerfilePath == platform2.DockerfilePath && - platform1.Model.OsVersion == platform2.Model.OsVersion && - platform1.Model.Architecture == platform2.Model.Architecture && - image1.ProductVersion == image2.ProductVersion; + platform1.GetUniqueKey(image1) == platform2.GetUniqueKey(image2); + + public string GetUniqueKey(ImageInfo parentImageInfo) => + $"{DockerfilePathRelativeToManifest}-{Model.OS}-{Model.OsVersion}-{Model.Architecture}-{parentImageInfo.ProductVersion}"; private static bool IsStageReference(string fromImage, IList fromMatches) { diff --git a/src/Microsoft.DotNet.ImageBuilder/tests/GenerateBuildMatrixCommandTests.cs b/src/Microsoft.DotNet.ImageBuilder/tests/GenerateBuildMatrixCommandTests.cs index 20e729a2..875dbb5d 100644 --- a/src/Microsoft.DotNet.ImageBuilder/tests/GenerateBuildMatrixCommandTests.cs +++ b/src/Microsoft.DotNet.ImageBuilder/tests/GenerateBuildMatrixCommandTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -694,13 +695,15 @@ public void PlatformDependencyGraph_CrossReferencedDockerfileFromMultipleRepos_S Assert.Equal($"--path {runtimeDepsRelativeDir}", imageBuilderPaths); } - [Fact] - public void PlatformDependencyGraph_CrossReferencedDockerfileFromMultipleRepos_ImageGraph() + [Theory] + [InlineData(MatrixType.PlatformVersionedOs)] + [InlineData(MatrixType.PlatformDependencyGraph)] + public void CrossReferencedDockerfileFromMultipleRepos_ImageGraph(MatrixType matrixType) { TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); GenerateBuildMatrixCommand command = new GenerateBuildMatrixCommand(); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); - command.Options.MatrixType = MatrixType.PlatformDependencyGraph; + command.Options.MatrixType = matrixType; command.Options.ProductVersionComponents = 2; Manifest manifest = CreateManifest( @@ -744,16 +747,87 @@ public void PlatformDependencyGraph_CrossReferencedDockerfileFromMultipleRepos_I File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest)); + command.LoadManifest(); + IEnumerable matrixInfos = command.GenerateMatrixInfo(); + Assert.Single(matrixInfos); + BuildMatrixInfo matrixInfo = matrixInfos.First(); + + if (matrixType == MatrixType.PlatformDependencyGraph) + { + Assert.Single(matrixInfo.Legs); + + Assert.Equal("3.1-runtime-deps-os-Dockerfile-graph", matrixInfo.Legs[0].Name); + string imageBuilderPaths = matrixInfo.Legs[0].Variables.First(variable => variable.Name == "imageBuilderPaths").Value; + Assert.Equal($"--path 3.1/runtime-deps/os/Dockerfile --path 3.1/runtime/os/Dockerfile --path 5.0/runtime/os/Dockerfile", imageBuilderPaths); + } + else + { + Assert.Equal(2, matrixInfo.Legs.Count); + + Assert.Equal("3.1-disco", matrixInfo.Legs[0].Name); + string imageBuilderPaths = matrixInfo.Legs[0].Variables.First(variable => variable.Name == "imageBuilderPaths").Value; + Assert.Equal($"--path 3.1/runtime-deps/os/Dockerfile --path 3.1/runtime/os/Dockerfile", imageBuilderPaths); + + Assert.Equal("5.0-disco", matrixInfo.Legs[1].Name); + imageBuilderPaths = matrixInfo.Legs[1].Variables.First(variable => variable.Name == "imageBuilderPaths").Value; + Assert.Equal($"--path 3.1/runtime-deps/os/Dockerfile --path 5.0/runtime/os/Dockerfile", imageBuilderPaths); + } + } + + [Theory] + [InlineData(MatrixType.PlatformVersionedOs)] + [InlineData(MatrixType.PlatformDependencyGraph)] + public void DuplicatedPlatforms(MatrixType matrixType) + { + TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); + GenerateBuildMatrixCommand command = new GenerateBuildMatrixCommand(); + command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); + command.Options.MatrixType = matrixType; + command.Options.ProductVersionComponents = 2; + + Manifest manifest = CreateManifest( + CreateRepo("runtime", + CreateImage( + new Platform[] + { + CreatePlatform( + DockerfileHelper.CreateDockerfile("3.1/runtime/os", tempFolderContext), + new string[] { "tag" }) + }, + productVersion: "3.1"), + CreateImage( + new Platform[] + { + CreatePlatform( + DockerfileHelper.CreateDockerfile("3.1/runtime/os", tempFolderContext), + Array.Empty()) + }, + productVersion: "3.1"))); + + File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest)); + command.LoadManifest(); IEnumerable matrixInfos = command.GenerateMatrixInfo(); Assert.Single(matrixInfos); BuildMatrixInfo matrixInfo = matrixInfos.First(); + Assert.Single(matrixInfo.Legs); - Assert.Equal("3.1-runtime-deps-os-Dockerfile-graph", matrixInfo.Legs[0].Name); + string expectedLegName; + if (matrixType == MatrixType.PlatformDependencyGraph) + { + expectedLegName = "3.1-runtime-os-Dockerfile-graph"; + } + else + { + expectedLegName = "3.1-disco"; + } + + Assert.Equal(expectedLegName, matrixInfo.Legs[0].Name); + string imageBuilderPaths = matrixInfo.Legs[0].Variables.First(variable => variable.Name == "imageBuilderPaths").Value; - Assert.Equal($"--path 3.1/runtime-deps/os/Dockerfile --path 3.1/runtime/os/Dockerfile --path 5.0/runtime/os/Dockerfile", imageBuilderPaths); + Assert.Equal($"--path 3.1/runtime/os/Dockerfile", imageBuilderPaths); } private static PlatformData CreateSimplePlatformData(string dockerfilePath, bool isCached = false)