Skip to content

Commit

Permalink
Fix matrix generation for duplicated platforms (#667)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mthalman
mthalman authored Oct 6, 2020
1 parent f31211a commit 16f48b8
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 19 deletions.
27 changes: 17 additions & 10 deletions src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateBuildMatrixCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ public override Task ExecuteAsync()
return Task.CompletedTask;
}

private static IEnumerable<IEnumerable<PlatformInfo>> ConsolidateSubgraphsWithCommonRootDockerfile(
IEnumerable<IEnumerable<PlatformInfo>> subgraphs)
private static IEnumerable<IEnumerable<PlatformInfo>> ConsolidateSubgraphsWithCommonRoot(
IEnumerable<IEnumerable<PlatformInfo>> subgraphs, Func<PlatformInfo, string> getKey)
{
List<List<PlatformInfo>> subGraphsList = subgraphs.Select(subgraph => subgraph.ToList()).ToList();
Dictionary<string, List<PlatformInfo>> subgraphsByRootDockerfilePath = new Dictionary<string, List<PlatformInfo>>();
Expand All @@ -57,15 +57,16 @@ private static IEnumerable<IEnumerable<PlatformInfo>> ConsolidateSubgraphsWithCo
foreach (List<PlatformInfo> subgraph in subGraphsList)
{
PlatformInfo rootPlatform = subgraph.First();
string key = getKey(rootPlatform);

if (subgraphsByRootDockerfilePath.TryGetValue(rootPlatform.DockerfilePath, out List<PlatformInfo> commonSubgraph))
if (subgraphsByRootDockerfilePath.TryGetValue(key, out List<PlatformInfo> commonSubgraph))
{
commonSubgraph.AddRange(subgraph);
subgraphsToDelete.Add(subgraph);
}
else
{
subgraphsByRootDockerfilePath.Add(rootPlatform.DockerfilePath, subgraph);
subgraphsByRootDockerfilePath.Add(key, subgraph);
}
}

Expand All @@ -82,7 +83,7 @@ private void AddDockerfilePathLegs(
platform => GetPlatformDependencies(platform, platformGrouping));

// Pass 2: Combine subgraphs that have a common Dockerfile path for the root image
subgraphs = ConsolidateSubgraphsWithCommonRootDockerfile(subgraphs);
subgraphs = ConsolidateSubgraphsWithCommonRoot(subgraphs, platform => platform.DockerfilePath);

// Pass 3: Find dependencies amongst the subgraphs that result from custom leg groups
// to produce a new set of subgraphs.
Expand Down Expand Up @@ -223,19 +224,24 @@ private static string GetNormalizedOsVersion(string osVersion) =>
.Replace("windowsservercore", "windows")
.Replace("ltsc2019", "1809");

private void AddVersionedOsLegs(BuildMatrixInfo matrix, IGrouping<PlatformId, PlatformInfo> platformGrouping)
private void AddVersionedOsLegs(BuildMatrixInfo matrix,
IGrouping<PlatformId, PlatformInfo> platformGrouping)
{
// Get the set of subgraphs grouped by their FROM dependencies as well as any integral custom leg dependencies.
// Pass 1: Get the set of subgraphs grouped by their FROM dependencies as well as any integral custom leg dependencies.
IEnumerable<IEnumerable<PlatformInfo>> subgraphs = platformGrouping
.GetCompleteSubgraphs(platform =>
GetPlatformDependencies(platform, platformGrouping)
.Union(GetCustomLegGroupPlatforms(platform, CustomBuildLegDependencyType.Integral)));

// Append any supplemental custom leg dependencies to each subgraph
// Pass 2: Combine subgraphs that have matching roots. This combines any duplicated platforms into a single subgraph.
subgraphs = ConsolidateSubgraphsWithCommonRoot(subgraphs,
platform => platform.GetUniqueKey(Manifest.GetImageByPlatform(platform)));

// Pass 3: Append any supplemental custom leg dependencies to each subgraph
subgraphs = subgraphs
.Select(subgraph => subgraph.Union(subgraph.SelectMany(platform => GetCustomLegGroupPlatforms(platform, CustomBuildLegDependencyType.Supplemental))));

// Append the parent graph of each platform to each respective subgraph
// Pass 4: Append the parent graph of each platform to each respective subgraph
subgraphs = subgraphs.GetCompleteSubgraphs(
subgraph => subgraph.Select(platform => GetParents(platform, platformGrouping)))
.Select(set => set
Expand Down Expand Up @@ -331,7 +337,8 @@ public IEnumerable<BuildMatrixInfo> GenerateMatrixInfo()
.GroupBy(platform => new PlatformId()
{
OS = platform.Model.OS,
OsVersion = platform.Model.OS == OS.Linux ? null : GetNormalizedOsVersion(platform.BaseOsVersion),
OsVersion = platform.Model.OS == OS.Linux ?
null : GetNormalizedOsVersion(platform.BaseOsVersion),
Architecture = platform.Model.Architecture,
Variant = platform.Model.Variant
})
Expand Down
8 changes: 4 additions & 4 deletions src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,10 +211,10 @@ public string GetOSDisplayName()
}

public static bool AreMatchingPlatforms(ImageInfo image1, PlatformInfo platform1, ImageInfo image2, PlatformInfo platform2) =>
platform1.DockerfilePath == platform2.DockerfilePath &&
platform1.Model.OsVersion == platform2.Model.OsVersion &&
platform1.Model.Architecture == platform2.Model.Architecture &&
image1.ProductVersion == image2.ProductVersion;
platform1.GetUniqueKey(image1) == platform2.GetUniqueKey(image2);

public string GetUniqueKey(ImageInfo parentImageInfo) =>
$"{DockerfilePathRelativeToManifest}-{Model.OS}-{Model.OsVersion}-{Model.Architecture}-{parentImageInfo.ProductVersion}";

private static bool IsStageReference(string fromImage, IList<Match> fromMatches)
{
Expand Down
84 changes: 79 additions & 5 deletions src/Microsoft.DotNet.ImageBuilder/tests/GenerateBuildMatrixCommandTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
Expand Down Expand Up @@ -694,13 +695,15 @@ public void PlatformDependencyGraph_CrossReferencedDockerfileFromMultipleRepos_S
Assert.Equal($"--path {runtimeDepsRelativeDir}", imageBuilderPaths);
}

[Fact]
public void PlatformDependencyGraph_CrossReferencedDockerfileFromMultipleRepos_ImageGraph()
[Theory]
[InlineData(MatrixType.PlatformVersionedOs)]
[InlineData(MatrixType.PlatformDependencyGraph)]
public void CrossReferencedDockerfileFromMultipleRepos_ImageGraph(MatrixType matrixType)
{
TempFolderContext tempFolderContext = TestHelper.UseTempFolder();
GenerateBuildMatrixCommand command = new GenerateBuildMatrixCommand();
command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json");
command.Options.MatrixType = MatrixType.PlatformDependencyGraph;
command.Options.MatrixType = matrixType;
command.Options.ProductVersionComponents = 2;

Manifest manifest = CreateManifest(
Expand Down Expand Up @@ -744,16 +747,87 @@ public void PlatformDependencyGraph_CrossReferencedDockerfileFromMultipleRepos_I

File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest));

command.LoadManifest();
IEnumerable<BuildMatrixInfo> matrixInfos = command.GenerateMatrixInfo();
Assert.Single(matrixInfos);
BuildMatrixInfo matrixInfo = matrixInfos.First();

if (matrixType == MatrixType.PlatformDependencyGraph)
{
Assert.Single(matrixInfo.Legs);

Assert.Equal("3.1-runtime-deps-os-Dockerfile-graph", matrixInfo.Legs[0].Name);
string imageBuilderPaths = matrixInfo.Legs[0].Variables.First(variable => variable.Name == "imageBuilderPaths").Value;
Assert.Equal($"--path 3.1/runtime-deps/os/Dockerfile --path 3.1/runtime/os/Dockerfile --path 5.0/runtime/os/Dockerfile", imageBuilderPaths);
}
else
{
Assert.Equal(2, matrixInfo.Legs.Count);

Assert.Equal("3.1-disco", matrixInfo.Legs[0].Name);
string imageBuilderPaths = matrixInfo.Legs[0].Variables.First(variable => variable.Name == "imageBuilderPaths").Value;
Assert.Equal($"--path 3.1/runtime-deps/os/Dockerfile --path 3.1/runtime/os/Dockerfile", imageBuilderPaths);

Assert.Equal("5.0-disco", matrixInfo.Legs[1].Name);
imageBuilderPaths = matrixInfo.Legs[1].Variables.First(variable => variable.Name == "imageBuilderPaths").Value;
Assert.Equal($"--path 3.1/runtime-deps/os/Dockerfile --path 5.0/runtime/os/Dockerfile", imageBuilderPaths);
}
}

[Theory]
[InlineData(MatrixType.PlatformVersionedOs)]
[InlineData(MatrixType.PlatformDependencyGraph)]
public void DuplicatedPlatforms(MatrixType matrixType)
{
TempFolderContext tempFolderContext = TestHelper.UseTempFolder();
GenerateBuildMatrixCommand command = new GenerateBuildMatrixCommand();
command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json");
command.Options.MatrixType = matrixType;
command.Options.ProductVersionComponents = 2;

Manifest manifest = CreateManifest(
CreateRepo("runtime",
CreateImage(
new Platform[]
{
CreatePlatform(
DockerfileHelper.CreateDockerfile("3.1/runtime/os", tempFolderContext),
new string[] { "tag" })
},
productVersion: "3.1"),
CreateImage(
new Platform[]
{
CreatePlatform(
DockerfileHelper.CreateDockerfile("3.1/runtime/os", tempFolderContext),
Array.Empty<string>())
},
productVersion: "3.1")));

File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest));

command.LoadManifest();
IEnumerable<BuildMatrixInfo> matrixInfos = command.GenerateMatrixInfo();
Assert.Single(matrixInfos);

BuildMatrixInfo matrixInfo = matrixInfos.First();

Assert.Single(matrixInfo.Legs);

Assert.Equal("3.1-runtime-deps-os-Dockerfile-graph", matrixInfo.Legs[0].Name);
string expectedLegName;
if (matrixType == MatrixType.PlatformDependencyGraph)
{
expectedLegName = "3.1-runtime-os-Dockerfile-graph";
}
else
{
expectedLegName = "3.1-disco";
}

Assert.Equal(expectedLegName, matrixInfo.Legs[0].Name);

string imageBuilderPaths = matrixInfo.Legs[0].Variables.First(variable => variable.Name == "imageBuilderPaths").Value;
Assert.Equal($"--path 3.1/runtime-deps/os/Dockerfile --path 3.1/runtime/os/Dockerfile --path 5.0/runtime/os/Dockerfile", imageBuilderPaths);
Assert.Equal($"--path 3.1/runtime/os/Dockerfile", imageBuilderPaths);
}

private static PlatformData CreateSimplePlatformData(string dockerfilePath, bool isCached = false)
Expand Down

0 comments on commit 16f48b8

Please sign in to comment.