Skip to content

Commit

Permalink
Manifest publishing support for duplicated platforms (#666)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mthalman
mthalman authored Oct 2, 2020
1 parent b7e183d commit f31211a
Show file tree
Hide file tree
Showing 9 changed files with 316 additions and 32 deletions.
3 changes: 2 additions & 1 deletion src/Microsoft.DotNet.ImageBuilder/src/Commands/BuildCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,9 @@ private void PublishImageInfo()
}

SetPlatformDataCreatedDate(platform, tag.FullyQualifiedName);
platform.CommitUrl = _gitService.GetDockerfileCommitUrl(platform.PlatformInfo, Options.SourceRepoUrl);
}

platform.CommitUrl = _gitService.GetDockerfileCommitUrl(platform.PlatformInfo, Options.SourceRepoUrl);
}

string imageInfoString = JsonHelper.SerializeObject(_imageArtifactDetails);
Expand Down
40 changes: 34 additions & 6 deletions src/Microsoft.DotNet.ImageBuilder/src/Commands/PublishManifestCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,17 @@ public override Task ExecuteAsync()

ExecuteWithUser(() =>
{
IEnumerable<ImageInfo> multiArchImages = Manifest.GetFilteredImages()
.Where(image => image.SharedTags.Any())
IEnumerable<(RepoInfo Repo, ImageInfo Image)> multiArchRepoImages = Manifest.FilteredRepos
.SelectMany(repo =>
repo.FilteredImages
.Where(image => image.SharedTags.Any())
.Select(image => (repo, image)))
.ToList();

DateTime createdDate = DateTime.Now.ToUniversalTime();
Parallel.ForEach(multiArchImages, image =>
Parallel.ForEach(multiArchRepoImages, repoImage =>
{
string manifest = GenerateManifest(image);
string manifest = GenerateManifest(repoImage.Repo, repoImage.Image);

string manifestFilename = $"manifest.{Guid.NewGuid()}.yml";
_loggerService.WriteSubheading($"PUBLISHING MANIFEST: '{manifestFilename}'{Environment.NewLine}{manifest}");
Expand Down Expand Up @@ -91,7 +94,7 @@ private void SaveTagInfoToImageInfoFile(DateTime createdDate, ImageArtifactDetai
File.WriteAllText(Options.ImageInfoPath, imageInfoString);
}

private string GenerateManifest(ImageInfo image)
private string GenerateManifest(RepoInfo repo, ImageInfo image)
{
StringBuilder manifestYml = new StringBuilder();
manifestYml.AppendLine($"image: {image.SharedTags.First().FullyQualifiedName}");
Expand All @@ -107,7 +110,32 @@ private string GenerateManifest(ImageInfo image)
manifestYml.AppendLine("manifests:");
foreach (PlatformInfo platform in image.AllPlatforms)
{
manifestYml.AppendLine($"- image: {platform.Tags.First().FullyQualifiedName}");
string imageTag;
if (platform.Tags.Any())
{
imageTag = platform.Tags.First().FullyQualifiedName;
}
else
{
(ImageInfo Image, PlatformInfo Platform) matchingImagePlatform = repo.AllImages
.SelectMany(image =>
image.AllPlatforms
.Select(p => (Image: image, Platform: p))
.Where(imagePlatform => platform != imagePlatform.Platform &&
PlatformInfo.AreMatchingPlatforms(image, platform, imagePlatform.Image, imagePlatform.Platform) &&
imagePlatform.Platform.Tags.Any()))
.FirstOrDefault();

if (matchingImagePlatform.Platform is null)
{
throw new InvalidOperationException(
$"Could not find a platform with concrete tags for '{platform.DockerfilePathRelativeToManifest}'.");
}

imageTag = matchingImagePlatform.Platform.Tags.First().FullyQualifiedName;
}

manifestYml.AppendLine($"- image: {imageTag}");
manifestYml.AppendLine($" platform:");
manifestYml.AppendLine($" architecture: {platform.Model.Architecture.GetDockerName()}");
manifestYml.AppendLine($" os: {platform.Model.OS.GetDockerName()}");
Expand Down
5 changes: 1 addition & 4 deletions src/Microsoft.DotNet.ImageBuilder/src/McrTagsMetadataGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,10 +140,7 @@ private string GetVariableValue(string variableType, string variableName)
// duplicated in another image in order to associate it within a distinct set of shared tags.
IEnumerable<ImageDocumentationInfo> matchingDocInfos = _imageDocInfos
.Where(docInfo => docInfo != info &&
docInfo.Platform.DockerfilePath == info.Platform.DockerfilePath &&
docInfo.Platform.Model.OsVersion == info.Platform.Model.OsVersion &&
docInfo.Platform.Model.Architecture == info.Platform.Model.Architecture &&
docInfo.Image.ProductVersion == info.Image.ProductVersion)
PlatformInfo.AreMatchingPlatforms(docInfo.Image, docInfo.Platform, info.Image, info.Platform))
.Prepend(info)
.ToArray();

Expand Down
22 changes: 13 additions & 9 deletions src/Microsoft.DotNet.ImageBuilder/src/Models/Image/PlatformData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,25 +69,29 @@ public int CompareTo([AllowNull] PlatformData other)
return 1;
}

// If either of the platforms has no simple tags while the other does have simple tags, they are not equal
if ((SimpleTags?.Count == 0 && other.SimpleTags?.Count > 0) ||
(SimpleTags?.Count > 0 && other.SimpleTags?.Count == 0))
{
return 1;
}

return GetIdentifier().CompareTo(other.GetIdentifier());
}

public bool Equals(PlatformInfo platformInfo)
{
return GetIdentifier() == FromPlatformInfo(platformInfo, null).GetIdentifier();
}
public bool Equals(PlatformInfo platformInfo) =>
CompareTo(FromPlatformInfo(platformInfo, null)) == 0;

public string GetIdentifier() => $"{Dockerfile}-{Architecture}-{OsType}-{OsVersion}";

public static PlatformData FromPlatformInfo(PlatformInfo platform, ImageInfo image)
{
return new PlatformData(image, platform)
public static PlatformData FromPlatformInfo(PlatformInfo platform, ImageInfo image) =>
new PlatformData(image, platform)
{
Dockerfile = platform.DockerfilePathRelativeToManifest,
Architecture = platform.Model.Architecture.GetDisplayName(),
OsType = platform.Model.OS.ToString(),
OsVersion = platform.GetOSDisplayName()
OsVersion = platform.GetOSDisplayName(),
SimpleTags = platform.Tags.Select(tag => tag.Name).ToList()
};
}
}
}
6 changes: 6 additions & 0 deletions src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,12 @@ public string GetOSDisplayName()
return displayName;
}

public static bool AreMatchingPlatforms(ImageInfo image1, PlatformInfo platform1, ImageInfo image2, PlatformInfo platform2) =>
platform1.DockerfilePath == platform2.DockerfilePath &&
platform1.Model.OsVersion == platform2.Model.OsVersion &&
platform1.Model.Architecture == platform2.Model.Architecture &&
image1.ProductVersion == image2.ProductVersion;

private static bool IsStageReference(string fromImage, IList<Match> fromMatches)
{
bool isStageReference = false;
Expand Down
105 changes: 103 additions & 2 deletions src/Microsoft.DotNet.ImageBuilder/tests/ImageInfoHelperTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,16 @@ public void LoadFromContent()
Dockerfile = "1.0/runtime/linux/Dockerfile",
OsType = "Linux",
OsVersion = "Ubuntu 20.04",
Architecture = "amd64"
Architecture = "amd64",
SimpleTags = new List<string> { "linux" }
},
new PlatformData
{
Dockerfile = "1.0/runtime/windows/Dockerfile",
OsType = "Windows",
OsVersion = "Windows Server, version 2004",
Architecture = "amd64"
Architecture = "amd64",
SimpleTags = new List<string> { "windows" }
}
},
Manifest = new ManifestData
Expand Down Expand Up @@ -668,6 +670,105 @@ public void ImageInfoHelper_MergeRepos_RemoveTag()
CompareImageArtifactDetails(expected, targetImageArtifactDetails);
}

[Fact]
public void Merge_DuplicatedPlatforms()
{
ImageArtifactDetails imageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo",
Images =
{
new ImageData
{
ManifestImage = CreateImageInfo(),
Platforms =
{
new PlatformData
{
Dockerfile = "image1"
}
}
}
}
}
}
};

ImageArtifactDetails targetImageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo",
Images =
{
new ImageData
{
ManifestImage = CreateImageInfo(),
Platforms =
{
new PlatformData
{
Dockerfile = "image1",
SimpleTags = new List<string>
{
"tag1"
}
}
}
}
}
}
}
};

ImageArtifactDetails expectedImageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo",
Images =
{
new ImageData
{
Platforms =
{
new PlatformData
{
Dockerfile = "image1"
}
}
},
new ImageData
{
Platforms =
{
new PlatformData
{
Dockerfile = "image1",
SimpleTags = new List<string>
{
"tag1"
}
}
}
},
}
}
}
};

ImageInfoHelper.MergeImageArtifactDetails(imageArtifactDetails, targetImageArtifactDetails);
CompareImageArtifactDetails(expectedImageArtifactDetails, targetImageArtifactDetails);
}

public static void CompareImageArtifactDetails(ImageArtifactDetails expected, ImageArtifactDetails actual)
{
Assert.Equal(JsonHelper.SerializeObject(expected), JsonHelper.SerializeObject(actual));
Expand Down
16 changes: 8 additions & 8 deletions src/Microsoft.DotNet.ImageBuilder/tests/MergeImageInfoFilesCommandTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,15 +178,15 @@ public async Task MergeImageInfoFilesCommand_HappyPath()
CreateRepo("repo1"),
CreateRepo("repo2",
CreateImage(
CreatePlatform(repo2Image1Dockerfile, new string[0])),
CreatePlatform(repo2Image1Dockerfile, new string[] { "tag1" })),
CreateImage(
CreatePlatform(repo2Image2Dockerfile, new string[0]))),
CreatePlatform(repo2Image2Dockerfile, new string[] { "tag2" }))),
CreateRepo("repo3"),
CreateRepo("repo4",
CreateImage(
CreatePlatform(repo4Image2Dockerfile, new string[0])),
CreatePlatform(repo4Image2Dockerfile, new string[] { "tag1" })),
CreateImage(
CreatePlatform(repo4Image3Dockerfile, new string[0])))
CreatePlatform(repo4Image3Dockerfile, new string[] { "tag2" })))
);
File.WriteAllText(Path.Combine(context.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest));

Expand Down Expand Up @@ -456,12 +456,12 @@ public async Task MergeImageInfoFilesCommand_DuplicateDockerfilePaths()
Manifest manifest = CreateManifest(
CreateRepo("repo1",
CreateImage(
CreatePlatform(dockerfile1, new string[0], osVersion: Os1, architecture: Architecture.ARM),
CreatePlatform(dockerfile1, new string[0], osVersion: Os1, architecture: Architecture.ARM64)),
CreatePlatform(dockerfile1, new string[] { "tag1" }, osVersion: Os1, architecture: Architecture.ARM),
CreatePlatform(dockerfile1, new string[] { "tag2" }, osVersion: Os1, architecture: Architecture.ARM64)),
CreateImage(
CreatePlatform(dockerfile1, new string[0], osVersion: Os2, architecture: Architecture.ARM)),
CreatePlatform(dockerfile1, new string[] { "tag3" }, osVersion: Os2, architecture: Architecture.ARM)),
CreateImage(
CreatePlatform(dockerfile2, new string[0], osVersion: Os1)))
CreatePlatform(dockerfile2, new string[] { "tag4" }, osVersion: Os1)))
);
File.WriteAllText(Path.Combine(context.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest));

Expand Down
4 changes: 2 additions & 2 deletions src/Microsoft.DotNet.ImageBuilder/tests/PublishImageInfoCommandTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@ public async Task PublishImageInfoCommand_ReplaceTags()
Manifest manifest = CreateManifest(
CreateRepo("repo1",
CreateImage(
CreatePlatform(repo1Image1DockerfilePath, new string[0]))),
CreatePlatform(repo1Image1DockerfilePath, new string[] { "tag1" }))),
CreateRepo("repo2",
CreateImage(
CreatePlatform(repo2Image2DockerfilePath, new string[0])))
CreatePlatform(repo2Image2DockerfilePath, new string[] { "tag1" })))
);

RepoData repo2;
Expand Down
Loading

0 comments on commit f31211a

Please sign in to comment.