Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update image info schema to have image/platform concepts #437

Merged
merged 3 commits into from
Mar 13, 2020
Merged

Update image info schema to have image/platform concepts #437

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bafa178
Update image info schema to have image/platform concepts
mthalman Mar 11, 2020
b596769
Updates from code review
mthalman Mar 12, 2020
5503bd3
Updates from code review
mthalman Mar 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 15 additions & 18 deletions src/Microsoft.DotNet.ImageBuilder/src/Commands/BuildCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public override Task ExecuteAsync()
PullBaseImages();
BuildImages();

if (GetBuiltImages().Any())
if (GetBuiltPlatforms().Any())
{
PushImages();
}
Expand All @@ -58,7 +58,7 @@ private void PublishImageInfo()
return;
}

foreach (var image in GetBuiltImages())
foreach (var image in GetBuiltPlatforms())
{
foreach (string tag in image.FullyQualifiedSimpleTags)
{
Expand Down Expand Up @@ -95,14 +95,15 @@ private void BuildImages()
};
repoList.Add(repoData);

SortedDictionary<string, ImageData> images = new SortedDictionary<string, ImageData>();

foreach (ImageInfo image in repoInfo.FilteredImages)
{
ImageData imageData = new ImageData();
repoData.Images.Add(imageData);

foreach (PlatformInfo platform in image.FilteredPlatforms)
{
ImageData imageData = new ImageData();
images.Add(platform.DockerfilePathRelativeToManifest, imageData);
PlatformData platformData = PlatformData.FromPlatformInfo(platform);
imageData.Platforms.Add(platformData);

bool createdPrivateDockerfile = UpdateDockerfileFromCommands(platform, out string dockerfilePath);

Expand Down Expand Up @@ -145,24 +146,19 @@ private void BuildImages()
SortedDictionary<string, string> baseImageDigests = GetBaseImageDigests(platform);
if (baseImageDigests.Any())
{
imageData.BaseImages = baseImageDigests;
platformData.BaseImages = baseImageDigests;
}

imageData.SimpleTags = GetPushTags(platform.Tags)
platformData.SimpleTags = GetPushTags(platform.Tags)
.Select(tag => tag.Name)
.OrderBy(name => name)
.ToList();
imageData.FullyQualifiedSimpleTags = imageData.SimpleTags
platformData.FullyQualifiedSimpleTags = platformData.SimpleTags
.Select(tag => TagInfo.GetFullyQualifiedName(repoInfo.Name, tag))
.ToList();
imageData.AllTags = allTags;
platformData.AllTags = allTags;
}
}

if (images.Any())
{
repoData.Images = images;
}
}
}

Expand Down Expand Up @@ -297,12 +293,13 @@ private void PullBaseImages()
}
}

private IEnumerable<ImageData> GetBuiltImages() => this.repoList
private IEnumerable<PlatformData> GetBuiltPlatforms() => this.repoList
.Where(repoData => repoData.Images != null)
.SelectMany(repoData => repoData.Images.Values);
.SelectMany(repoData => repoData.Images)
.SelectMany(imageData => imageData.Platforms);

private IEnumerable<string> GetBuiltTags() =>
GetBuiltImages().SelectMany(image => image.AllTags);
GetBuiltPlatforms().SelectMany(image => image.AllTags);

private void PushImages()
{
Expand Down
14 changes: 7 additions & 7 deletions src/Microsoft.DotNet.ImageBuilder/src/Commands/CopyAcrImagesCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Azure.Management.ContainerRegistry.Fluent;
Expand All @@ -16,7 +15,6 @@
using Microsoft.DotNet.ImageBuilder.Models.Image;
using Microsoft.DotNet.ImageBuilder.Services;
using Microsoft.DotNet.ImageBuilder.ViewModel;
using Newtonsoft.Json;
using ImportSource = Microsoft.Azure.Management.ContainerRegistry.Fluent.Models.ImportSource;

namespace Microsoft.DotNet.ImageBuilder.Commands
Expand All @@ -37,7 +35,7 @@ public CopyAcrImagesCommand(IAzureManagementFactory azureManagementFactory, IEnv
{
if (!String.IsNullOrEmpty(Options.ImageInfoPath))
{
return JsonConvert.DeserializeObject<RepoData[]>(File.ReadAllText(Options.ImageInfoPath));
return ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest);
}

return null;
Expand Down Expand Up @@ -107,9 +105,12 @@ private IEnumerable<string> GetDestinationTagNames(RepoInfo repo, PlatformInfo p
RepoData repoData = imageInfoRepos.Value.FirstOrDefault(repoData => repoData.Repo == repo.Model.Name);
if (repoData != null)
{
if (repoData.Images.TryGetValue(platform.DockerfilePathRelativeToManifest, out ImageData image))
PlatformData platformData = repoData.Images
.SelectMany(image => image.Platforms)
.FirstOrDefault(platformData => platformData.Dockerfile == platform.DockerfilePathRelativeToManifest);
if (platformData != null)
{
destTagNames = image.SimpleTags
destTagNames = platformData.SimpleTags
.Select(tag => TagInfo.GetFullyQualifiedName(repo.Name, tag));
}
else
Expand All @@ -124,8 +125,7 @@ private IEnumerable<string> GetDestinationTagNames(RepoInfo repo, PlatformInfo p
this.environmentService.Exit(1);
}
}

if (destTagNames == null)
else
{
destTagNames = platform.Tags
.Select(tag => tag.FullyQualifiedName);
Expand Down
112 changes: 71 additions & 41 deletions src/Microsoft.DotNet.ImageBuilder/src/Commands/GetStaleImagesCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,6 @@ private async Task<IEnumerable<string>> GetPathsToRebuildAsync(Subscription subs

this.loggerService.WriteMessage($"Processing subscription: {subscription.Id}");

RepoData[] repos = await GetImageInfoForSubscriptionAsync(subscription);

string repoPath = await GetGitRepoPath(subscription);

TempManifestOptions manifestOptions = new TempManifestOptions(Options.FilterOptions)
Expand All @@ -110,6 +108,8 @@ private async Task<IEnumerable<string>> GetPathsToRebuildAsync(Subscription subs

ManifestInfo manifest = ManifestInfo.Load(manifestOptions);

RepoData[] repos = await GetImageInfoForSubscriptionAsync(subscription, manifest);

List<string> pathsToRebuild = new List<string>();

IEnumerable<PlatformInfo> allPlatforms = manifest.GetAllPlatforms().ToList();
Expand All @@ -123,60 +123,90 @@ private async Task<IEnumerable<string>> GetPathsToRebuildAsync(Subscription subs
RepoData repoData = repos
.FirstOrDefault(s => s.Repo == repo.Model.Name);

foreach (var platform in platforms)
foreach (PlatformInfo platform in platforms)
{
if (repoData != null &&
repoData.Images != null &&
repoData.Images.TryGetValue(platform.DockerfilePathRelativeToManifest, out ImageData imageData))
{
string fromImage = platform.FinalStageFromImage;
string currentDigest;
pathsToRebuild.AddRange(await GetPathsToRebuildAsync(allPlatforms, platform, repoData));
}
}

await this.imageDigestsSemaphore.WaitAsync();
try
{
if (!this.imageDigests.TryGetValue(fromImage, out currentDigest))
{
this.dockerService.PullImage(fromImage, Options.IsDryRun);
currentDigest = this.dockerService.GetImageDigest(fromImage, Options.IsDryRun);
this.imageDigests.Add(fromImage, currentDigest);
}
}
finally
{
this.imageDigestsSemaphore.Release();
}
return pathsToRebuild.Distinct().ToList();
}

string lastDigest = null;
imageData.BaseImages?.TryGetValue(fromImage, out lastDigest);
private async Task<List<string>> GetPathsToRebuildAsync(
IEnumerable<PlatformInfo> allPlatforms, PlatformInfo platform, RepoData repoData)
{
bool foundImageInfo = false;

List<string> pathsToRebuild = new List<string>();

void processPlatformWithMissingImageInfo(PlatformInfo platform)
{
this.loggerService.WriteMessage(
$"WARNING: Image info not found for '{platform.DockerfilePath}'. Adding path to build to be queued anyway.");
pathsToRebuild.Add(platform.Model.Dockerfile);
}

bool rebuildImage = lastDigest != currentDigest;
if (repoData == null || repoData.Images == null)
{
processPlatformWithMissingImageInfo(platform);
return pathsToRebuild;
}

this.loggerService.WriteMessage(
$"Checking base image '{fromImage}' from '{platform.DockerfilePath}'{Environment.NewLine}"
+ $"\tLast build digest: {lastDigest}{Environment.NewLine}"
+ $"\tCurrent digest: {currentDigest}{Environment.NewLine}"
+ $"\tImage is up-to-date: {!rebuildImage}{Environment.NewLine}");
foreach (ImageData imageData in repoData.Images)
{
PlatformData platformData = imageData.Platforms
.FirstOrDefault(platformData => platformData.Dockerfile == platform.DockerfilePathRelativeToManifest);
if (platformData != null)
{
foundImageInfo = true;
string fromImage = platform.FinalStageFromImage;
string currentDigest;

if (rebuildImage)
await this.imageDigestsSemaphore.WaitAsync();
try
{
if (!this.imageDigests.TryGetValue(fromImage, out currentDigest))
{
IEnumerable<PlatformInfo> dependentPlatforms = platform.GetDependencyGraph(allPlatforms);
pathsToRebuild.AddRange(dependentPlatforms.Select(p => p.Model.Dockerfile));
this.dockerService.PullImage(fromImage, Options.IsDryRun);
currentDigest = this.dockerService.GetImageDigest(fromImage, Options.IsDryRun);
this.imageDigests.Add(fromImage, currentDigest);
}
}
else
finally
{
this.imageDigestsSemaphore.Release();
}

string lastDigest = null;
platformData.BaseImages?.TryGetValue(fromImage, out lastDigest);

bool rebuildImage = lastDigest != currentDigest;

this.loggerService.WriteMessage(
$"Checking base image '{fromImage}' from '{platform.DockerfilePath}'{Environment.NewLine}"
+ $"\tLast build digest: {lastDigest}{Environment.NewLine}"
+ $"\tCurrent digest: {currentDigest}{Environment.NewLine}"
+ $"\tImage is up-to-date: {!rebuildImage}{Environment.NewLine}");

if (rebuildImage)
{
this.loggerService.WriteMessage(
$"WARNING: Image info not found for '{platform.DockerfilePath}'. Adding path to build to be queued anyway.");
pathsToRebuild.Add(platform.Model.Dockerfile);
IEnumerable<PlatformInfo> dependentPlatforms = platform.GetDependencyGraph(allPlatforms);
pathsToRebuild.AddRange(dependentPlatforms.Select(p => p.Model.Dockerfile));
}

break;
}
}

return pathsToRebuild.Distinct().ToList();
if (!foundImageInfo)
{
processPlatformWithMissingImageInfo(platform);
}

return pathsToRebuild;
}

private async Task<RepoData[]> GetImageInfoForSubscriptionAsync(Subscription subscription)
private async Task<RepoData[]> GetImageInfoForSubscriptionAsync(Subscription subscription, ManifestInfo manifest)
{
string imageDataJson;
using (IGitHubClient gitHubClient = this.gitHubClientFactory.GetClient(Options.GitOptions.ToGitHubAuth(), Options.IsDryRun))
Expand All @@ -189,7 +219,7 @@ private async Task<RepoData[]> GetImageInfoForSubscriptionAsync(Subscription sub
imageDataJson = await gitHubClient.GetGitHubFileContentsAsync(imageInfoPath, branch);
}

return JsonConvert.DeserializeObject<RepoData[]>(imageDataJson);
return ImageInfoHelper.LoadFromContent(imageDataJson, manifest);
}

private async Task<string> GetGitRepoPath(Subscription sub)
Expand Down
5 changes: 2 additions & 3 deletions src/Microsoft.DotNet.ImageBuilder/src/Commands/MergeImageInfoCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,11 @@
using System.Linq;
using System.Threading.Tasks;
using Microsoft.DotNet.ImageBuilder.Models.Image;
using Newtonsoft.Json;

namespace Microsoft.DotNet.ImageBuilder.Commands
{
[Export(typeof(ICommand))]
public class MergeImageInfoCommand : Command<MergeImageInfoOptions>
public class MergeImageInfoCommand : ManifestCommand<MergeImageInfoOptions>
{
public override Task ExecuteAsync()
{
Expand All @@ -25,7 +24,7 @@ public override Task ExecuteAsync()

List<RepoData[]> srcReposList = imageInfoFiles
.OrderBy(file => file) // Ensure the files are ordered for testing consistency between OS's.
.Select(imageDataPath => JsonConvert.DeserializeObject<RepoData[]>(File.ReadAllText(imageDataPath)))
.Select(imageDataPath => ImageInfoHelper.LoadFromFile(imageDataPath, Manifest))
.ToList();

if (!srcReposList.Any())
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.DotNet.ImageBuilder/src/Commands/MergeImageInfoOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

namespace Microsoft.DotNet.ImageBuilder.Commands
{
public class MergeImageInfoOptions : Options
public class MergeImageInfoOptions : ManifestOptions
{
protected override string CommandHelp => "Merges the content of multiple image info files into one file";

Expand Down
5 changes: 2 additions & 3 deletions src/Microsoft.DotNet.ImageBuilder/src/Commands/PublishImageInfoCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.DotNet.ImageBuilder.Models.Image;
Expand All @@ -15,7 +14,7 @@
namespace Microsoft.DotNet.ImageBuilder.Commands
{
[Export(typeof(ICommand))]
public class PublishImageInfoCommand : Command<PublishImageInfoOptions>
public class PublishImageInfoCommand : ManifestCommand<PublishImageInfoOptions>
{
private readonly IGitHubClientFactory gitHubClientFactory;

Expand All @@ -27,7 +26,7 @@ public PublishImageInfoCommand(IGitHubClientFactory gitHubClientFactory)

public override async Task ExecuteAsync()
{
RepoData[] srcRepos = JsonConvert.DeserializeObject<RepoData[]>(File.ReadAllText(Options.ImageInfoPath));
RepoData[] srcRepos = ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest);

using (IGitHubClient gitHubClient = this.gitHubClientFactory.GetClient(Options.GitOptions.ToGitHubAuth(), Options.IsDryRun))
{
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.DotNet.ImageBuilder/src/Commands/PublishImageInfoOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

namespace Microsoft.DotNet.ImageBuilder.Commands
{
public class PublishImageInfoOptions : Options, IGitOptionsHost
public class PublishImageInfoOptions : ManifestOptions, IGitOptionsHost
{
protected override string CommandHelp => "Publishes a build's merged image info.";

Expand Down
Loading