Skip to content

Commit

Permalink
Enable the partial restore optimization for legacy package reference …
Browse files Browse the repository at this point in the history 
…projects (#3506)
  • Loading branch information
@nkolev92
nkolev92 authored Jul 16, 2020
1 parent 90ad7af commit f359ca0
Show file tree
Hide file tree
Showing 6 changed files with 270 additions and 76 deletions.
14 changes: 7 additions & 7 deletions src/NuGet.Clients/NuGet.SolutionRestoreManager/ISolutionRestoreChecker.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System.Collections.Generic;
using System.Threading.Tasks;

using NuGet.Commands;
using NuGet.Common;
using NuGet.ProjectModel;

namespace NuGet.SolutionRestoreManager
Expand All @@ -19,14 +18,16 @@ public interface ISolutionRestoreChecker
/// The checker itself caches the DependencyGraphSpec it is provided and the last restore status, reported through <see cref="ReportStatus(IReadOnlyList{RestoreSummary})"/>.
/// Accounts for changes in the PackageSpec and marks all the parent projects as dirty as well.
/// Additionally, ensures that the expected output files have the same timestamps as the last reported status
/// <see cref="ReportStatus(IReadOnlyList{RestoreSummary})"/>.
/// <see cref="ReportStatus(IReadOnlyList{RestoreSummary})"/>
/// Finally we will use the logger provided to replay any warnings necessary. Whether warnings are replayed in conditional on the <see cref="ProjectRestoreSettings"/> in the <see cref="PackageSpec"/>.
/// </summary>
/// <param name="dependencyGraphSpec">The current dependency graph spec.</param>
/// <param name="logger">A logger that will be used to replay warnings for projects that no-op if necessary.</param>
/// <returns>Unique ids of the dirty projects</returns>
/// <remarks>Note that this call is stateful. This method may end up caching the dependency graph spec, so do not invoke multiple times.
/// Ideally <see cref="PerformUpToDateCheck(DependencyGraphSpec)"/> call should be followed by a <see cref="ReportStatus(IReadOnlyList{RestoreSummary})"/> call.
/// Ideally each <see cref="PerformUpToDateCheck(DependencyGraphSpec, ILogger)"/> call should be followed by a <see cref="ReportStatus(IReadOnlyList{RestoreSummary})"/> call.
/// </remarks>
IEnumerable<string> PerformUpToDateCheck(DependencyGraphSpec dependencyGraphSpec);
IEnumerable<string> PerformUpToDateCheck(DependencyGraphSpec dependencyGraphSpec, ILogger logger);

/// <summary>
/// Report the status of all the projects restored.
Expand All @@ -35,12 +36,11 @@ public interface ISolutionRestoreChecker
/// <remarks>Note that this call is stateful. This method may end up caching the dependency graph spec, so do not invoke multiple times.
/// Ideally <see cref="PerformUpToDateCheck(DependencyGraphSpec)"/> call should be followed by a <see cref="ReportStatus(IReadOnlyList{RestoreSummary})"/> call.
/// </remarks>
void ReportStatus(IReadOnlyList<RestoreSummary> restoreSummaries);
void SaveRestoreStatus(IReadOnlyList<RestoreSummary> restoreSummaries);

/// <summary>
/// Clears any cached values. This is meant to mimic restores that overwrite the incremental restore optimizations.
/// </summary>
/// <returns></returns>
void CleanCache();
}
}
11 changes: 2 additions & 9 deletions src/NuGet.Clients/NuGet.SolutionRestoreManager/SolutionRestoreJob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,8 +355,7 @@ private async Task RestorePackageSpecProjectsAsync(
using (intervalTracker.Start(RestoreTelemetryEvent.SolutionUpToDateCheck))
{
// Run solution based up to date check.
var projectsNeedingRestore = _solutionUpToDateChecker.PerformUpToDateCheck(originalDgSpec).AsList();

var projectsNeedingRestore = _solutionUpToDateChecker.PerformUpToDateCheck(originalDgSpec, _logger).AsList();
dgSpec = originalDgSpec;
// Only use the optimization results if the restore is not `force`.
// Still run the optimization check anyways to prep the cache.
Expand All @@ -368,12 +367,6 @@ private async Task RestorePackageSpecProjectsAsync(
{
dgSpec.AddRestore(uniqueProjectId);
}
// loop through all legacy PackageReference projects. We don't know how to replay their warnings & errors yet. TODO: https://github.com/NuGet/Home/issues/9565
foreach(var project in (await _solutionManager.GetNuGetProjectsAsync()).Where(e => e is LegacyPackageReferenceProject).Select(e => e as LegacyPackageReferenceProject))
{
dgSpec.AddRestore(project.MSBuildProjectPath);
}

// recorded the number of up to date projects
_upToDateProjectCount = originalDgSpec.Restore.Count - projectsNeedingRestore.Count;
_noOpProjectsCount = _upToDateProjectCount;
Expand Down Expand Up @@ -419,7 +412,7 @@ await _logger.RunWithProgressAsync(
_packageCount += restoreSummaries.Select(summary => summary.InstallCount).Sum();
var isRestoreFailed = restoreSummaries.Any(summary => summary.Success == false);
_noOpProjectsCount += restoreSummaries.Where(summary => summary.NoOpRestore == true).Count();
_solutionUpToDateChecker.ReportStatus(restoreSummaries);
_solutionUpToDateChecker.SaveRestoreStatus(restoreSummaries);
if (isRestoreFailed)
{
Expand Down
68 changes: 56 additions & 12 deletions src/NuGet.Clients/NuGet.SolutionRestoreManager/SolutionUpToDateChecker.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,15 @@ public class SolutionUpToDateChecker : ISolutionRestoreChecker
{
private IList<string> _failedProjects = new List<string>();
private DependencyGraphSpec _cachedDependencyGraphSpec;
private Dictionary<string, RestoreOutputData> _outputWriteTimes = new Dictionary<string, RestoreOutputData>();
private Dictionary<string, RestoreData> _restoreData = new Dictionary<string, RestoreData>();

public void ReportStatus(IReadOnlyList<RestoreSummary> restoreSummaries)
public void SaveRestoreStatus(IReadOnlyList<RestoreSummary> restoreSummaries)
{
if (restoreSummaries == null)
{
throw new ArgumentNullException(nameof(restoreSummaries));
}

_failedProjects.Clear();

foreach (var summary in restoreSummaries)
Expand All @@ -30,22 +35,25 @@ public void ReportStatus(IReadOnlyList<RestoreSummary> restoreSummaries)
{
var packageSpec = _cachedDependencyGraphSpec.GetProjectSpec(summary.InputPath);
GetOutputFilePaths(packageSpec, out string assetsFilePath, out string cacheFilePath, out string targetsFilePath, out string propsFilePath, out string lockFilePath);
var messages = !packageSpec.RestoreSettings.HideWarningsAndErrors && summary.Errors.Count > 0 ?
summary.Errors :
null;

_outputWriteTimes[summary.InputPath] = new RestoreOutputData()
_restoreData[summary.InputPath] = new RestoreData()
{
_lastAssetsFileWriteTime = GetLastWriteTime(assetsFilePath),
_lastCacheFileWriteTime = GetLastWriteTime(cacheFilePath),
_lastTargetsFileWriteTime = GetLastWriteTime(targetsFilePath),
_lastPropsFileWriteTime = GetLastWriteTime(propsFilePath),
_lastLockFileWriteTime = GetLastWriteTime(lockFilePath),
_globalPackagesFolderCreationTime = GetCreationTime(packageSpec.RestoreMetadata.PackagesPath)
_globalPackagesFolderCreationTime = GetCreationTime(packageSpec.RestoreMetadata.PackagesPath),
_messages = messages
};
}
else
{
_failedProjects.Add(summary.InputPath);
}

}
}

Expand All @@ -61,8 +69,18 @@ public void ReportStatus(IReadOnlyList<RestoreSummary> restoreSummaries)
// Finally we only update the cache specs if Pass #1 determined that there are projects that are not up to date.
// Result
// Lastly all the projects marked as having dirty specs & dirty outputs are returned.
public IEnumerable<string> PerformUpToDateCheck(DependencyGraphSpec dependencyGraphSpec)
// Before we return the list of projects that are not up to date, we always make sure to replay the warnings for the up to date projects.
public IEnumerable<string> PerformUpToDateCheck(DependencyGraphSpec dependencyGraphSpec, ILogger logger)
{
if (dependencyGraphSpec == null)
{
throw new ArgumentNullException(nameof(dependencyGraphSpec));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}

if (_cachedDependencyGraphSpec != null)
{
var dirtySpecs = new List<string>();
Expand All @@ -72,6 +90,7 @@ public IEnumerable<string> PerformUpToDateCheck(DependencyGraphSpec dependencyGr
// Pass #1. Validate all the data (i/o)
// 1a. Validate the package specs (references & settings)
// 1b. Validate the expected outputs (assets file, nuget.g.*, lock file)
var unloadedProjects = _restoreData.Keys.ToHashSet();
foreach (var project in dependencyGraphSpec.Projects)
{
var projectUniqueName = project.RestoreMetadata.ProjectUniqueName;
Expand All @@ -81,14 +100,15 @@ public IEnumerable<string> PerformUpToDateCheck(DependencyGraphSpec dependencyGr
{
dirtySpecs.Add(projectUniqueName);
}
unloadedProjects.Remove(projectUniqueName);

if (project.RestoreMetadata.ProjectStyle == ProjectStyle.PackageReference ||
project.RestoreMetadata.ProjectStyle == ProjectStyle.ProjectJson)
{
if (!_failedProjects.Contains(projectUniqueName) && _outputWriteTimes.TryGetValue(projectUniqueName, out RestoreOutputData outputWriteTime))
if (!_failedProjects.Contains(projectUniqueName) && _restoreData.TryGetValue(projectUniqueName, out RestoreData restoreData))
{
GetOutputFilePaths(project, out string assetsFilePath, out string cacheFilePath, out string targetsFilePath, out string propsFilePath, out string lockFilePath);
if (!AreOutputsUpToDate(assetsFilePath, cacheFilePath, targetsFilePath, propsFilePath, lockFilePath, project.RestoreMetadata.PackagesPath, outputWriteTime))
if (!AreOutputsUpToDate(assetsFilePath, cacheFilePath, targetsFilePath, propsFilePath, lockFilePath, project.RestoreMetadata.PackagesPath, restoreData))
{
dirtyOutputs.Add(projectUniqueName);
}
Expand All @@ -104,9 +124,16 @@ public IEnumerable<string> PerformUpToDateCheck(DependencyGraphSpec dependencyGr
}
}

// Remove the cached data of the unloaded projects if any.
foreach (var project in unloadedProjects)
{
_restoreData.Remove(project);
}

// Fast path. Skip Pass #2
if (dirtySpecs.Count == 0 && dirtyOutputs.Count == 0)
{
ReplayAllWarnings(_restoreData, (string projectName) => true , logger);
return Enumerable.Empty<string>();
}
// Update the cache before Pass #2
Expand All @@ -116,12 +143,14 @@ public IEnumerable<string> PerformUpToDateCheck(DependencyGraphSpec dependencyGr
var dirtyProjects = GetParents(dirtySpecs, dependencyGraphSpec);

// All dirty projects + projects with outputs that need to be restored
// - the projects that are non transitive that never needed restore anyways, hence the insertion with the provider restore projects!
// - the projects that are non transitive that never needed restore anyways, hence the intersection with the provider restore projects!
var resultSpecs = dirtyProjects.Union(dirtyOutputs);
if (hasDirtyNonTransitiveSpecs)
{
resultSpecs = dependencyGraphSpec.Restore.Intersect(resultSpecs);
}

ReplayAllWarnings(_restoreData, (string projectName) => !resultSpecs.Contains(projectName), logger);
return resultSpecs;
}
else
Expand All @@ -132,6 +161,20 @@ public IEnumerable<string> PerformUpToDateCheck(DependencyGraphSpec dependencyGr
}
}

private void ReplayAllWarnings(Dictionary<string, RestoreData> restoreData, Func<string, bool> shouldReplayWarnings, ILogger logger)
{
foreach (var restoreOutputs in restoreData)
{
if (shouldReplayWarnings(restoreOutputs.Key) && restoreOutputs.Value._messages != null)
{
foreach (var logMessage in restoreOutputs.Value._messages)
{
logger.Log(logMessage);
}
}
}
}

/// <summary>
/// Given a list of project unique names, goes through the dg spec and returns the current projects + all their parents
/// </summary>
Expand Down Expand Up @@ -186,7 +229,7 @@ internal static void GetOutputFilePaths(PackageSpec packageSpec, out string asse
null;
}

private static bool AreOutputsUpToDate(string assetsFilePath, string cacheFilePath, string targetsFilePath, string propsFilePath, string lockFilePath, string globalPackagesFolderPath, RestoreOutputData outputWriteTime)
private static bool AreOutputsUpToDate(string assetsFilePath, string cacheFilePath, string targetsFilePath, string propsFilePath, string lockFilePath, string globalPackagesFolderPath, RestoreData outputWriteTime)
{
DateTime currentAssetsFileWriteTime = GetLastWriteTime(assetsFilePath);
DateTime currentCacheFilePath = GetLastWriteTime(cacheFilePath);
Expand Down Expand Up @@ -248,17 +291,18 @@ public void CleanCache()
{
_failedProjects.Clear();
_cachedDependencyGraphSpec = null;
_outputWriteTimes.Clear();
_restoreData.Clear();
}

internal struct RestoreOutputData
internal struct RestoreData
{
internal DateTime _lastAssetsFileWriteTime;
internal DateTime _lastCacheFileWriteTime;
internal DateTime _lastTargetsFileWriteTime;
internal DateTime _lastPropsFileWriteTime;
internal DateTime _lastLockFileWriteTime;
internal DateTime _globalPackagesFolderCreationTime;
internal IReadOnlyList<IRestoreLogMessage> _messages;
}
}
}
3 changes: 3 additions & 0 deletions src/NuGet.Core/NuGet.Commands/RestoreCommand/RestoreSummary.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ public class RestoreSummary

public int InstallCount { get; }

/// <summary>
/// All the warnings and errors that were produced as a result of the restore.
/// </summary>
public IReadOnlyList<IRestoreLogMessage> Errors { get; }

public RestoreSummary(bool success)
Expand Down
Loading

0 comments on commit f359ca0

Please sign in to comment.