diff --git a/Scalar.Common/Git/GitProcess.cs b/Scalar.Common/Git/GitProcess.cs index 2ebc5e3f58..24fb3d0f12 100644 --- a/Scalar.Common/Git/GitProcess.cs +++ b/Scalar.Common/Git/GitProcess.cs @@ -424,6 +424,23 @@ public Result ForceCheckoutAllFiles() return this.InvokeGitInWorkingDirectoryRoot("checkout HEAD -- .", fetchMissingObjects: true); } + public Result BackgroundFetch(string remote) + { + // By using "--no-update-remote-refs", the user will see their remote refs update + // normally when they do a foreground fetch. + // By using this refspec, we do not create local refs, but instead store them in the "hidden" + // namespace. These refs are never visible to the user (unless they open the .git/refs dir) + // but still allow us to run reachability questions like updating the commit-graph. + return this.InvokeGitInWorkingDirectoryRoot($"fetch {remote} --quiet --no-update-remote-refs +refs/heads/*:refs/{ScalarConstants.DotGit.Refs.Hidden.Name}/{remote}/*", fetchMissingObjects: true); + } + + public string[] GetRemotes() + { + return this.InvokeGitInWorkingDirectoryRoot("remote", fetchMissingObjects: false) + .Output + .Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); + } + public Result SparseCheckoutSet(List foldersToSet) { return this.InvokeGitInWorkingDirectoryRoot( diff --git a/Scalar.Common/Maintenance/ConfigStep.cs b/Scalar.Common/Maintenance/ConfigStep.cs index 80c855bc7d..90a99ba171 100644 --- a/Scalar.Common/Maintenance/ConfigStep.cs +++ b/Scalar.Common/Maintenance/ConfigStep.cs @@ -83,6 +83,11 @@ public bool TrySetConfig(out string error) string expectedHooksPath = Path.Combine(this.Context.Enlistment.WorkingDirectoryRoot, ScalarConstants.DotGit.Hooks.Root); expectedHooksPath = Paths.ConvertPathToGitFormat(expectedHooksPath); + if (!this.UseGvfsProtocol.HasValue) + { + this.UseGvfsProtocol = this.Context.Enlistment.UsesGvfsProtocol; + } + // These settings are required for normal Scalar functionality. // They will override any existing local configuration values. // @@ -131,7 +136,9 @@ public bool TrySetConfig(out string error) requiredSettings.Add("http.sslBackend", "schannel"); } - if (!this.TrySetConfig(requiredSettings, isRequired: true, out error)) + // If we do not use the GVFS protocol, then these config settings + // are in fact optional. + if (!this.TrySetConfig(requiredSettings, isRequired: this.UseGvfsProtocol.Value, out error)) { error = $"Failed to set some required settings: {error}"; this.Context.Tracer.RelatedError(error); diff --git a/Scalar.Common/Maintenance/FetchCommitsAndTreesStep.cs b/Scalar.Common/Maintenance/FetchStep.cs similarity index 78% rename from Scalar.Common/Maintenance/FetchCommitsAndTreesStep.cs rename to Scalar.Common/Maintenance/FetchStep.cs index 00d126783d..0041e985dc 100644 --- a/Scalar.Common/Maintenance/FetchCommitsAndTreesStep.cs +++ b/Scalar.Common/Maintenance/FetchStep.cs @@ -9,34 +9,97 @@ namespace Scalar.Common.Maintenance { - public class FetchCommitsAndTreesStep : GitMaintenanceStep + public class FetchStep : GitMaintenanceStep { private const int IoFailureRetryDelayMS = 50; private const int LockWaitTimeMs = 100; private const int WaitingOnLockLogThreshold = 50; private const string FetchCommitsAndTreesLock = "fetch-commits-trees.lock"; + private const string FetchTimeFile = "fetch.time"; private readonly TimeSpan timeBetweenFetches = TimeSpan.FromMinutes(70); private readonly TimeSpan timeBetweenFetchesNoCacheServer = TimeSpan.FromDays(1); + private readonly bool forceRun; - public FetchCommitsAndTreesStep(ScalarContext context, GitObjects gitObjects, bool requireCacheLock = true) + public FetchStep( + ScalarContext context, + GitObjects gitObjects, + bool requireCacheLock = true, + bool forceRun = false) : base(context, requireCacheLock) { this.GitObjects = gitObjects; + this.forceRun = forceRun; } public override string Area => "FetchCommitsAndTreesStep"; + // Used only for vanilla Git repos + protected override TimeSpan TimeBetweenRuns => this.timeBetweenFetches; + protected GitObjects GitObjects { get; } - public bool TryFetchCommitsAndTrees(out string error, GitProcess gitProcess = null) + public bool TryFetch(out string error, GitProcess gitProcess = null) { if (gitProcess == null) { gitProcess = new GitProcess(this.Context.Enlistment); } + if (this.Context.Enlistment.UsesGvfsProtocol) + { + return this.TryFetchUsingGvfsProtocol(gitProcess, out error); + } + else + { + return this.TryFetchUsingGitProtocol(gitProcess, out error); + } + + } + + protected override void PerformMaintenance() + { + string error = null; + + this.RunGitCommand( + process => + { + this.TryFetch(out error, process); + return null; + }, + nameof(this.TryFetch)); + + if (!string.IsNullOrEmpty(error)) + { + this.Context.Tracer.RelatedWarning( + metadata: this.CreateEventMetadata(), + message: $"{nameof(this.TryFetch)} failed with error '{error}'", + keywords: Keywords.Telemetry); + } + } + + private bool TryFetchUsingGvfsProtocol(GitProcess gitProcess, out string error) + { + if (!this.TryGetMaxGoodPrefetchPackTimestamp(out long last, out error)) + { + this.Context.Tracer.RelatedError(error); + return false; + } + + TimeSpan timeBetween = this.GitObjects.IsUsingCacheServer() + ? this.timeBetweenFetches + : this.timeBetweenFetchesNoCacheServer; + + DateTime lastDateTime = EpochConverter.FromUnixEpochSeconds(last); + DateTime now = DateTime.UtcNow; + if (!this.forceRun && now <= lastDateTime + timeBetween) + { + this.Context.Tracer.RelatedInfo(this.Area + ": Skipping fetch since most-recent fetch ({0}) is too close to now ({1})", lastDateTime, now); + error = null; + return true; + } + // We take our own lock here to keep background and foreground fetches - // (i.e. a user running 'scalar maintenance --fetch-commits-and-trees') + // (i.e. a user running 'scalar maintenance --task fetch') // from running at the same time. using (FileBasedLock fetchLock = ScalarPlatform.Instance.CreateFileBasedLock( this.Context.FileSystem, @@ -63,43 +126,50 @@ public bool TryFetchCommitsAndTrees(out string error, GitProcess gitProcess = nu return true; } - protected override void PerformMaintenance() + private bool TryFetchUsingGitProtocol(GitProcess gitProcess, out string error) { - long last; - string error = null; + this.LastRunTimeFilePath = Path.Combine(this.Context.Enlistment.ScalarLogsRoot, FetchTimeFile); - if (!this.TryGetMaxGoodPrefetchPackTimestamp(out last, out error)) + if (!this.forceRun && !this.EnoughTimeBetweenRuns()) { - this.Context.Tracer.RelatedError(error); - return; + this.Context.Tracer.RelatedInfo($"Skipping {nameof(FetchStep)} due to not enough time between runs"); + error = null; + return true; } - TimeSpan timeBetween = this.GitObjects.IsUsingCacheServer() - ? this.timeBetweenFetches - : this.timeBetweenFetchesNoCacheServer; - - DateTime lastDateTime = EpochConverter.FromUnixEpochSeconds(last); - DateTime now = DateTime.UtcNow; - if (now <= lastDateTime + timeBetween) + using (ITracer activity = this.Context.Tracer.StartActivity(nameof(GitProcess.BackgroundFetch), EventLevel.LogAlways)) { - this.Context.Tracer.RelatedInfo(this.Area + ": Skipping fetch since most-recent fetch ({0}) is too close to now ({1})", lastDateTime, now); - return; - } + // Clear hidden refs to avoid arbitrarily large growth + string hiddenRefspace = Path.Combine(this.Context.Enlistment.WorkingDirectoryRoot, ScalarConstants.DotGit.Refs.Hidden.Root); - this.RunGitCommand( - process => + if (this.Context.FileSystem.DirectoryExists(hiddenRefspace)) { - this.TryFetchCommitsAndTrees(out error, process); - return null; - }, - nameof(this.TryFetchCommitsAndTrees)); + this.Context.FileSystem.DeleteDirectory(hiddenRefspace, recursive: true); + } - if (!string.IsNullOrEmpty(error)) - { - this.Context.Tracer.RelatedWarning( - metadata: this.CreateEventMetadata(), - message: $"{nameof(this.TryFetchCommitsAndTrees)} failed with error '{error}'", - keywords: Keywords.Telemetry); + string[] remotes = gitProcess.GetRemotes(); + bool response = true; + + error = ""; + foreach (string remote in remotes) + { + GitProcess.Result result = gitProcess.BackgroundFetch(remote); + + if (!string.IsNullOrWhiteSpace(result.Errors)) + { + error += result.Errors; + activity.RelatedError($"Background fetch from '{remote}' completed with stderr: {result.Errors}"); + } + + if (result.ExitCodeIsFailure) + { + response = false; + // Keep going through other remotes, but the overall result will still be false. + } + } + + this.SaveLastRunTimeToFile(); + return response; } } diff --git a/Scalar.Common/Maintenance/MaintenanceTasks.cs b/Scalar.Common/Maintenance/MaintenanceTasks.cs index 4f0615e724..048eed0e66 100644 --- a/Scalar.Common/Maintenance/MaintenanceTasks.cs +++ b/Scalar.Common/Maintenance/MaintenanceTasks.cs @@ -19,7 +19,7 @@ public static string GetVerbTaskName(Task task) switch (task) { case Task.FetchCommitsAndTrees: - return ScalarConstants.VerbParameters.Maintenance.FetchCommitsAndTreesTaskName; + return ScalarConstants.VerbParameters.Maintenance.FetchTaskName; case Task.LooseObjects: return ScalarConstants.VerbParameters.Maintenance.LooseObjectsTaskName; case Task.PackFiles: diff --git a/Scalar.Common/Maintenance/PackfileMaintenanceStep.cs b/Scalar.Common/Maintenance/PackfileMaintenanceStep.cs index 1fefc1b3da..6064013f12 100644 --- a/Scalar.Common/Maintenance/PackfileMaintenanceStep.cs +++ b/Scalar.Common/Maintenance/PackfileMaintenanceStep.cs @@ -110,7 +110,7 @@ protected override void PerformMaintenance() this.GetPackFilesInfo(out int beforeCount, out long beforeSize, out _, out bool hasKeep); - if (!hasKeep) + if (!hasKeep && this.Context.Enlistment.UsesGvfsProtocol) { activity.RelatedWarning(this.CreateEventMetadata(), "Skipping pack maintenance due to no .keep file."); return; diff --git a/Scalar.Common/ScalarConstants.cs b/Scalar.Common/ScalarConstants.cs index 0619415561..b82e43059a 100644 --- a/Scalar.Common/ScalarConstants.cs +++ b/Scalar.Common/ScalarConstants.cs @@ -166,6 +166,12 @@ public static class Pack public static class Refs { public static readonly string Root = Path.Combine(DotGit.Root, "refs"); + + public static class Hidden + { + public static readonly string Name = "hidden"; + public static readonly string Root = Path.Combine(DotGit.Refs.Root, Name); + } } } @@ -194,7 +200,7 @@ public static class Maintenance { public const string Task = "task"; - public const string FetchCommitsAndTreesTaskName = "fetch-commits-and-trees"; + public const string FetchTaskName = "fetch"; public const string LooseObjectsTaskName = "loose-objects"; public const string PackFilesTaskName = "pack-files"; public const string CommitGraphTaskName = "commit-graph"; diff --git a/Scalar.Common/ScalarEnlistment.cs b/Scalar.Common/ScalarEnlistment.cs index 3561711828..cf83da70b7 100644 --- a/Scalar.Common/ScalarEnlistment.cs +++ b/Scalar.Common/ScalarEnlistment.cs @@ -43,6 +43,8 @@ private ScalarEnlistment(string enlistmentRoot, string workingDirectory, string public override string LocalObjectsRoot { get; protected set; } public override string GitPackRoot { get; protected set; } + public bool UsesGvfsProtocol { get; protected set; } + // These version properties are only used in logging during clone and mount to track version numbers public string GitVersion { @@ -186,6 +188,9 @@ public void InitializeCachePaths(string localCacheRoot, string gitObjectsRoot) this.LocalCacheRoot = localCacheRoot; this.GitObjectsRoot = gitObjectsRoot; this.GitPackRoot = Path.Combine(this.GitObjectsRoot, ScalarConstants.DotGit.Objects.Pack.Name); + + // When using the GVFS protocol, we have a different cache location than local objects. + this.UsesGvfsProtocol = !this.LocalCacheRoot.Equals(this.LocalObjectsRoot); } public bool TryCreateEnlistmentFolders() diff --git a/Scalar.FunctionalTests/Categories.cs b/Scalar.FunctionalTests/Categories.cs index b989f05135..2a635caa31 100644 --- a/Scalar.FunctionalTests/Categories.cs +++ b/Scalar.FunctionalTests/Categories.cs @@ -8,6 +8,8 @@ public static class Categories public const string WindowsOnly = "WindowsOnly"; public const string MacOnly = "MacOnly"; + public const string GitRepository = "GitRepository"; + public const string NeedsUpdatesForNonVirtualizedMode = "NeedsUpdatesForNonVirtualizedMode"; public static class MacTODO diff --git a/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchCommitsAndTreesStepTests.cs b/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchStepTests.cs similarity index 83% rename from Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchCommitsAndTreesStepTests.cs rename to Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchStepTests.cs index 2ce3e7670b..f818b40f14 100644 --- a/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchCommitsAndTreesStepTests.cs +++ b/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchStepTests.cs @@ -11,22 +11,22 @@ namespace Scalar.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] [NonParallelizable] - public class FetchCommitsAndTreesStepTests : TestsWithEnlistmentPerFixture + public class FetchStepTests : TestsWithEnlistmentPerFixture { private const string FetchCommitsAndTreesLock = "fetch-commits-trees.lock"; private FileSystemRunner fileSystem; - public FetchCommitsAndTreesStepTests() + public FetchStepTests() { this.fileSystem = new SystemIORunner(); } [TestCase] [Category(Categories.MacTODO.TestNeedsToLockFile)] - public void FetchCommitsAndTreesCleansUpStaleFetchLock() + public void FetchStepCleansUpStaleFetchLock() { - this.Enlistment.FetchCommitsAndTrees(); + this.Enlistment.FetchStep(); string fetchCommitsLockFile = Path.Combine( ScalarHelpers.GetObjectsRootFromGitConfig(this.Enlistment.RepoRoot), "pack", @@ -42,7 +42,7 @@ public void FetchCommitsAndTreesCleansUpStaleFetchLock() .Count() .ShouldEqual(1, "Incorrect number of .keep files in pack directory"); - this.Enlistment.FetchCommitsAndTrees(); + this.Enlistment.FetchStep(); fetchCommitsLockFile.ShouldNotExistOnDisk(this.fileSystem); } } diff --git a/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchCommitsAndTreesWithoutSharedCacheTests.cs b/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchStepWithoutSharedCacheTests.cs similarity index 91% rename from Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchCommitsAndTreesWithoutSharedCacheTests.cs rename to Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchStepWithoutSharedCacheTests.cs index 4b11716e40..ea80b3f5ea 100644 --- a/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchCommitsAndTreesWithoutSharedCacheTests.cs +++ b/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchStepWithoutSharedCacheTests.cs @@ -9,7 +9,7 @@ namespace Scalar.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] [Category(Categories.ExtraCoverage)] - public class FetchCommitsAndTreesWithoutSharedCacheTests : TestsWithEnlistmentPerFixture + public class FetchStepWithoutSharedCacheTests : TestsWithEnlistmentPerFixture { private const string PrefetchPackPrefix = "prefetch"; private const string TempPackFolder = "tempPacks"; @@ -18,7 +18,7 @@ public class FetchCommitsAndTreesWithoutSharedCacheTests : TestsWithEnlistmentPe // Set forcePerRepoObjectCache to true to avoid any of the tests inadvertently corrupting // the cache - public FetchCommitsAndTreesWithoutSharedCacheTests() + public FetchStepWithoutSharedCacheTests() : base(forcePerRepoObjectCache: true, skipFetchCommitsAndTreesDuringClone: true) { this.fileSystem = new SystemIORunner(); @@ -41,9 +41,9 @@ private string TempPackRoot } [TestCase, Order(1)] - public void FetchCommitsAndTreesCommitsToEmptyCache() + public void FetchStepCommitsToEmptyCache() { - this.Enlistment.FetchCommitsAndTrees(); + this.Enlistment.FetchStep(); // Verify prefetch pack(s) are in packs folder and have matching idx file string[] prefetchPacks = this.ReadPrefetchPackFileNames(); @@ -54,7 +54,7 @@ public void FetchCommitsAndTreesCommitsToEmptyCache() } [TestCase, Order(2)] - public void FetchCommitsAndTreesBuildsIdxWhenMissingFromPrefetchPack() + public void FetchStepBuildsIdxWhenMissingFromPrefetchPack() { string[] prefetchPacks = this.ReadPrefetchPackFileNames(); prefetchPacks.Length.ShouldBeAtLeast(1, "There should be at least one prefetch pack"); @@ -65,8 +65,8 @@ public void FetchCommitsAndTreesBuildsIdxWhenMissingFromPrefetchPack() this.fileSystem.DeleteFile(idxPath); idxPath.ShouldNotExistOnDisk(this.fileSystem); - // fetch-commits-and-trees should rebuild the missing idx - this.Enlistment.FetchCommitsAndTrees(); + // fetch should rebuild the missing idx + this.Enlistment.FetchStep(); idxPath.ShouldBeAFile(this.fileSystem); @@ -78,7 +78,7 @@ public void FetchCommitsAndTreesBuildsIdxWhenMissingFromPrefetchPack() } [TestCase, Order(3)] - public void FetchCommitsAndTreesCleansUpBadPrefetchPack() + public void FetchStepCleansUpBadPrefetchPack() { string[] prefetchPacks = this.ReadPrefetchPackFileNames(); long mostRecentPackTimestamp = this.GetMostRecentPackTimestamp(prefetchPacks); @@ -89,8 +89,8 @@ public void FetchCommitsAndTreesCleansUpBadPrefetchPack() this.fileSystem.WriteAllText(badPackPath, badContents); badPackPath.ShouldBeAFile(this.fileSystem).WithContents(badContents); - // fetch-commits-and-trees should delete the bad pack - this.Enlistment.FetchCommitsAndTrees(); + // fetch should delete the bad pack + this.Enlistment.FetchStep(); badPackPath.ShouldNotExistOnDisk(this.fileSystem); @@ -103,7 +103,7 @@ public void FetchCommitsAndTreesCleansUpBadPrefetchPack() [TestCase, Order(4)] [Category(Categories.MacTODO.TestNeedsToLockFile)] - public void FetchCommitsAndTreesFailsWhenItCannotRemoveABadPrefetchPack() + public void FetchStepFailsWhenItCannotRemoveABadPrefetchPack() { this.Enlistment.UnregisterRepo(); @@ -119,12 +119,12 @@ public void FetchCommitsAndTreesFailsWhenItCannotRemoveABadPrefetchPack() // Open a handle to the bad pack that will prevent fetch-commits-and-trees from being able to delete it using (FileStream stream = new FileStream(badPackPath, FileMode.Open, FileAccess.Read, FileShare.None)) { - string output = this.Enlistment.FetchCommitsAndTrees(failOnError: false); + string output = this.Enlistment.FetchStep(failOnError: false); output.ShouldContain($"Unable to delete {badPackPath}"); } // After handle is closed fetching commits and trees should succeed - this.Enlistment.FetchCommitsAndTrees(); + this.Enlistment.FetchStep(); badPackPath.ShouldNotExistOnDisk(this.fileSystem); @@ -164,7 +164,7 @@ public void FetchCommitsAndTreesCleansUpStaleTempPrefetchPacks() this.fileSystem.WriteAllText(otherFilePath, otherFileContents); otherFilePath.ShouldBeAFile(this.fileSystem).WithContents(otherFileContents); - this.Enlistment.FetchCommitsAndTrees(); + this.Enlistment.FetchStep(); // Validate stale prefetch packs are cleaned up Directory.GetFiles(this.TempPackRoot, $"{PrefetchPackPrefix}*.pack").ShouldBeEmpty("There should be no .pack files in the tempPack folder"); diff --git a/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/PackfileMaintenanceStepTests.cs b/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/PackfileMaintenanceStepTests.cs index 03cafb6b70..98ee3a0c42 100644 --- a/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/PackfileMaintenanceStepTests.cs +++ b/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/PackfileMaintenanceStepTests.cs @@ -2,7 +2,6 @@ using Scalar.FunctionalTests.FileSystemRunners; using Scalar.FunctionalTests.Tools; using Scalar.Tests.Should; -using System; using System.Collections.Generic; using System.IO; using System.Linq; diff --git a/Scalar.FunctionalTests/Tests/GitRepoPerFixture/MaintenanceVerbTests.cs b/Scalar.FunctionalTests/Tests/GitRepoPerFixture/MaintenanceVerbTests.cs new file mode 100644 index 0000000000..915cf6d615 --- /dev/null +++ b/Scalar.FunctionalTests/Tests/GitRepoPerFixture/MaintenanceVerbTests.cs @@ -0,0 +1,171 @@ +using NUnit.Framework; +using Scalar.FunctionalTests.FileSystemRunners; +using Scalar.FunctionalTests.Tools; +using Scalar.Tests.Should; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Scalar.FunctionalTests.Tests.GitRepoPerFixture +{ + [Category(Categories.GitRepository)] + public class MaintenanceVerbTests : TestsWithGitRepoPerFixture + { + private FileSystemRunner fileSystem; + + private string GitObjectRoot => Path.Combine(this.Enlistment.RepoRoot, ".git", "objects"); + private string CommitGraphChain => Path.Combine(this.GitObjectRoot, "info", "commit-graphs", "commit-graph-chain"); + private string PackRoot => Path.Combine(this.Enlistment.RepoRoot, ".git", "objects", "pack"); + + public MaintenanceVerbTests() + { + this.fileSystem = new SystemIORunner(); + } + + [TestCase] + [Order(1)] + public void CommitGraphStep() + { + this.fileSystem.FileExists(CommitGraphChain).ShouldBeFalse(); + this.Enlistment.CommitGraphStep(); + this.fileSystem.FileExists(CommitGraphChain).ShouldBeTrue(); + } + + [TestCase] + [Order(2)] + public void PackfileMaintenanceStep() + { + this.GetPackSizes(out int packCount, out long maxSize, out long minSize, out long totalSize); + minSize.ShouldNotEqual(0, "min size means empty pack-file?"); + + GitProcess.InvokeProcess( + this.Enlistment.RepoRoot, + $"repack -adf --max-pack-size={totalSize / 4}"); + + this.GetPackSizes(out int countAfterRepack, out maxSize, out minSize, out totalSize); + minSize.ShouldNotEqual(0, "min size means empty pack-file?"); + + this.Enlistment + .PackfileMaintenanceStep(batchSize: totalSize - minSize + 1) + .ShouldNotContain(false, "Skipping pack maintenance due to no .keep file."); + + this.GetPackSizes(out int countAfterStep, out maxSize, out minSize, out totalSize); + minSize.ShouldNotEqual(0, "min size means empty pack-file?"); + + countAfterStep.ShouldEqual(countAfterRepack + 1, nameof(countAfterStep)); + + this.Enlistment + .PackfileMaintenanceStep(batchSize: totalSize - minSize + 1) + .ShouldNotContain(false, "Skipping pack maintenance due to no .keep file."); + + this.GetPackSizes(out int countAfterStep2, out maxSize, out minSize, out totalSize); + minSize.ShouldNotEqual(0, "min size means empty pack-file?"); + countAfterStep2.ShouldEqual(1, nameof(countAfterStep2)); + } + + [TestCase] + [Order(3)] + public void LooseObjectsStep() + { + // Create loose objects using a Git command: + GitProcess.Invoke(this.Enlistment.RepoRoot, "commit -mtest --allow-empty"); + + this.GetLooseObjectFiles().Count.ShouldBeAtLeast(1); + this.GetPackSizes(out int countBeforeStep, out _, out long minSize, out _); + minSize.ShouldNotEqual(0, "min size means empty pack-file?"); + + // This step packs the loose object into a pack. + this.Enlistment.LooseObjectStep(); + this.GetPackSizes(out int countAfterStep1, out _, out minSize, out _); + minSize.ShouldNotEqual(0, "min size means empty pack-file?"); + this.GetLooseObjectFiles().Count.ShouldBeAtLeast(1); + countAfterStep1.ShouldEqual(countBeforeStep + 1, "First step should create a pack"); + + // This step deletes the loose object that is already in a pack + this.Enlistment.LooseObjectStep(); + this.GetPackSizes(out int countAfterStep2, out _, out minSize, out _); + minSize.ShouldNotEqual(0, "min size means empty pack-file?"); + this.GetLooseObjectFiles().Count.ShouldEqual(0); + countAfterStep2.ShouldEqual(countAfterStep1, "Second step should not create a pack"); + } + + [TestCase] + [Order(4)] + public void FetchStep() + { + string refsRoot = Path.Combine(this.Enlistment.RepoRoot, ".git", "refs"); + string refsHeads = Path.Combine(refsRoot, "heads"); + string refsRemotes = Path.Combine(refsRoot, "remotes"); + string refsHidden = Path.Combine(refsRoot, "hidden"); + + // Removing refs makes the next fetch need to download a new pack + this.fileSystem.DeleteDirectory(refsHeads); + this.fileSystem.DeleteDirectory(refsRemotes); + this.fileSystem.DeleteDirectory(this.PackRoot); + + this.Enlistment.FetchStep(); + + this.GetPackSizes(out int countAfterFetch, out _, out _, out _); + + countAfterFetch.ShouldEqual(1, "fetch should download one pack"); + + this.fileSystem.DirectoryExists(refsHidden).ShouldBeTrue("background fetch should have created refs/hidden/*"); + this.fileSystem.DirectoryExists(refsHeads).ShouldBeFalse("background fetch should not have created refs/heads/*"); + this.fileSystem.DirectoryExists(refsRemotes).ShouldBeFalse("background fetch should not have created refs/remotes/*"); + + this.Enlistment.FetchStep(); + + this.fileSystem.DirectoryExists(refsHeads).ShouldBeFalse("background fetch should not have created refs/heads/*"); + this.fileSystem.DirectoryExists(refsRemotes).ShouldBeFalse("background fetch should not have created refs/remotes/*"); + + this.GetPackSizes(out int countAfterFetch2, out _, out _, out _); + countAfterFetch2.ShouldEqual(1, "sceond fetch should not download a pack"); + } + + private List GetPackfiles() + { + return Directory.GetFiles(this.PackRoot, "*.pack").ToList(); + } + + private void GetPackSizes(out int packCount, out long maxSize, out long minSize, out long totalSize) + { + totalSize = 0; + maxSize = 0; + minSize = long.MaxValue; + packCount = 0; + + foreach (string file in this.GetPackfiles()) + { + packCount++; + long size = new FileInfo(Path.Combine(this.PackRoot, file)).Length; + totalSize += size; + + if (size > maxSize) + { + maxSize = size; + } + + if (size < minSize) + { + minSize = size; + } + } + } + private List GetLooseObjectFiles() + { + List looseObjectFiles = new List(); + foreach (string directory in Directory.GetDirectories(this.GitObjectRoot)) + { + // Check if the directory is 2 letter HEX + if (Regex.IsMatch(directory, @"[/\\][0-9a-fA-F]{2}$")) + { + string[] files = Directory.GetFiles(directory); + looseObjectFiles.AddRange(files); + } + } + + return looseObjectFiles; + } + } +} diff --git a/Scalar.FunctionalTests/Tests/GitRepoPerFixture/ReposVerbTests.cs b/Scalar.FunctionalTests/Tests/GitRepoPerFixture/ReposVerbTests.cs new file mode 100644 index 0000000000..6e98ff685f --- /dev/null +++ b/Scalar.FunctionalTests/Tests/GitRepoPerFixture/ReposVerbTests.cs @@ -0,0 +1,18 @@ +using NUnit.Framework; +using Scalar.Tests.Should; + +namespace Scalar.FunctionalTests.Tests.GitRepoPerFixture +{ + [Category(Categories.GitRepository)] + public class ReposVerbTests : TestsWithGitRepoPerFixture + { + [TestCase] + public void ReposVerbSucceedsInGitRepo() + { + this.Enlistment.IsScalarRepo.ShouldBeFalse(); + this.Enlistment.ReposAdd() + .Trim() + .ShouldEqual($"Successfully registered repo at '{this.Enlistment.EnlistmentRoot}'"); + } + } +} diff --git a/Scalar.FunctionalTests/Tests/GitRepoPerFixture/TestsWithGitRepoPerFixture.cs b/Scalar.FunctionalTests/Tests/GitRepoPerFixture/TestsWithGitRepoPerFixture.cs new file mode 100644 index 0000000000..6d9ae742c1 --- /dev/null +++ b/Scalar.FunctionalTests/Tests/GitRepoPerFixture/TestsWithGitRepoPerFixture.cs @@ -0,0 +1,29 @@ +using NUnit.Framework; +using Scalar.FunctionalTests.Tools; + +namespace Scalar.FunctionalTests.Tests.GitRepoPerFixture +{ + [TestFixture] + public class TestsWithGitRepoPerFixture + { + public ScalarFunctionalTestEnlistment Enlistment + { + get; private set; + } + + [OneTimeSetUp] + public virtual void CreateRepo() + { + this.Enlistment = ScalarFunctionalTestEnlistment.CloneGitRepo(ScalarTestConfig.PathToScalar); + } + + [OneTimeTearDown] + public virtual void DeleteRepo() + { + if (this.Enlistment != null) + { + this.Enlistment.DeleteAll(); + } + } + } +} diff --git a/Scalar.FunctionalTests/Tools/ScalarFunctionalTestEnlistment.cs b/Scalar.FunctionalTests/Tools/ScalarFunctionalTestEnlistment.cs index 3e363addb8..a3eb89e986 100644 --- a/Scalar.FunctionalTests/Tools/ScalarFunctionalTestEnlistment.cs +++ b/Scalar.FunctionalTests/Tools/ScalarFunctionalTestEnlistment.cs @@ -7,6 +7,7 @@ namespace Scalar.FunctionalTests.Tools { public class ScalarFunctionalTestEnlistment { + private const string GitRepoSrcDir = "repo"; private const string LockHeldByGit = "Scalar Lock: Held by {0}"; private const int SleepMSWaitingForStatusCheck = 100; private const int DefaultMaxWaitMSForStatusCheck = 5000; @@ -15,16 +16,30 @@ public class ScalarFunctionalTestEnlistment private ScalarProcess scalarProcess; - private ScalarFunctionalTestEnlistment(string pathToScalar, string enlistmentRoot, string repoUrl, string commitish, string localCacheRoot = null, bool fullClone = true) + private ScalarFunctionalTestEnlistment(string pathToScalar, string enlistmentRoot, string repoUrl, string commitish, string localCacheRoot = null, bool fullClone = true, bool isScalarRepo = true) { this.EnlistmentRoot = enlistmentRoot; this.RepoUrl = repoUrl; this.Commitish = commitish; this.fullClone = fullClone; + this.IsScalarRepo = isScalarRepo; + + if (isScalarRepo) + { + this.RepoRoot = Path.Combine(this.EnlistmentRoot, "src"); + } + else + { + this.RepoRoot = this.EnlistmentRoot; + } if (localCacheRoot == null) { - if (ScalarTestConfig.NoSharedCache) + if (!this.IsScalarRepo) + { + localCacheRoot = Path.Combine(this.RepoRoot, ".git", "objects"); + } + else if (ScalarTestConfig.NoSharedCache) { // eg C:\Repos\ScalarFunctionalTests\enlistment\7942ca69d7454acbb45ea39ef5be1d15\.scalar\.scalarCache localCacheRoot = GetRepoSpecificLocalCacheRoot(enlistmentRoot); @@ -38,6 +53,7 @@ private ScalarFunctionalTestEnlistment(string pathToScalar, string enlistmentRoo } this.LocalCacheRoot = localCacheRoot; + this.scalarProcess = new ScalarProcess(pathToScalar, this.EnlistmentRoot, this.LocalCacheRoot); } @@ -51,12 +67,11 @@ public string RepoUrl get; private set; } + public bool IsScalarRepo { get; } + public string LocalCacheRoot { get; } - public string RepoRoot - { - get { return Path.Combine(this.EnlistmentRoot, "src"); } - } + public string RepoRoot { get; } public string ScalarLogsRoot { @@ -81,21 +96,21 @@ public static ScalarFunctionalTestEnlistment CloneWithPerRepoCache(string pathTo } public static ScalarFunctionalTestEnlistment Clone( - string pathToGvfs, + string pathToScalar, string commitish = null, string localCacheRoot = null, bool skipFetchCommitsAndTrees = false, bool fullClone = true) { string enlistmentRoot = ScalarFunctionalTestEnlistment.GetUniqueEnlistmentRoot(); - return Clone(pathToGvfs, enlistmentRoot, commitish, localCacheRoot, skipFetchCommitsAndTrees, fullClone); + return Clone(pathToScalar, enlistmentRoot, commitish, localCacheRoot, skipFetchCommitsAndTrees, fullClone); } - public static ScalarFunctionalTestEnlistment CloneEnlistmentWithSpacesInPath(string pathToGvfs, string commitish = null) + public static ScalarFunctionalTestEnlistment CloneEnlistmentWithSpacesInPath(string pathToScalar, string commitish = null) { string enlistmentRoot = ScalarFunctionalTestEnlistment.GetUniqueEnlistmentRootWithSpaces(); string localCache = ScalarFunctionalTestEnlistment.GetRepoSpecificLocalCacheRoot(enlistmentRoot); - return Clone(pathToGvfs, enlistmentRoot, commitish, localCache); + return Clone(pathToScalar, enlistmentRoot, commitish, localCache); } public static string GetUniqueEnlistmentRoot() @@ -108,6 +123,32 @@ public static string GetUniqueEnlistmentRootWithSpaces() return Path.Combine(Properties.Settings.Default.EnlistmentRoot, "test " + Guid.NewGuid().ToString("N").Substring(0, 15)); } + public static ScalarFunctionalTestEnlistment CloneGitRepo(string pathToScalar) + { + string enlistmentRoot = Path.Combine(GetUniqueEnlistmentRoot(), GitRepoSrcDir); + + ScalarFunctionalTestEnlistment enlistment = new ScalarFunctionalTestEnlistment( + pathToScalar, + enlistmentRoot, + ScalarTestConfig.RepoToClone, + Properties.Settings.Default.Commitish, + ScalarTestConfig.LocalCacheRoot, + isScalarRepo: false); + + try + { + enlistment.CloneGitRepo(); + } + catch (Exception e) + { + Console.WriteLine($"Unhandled exception in {nameof(ScalarFunctionalTestEnlistment.Clone)}: " + e.ToString()); + TestResultsHelper.OutputScalarLogs(enlistment); + throw; + } + + return enlistment; + } + public string GetPackRoot(FileSystemRunner fileSystem) { return Path.Combine(ScalarHelpers.GetObjectsRootFromGitConfig(this.RepoRoot), "pack"); @@ -135,29 +176,20 @@ public void DeleteEnlistment() public void Clone(bool skipFetchCommitsAndTrees) { this.scalarProcess.Clone(this.RepoUrl, this.Commitish, skipFetchCommitsAndTrees, fullClone: this.fullClone); + this.InitializeConfig(); + } - GitProcess.Invoke(this.RepoRoot, "checkout " + this.Commitish); - GitProcess.Invoke(this.RepoRoot, "branch --unset-upstream"); - GitProcess.Invoke(this.RepoRoot, "config core.abbrev 40"); - GitProcess.Invoke(this.RepoRoot, "config user.name \"Functional Test User\""); - GitProcess.Invoke(this.RepoRoot, "config user.email \"functional@test.com\""); - - // If this repository has a .gitignore file in the root directory, force it to be - // hydrated. This is because if the GitStatusCache feature is enabled, it will run - // a "git status" command asynchronously, which will hydrate the .gitignore file - // as it reads the ignore rules. Hydrate this file here so that it is consistently - // hydrated and there are no race conditions depending on when / if it is hydrated - // as part of an asynchronous status scan to rebuild the GitStatusCache. - string rootGitIgnorePath = Path.Combine(this.RepoRoot, ".gitignore"); - if (File.Exists(rootGitIgnorePath)) - { - File.ReadAllBytes(rootGitIgnorePath); - } + public void CloneGitRepo() + { + string workDir = Directory.GetParent(this.RepoRoot).FullName; + Directory.CreateDirectory(workDir); + GitProcess.Invoke(workDir, $"clone {this.RepoUrl} {GitRepoSrcDir}"); + this.InitializeConfig(); } - public string FetchCommitsAndTrees(bool failOnError = true, string standardInput = null) + public string FetchStep(bool failOnError = true, string standardInput = null) { - return this.scalarProcess.FetchCommitsAndTrees(failOnError, standardInput); + return this.scalarProcess.FetchStep(failOnError, standardInput); } public void UnregisterRepo() @@ -195,6 +227,11 @@ public string Status(string trace = null) return this.scalarProcess.Status(trace); } + public string ReposAdd() + { + return this.scalarProcess.ReposAdd(this.EnlistmentRoot); + } + public string GetCacheServer() { return this.scalarProcess.CacheServer("--get"); @@ -231,17 +268,28 @@ public string GetObjectPathTo(string objectHash) objectHash.Substring(2)); } + private void InitializeConfig() + { + GitProcess.Invoke(this.RepoRoot, "checkout " + this.Commitish); + GitProcess.Invoke(this.RepoRoot, "branch --unset-upstream"); + GitProcess.Invoke(this.RepoRoot, "config core.abbrev 40"); + GitProcess.Invoke(this.RepoRoot, "config user.name \"Functional Test User\""); + GitProcess.Invoke(this.RepoRoot, "config user.email \"functional@test.com\""); + } + private static ScalarFunctionalTestEnlistment Clone( - string pathToGvfs, + string pathToScalar, string enlistmentRoot, string commitish, string localCacheRoot, bool skipFetchCommitsAndTrees = false, bool fullClone = true) { + enlistmentRoot = enlistmentRoot ?? GetUniqueEnlistmentRoot(); + ScalarFunctionalTestEnlistment enlistment = new ScalarFunctionalTestEnlistment( - pathToGvfs, - enlistmentRoot ?? GetUniqueEnlistmentRoot(), + pathToScalar, + enlistmentRoot, ScalarTestConfig.RepoToClone, commitish ?? Properties.Settings.Default.Commitish, localCacheRoot ?? ScalarTestConfig.LocalCacheRoot, diff --git a/Scalar.FunctionalTests/Tools/ScalarProcess.cs b/Scalar.FunctionalTests/Tools/ScalarProcess.cs index dc8a539a98..d92316d31d 100644 --- a/Scalar.FunctionalTests/Tools/ScalarProcess.cs +++ b/Scalar.FunctionalTests/Tools/ScalarProcess.cs @@ -56,10 +56,10 @@ public bool TryMount(out string output) return this.IsEnlistmentMounted(); } - public string FetchCommitsAndTrees(bool failOnError, string standardInput = null) + public string FetchStep(bool failOnError, string standardInput = null) { return this.CallScalar( - $"maintenance \"{this.enlistmentRoot}\" --task fetch-commits-and-trees", + $"maintenance \"{this.enlistmentRoot}\" --task fetch", failOnError ? SuccessExitCode : DoNotCheckExitCode, standardInput: standardInput); } @@ -184,6 +184,11 @@ private string CallScalar( processInfo.Arguments = args + " " + TestConstants.InternalUseOnlyFlag + " " + internalParameter; + if (!string.IsNullOrEmpty(workingDirectory)) + { + processInfo.WorkingDirectory = workingDirectory; + } + processInfo.WindowStyle = ProcessWindowStyle.Hidden; processInfo.UseShellExecute = false; processInfo.RedirectStandardOutput = true; diff --git a/Scalar/CommandLine/CloneVerb.cs b/Scalar/CommandLine/CloneVerb.cs index ecbdeba0eb..c1f0d4c41f 100644 --- a/Scalar/CommandLine/CloneVerb.cs +++ b/Scalar/CommandLine/CloneVerb.cs @@ -261,7 +261,7 @@ private Result DoClone(string fullEnlistmentRootPathParameter, string normalized this.enlistment, verb => { - verb.MaintenanceTask = ScalarConstants.VerbParameters.Maintenance.FetchCommitsAndTreesTaskName; + verb.MaintenanceTask = ScalarConstants.VerbParameters.Maintenance.FetchTaskName; verb.SkipVersionCheck = true; verb.ResolvedCacheServer = this.cacheServer; verb.ServerScalarConfig = this.serverScalarConfig; diff --git a/Scalar/CommandLine/MaintenanceVerb.cs b/Scalar/CommandLine/MaintenanceVerb.cs index 1f9727de71..e3a6c06033 100644 --- a/Scalar/CommandLine/MaintenanceVerb.cs +++ b/Scalar/CommandLine/MaintenanceVerb.cs @@ -91,7 +91,7 @@ protected override void Execute(ScalarEnlistment enlistment) this.PackfileMaintenanceBatchSize)).Execute(); return; - case ScalarConstants.VerbParameters.Maintenance.FetchCommitsAndTreesTaskName: + case ScalarConstants.VerbParameters.Maintenance.FetchTaskName: this.FailIfBatchSizeSet(tracer); this.FetchCommitsAndTrees(tracer, enlistment, cacheServerUrl); return; @@ -140,15 +140,20 @@ protected override void Execute(ScalarEnlistment enlistment) private void FetchCommitsAndTrees(ITracer tracer, ScalarEnlistment enlistment, string cacheServerUrl) { - GitObjectsHttpRequestor objectRequestor; - CacheServerInfo cacheServer; - this.InitializeServerConnection( - tracer, - enlistment, - cacheServerUrl, - out objectRequestor, - out cacheServer); - this.RunFetchCommitsAndTreesStep(tracer, enlistment, objectRequestor, cacheServer); + GitObjectsHttpRequestor objectRequestor = null; + CacheServerInfo cacheServer = null; + + if (enlistment.UsesGvfsProtocol) + { + this.InitializeServerConnection( + tracer, + enlistment, + cacheServerUrl, + out objectRequestor, + out cacheServer); + } + + this.RunFetchStep(tracer, enlistment, objectRequestor, cacheServer); } private void FailIfBatchSizeSet(ITracer tracer) @@ -201,7 +206,7 @@ private void InitializeServerConnection( objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig); } - private void RunFetchCommitsAndTreesStep(ITracer tracer, ScalarEnlistment enlistment, GitObjectsHttpRequestor objectRequestor, CacheServerInfo cacheServer) + private void RunFetchStep(ITracer tracer, ScalarEnlistment enlistment, GitObjectsHttpRequestor objectRequestor, CacheServerInfo cacheServer) { bool success; string error = string.Empty; @@ -210,17 +215,22 @@ private void RunFetchCommitsAndTreesStep(ITracer tracer, ScalarEnlistment enlist GitObjects gitObjects = new GitObjects(tracer, enlistment, objectRequestor, fileSystem); success = this.ShowStatusWhileRunning( - () => new FetchCommitsAndTreesStep(context, gitObjects, requireCacheLock: false).TryFetchCommitsAndTrees(out error), - "Fetching commits and trees " + this.GetCacheServerDisplay(cacheServer, enlistment.RepoUrl)); + () => new FetchStep(context, gitObjects, requireCacheLock: false, forceRun: !this.StartedByService).TryFetch(out error), + "Fetching " + this.GetCacheServerDisplay(cacheServer, enlistment.RepoUrl)); if (!success) { - this.ReportErrorAndExit(tracer, ReturnCode.GenericError, "Fetching commits and trees failed: " + error); + this.ReportErrorAndExit(tracer, ReturnCode.GenericError, "Fetch failed: " + error); } } private string GetCacheServerDisplay(CacheServerInfo cacheServer, string repoUrl) { + if (cacheServer == null) + { + return "from remotes"; + } + if (!cacheServer.IsNone(repoUrl)) { return "from cache server"; diff --git a/Scalar/CommandLine/ScalarVerb.cs b/Scalar/CommandLine/ScalarVerb.cs index e6218156de..b7b4194c6a 100644 --- a/Scalar/CommandLine/ScalarVerb.cs +++ b/Scalar/CommandLine/ScalarVerb.cs @@ -610,12 +610,17 @@ protected void InitializeCachePaths( this.ReportErrorAndExit("Failed to determine git objects root from git config: " + error); } + string localCacheRoot; if (string.IsNullOrWhiteSpace(gitObjectsRoot)) { - this.ReportErrorAndExit(tracer, "Invalid git objects root (empty or whitespace)"); + // We do not have an object cache. This is a vanilla Git repo! + localCacheRoot = enlistment.LocalObjectsRoot; + gitObjectsRoot = enlistment.LocalObjectsRoot; + } + else + { + localCacheRoot = Path.GetDirectoryName(gitObjectsRoot); } - - string localCacheRoot = Path.GetDirectoryName(gitObjectsRoot); if (string.IsNullOrWhiteSpace(localCacheRoot)) {