Merge pull request #275: Maintain vanilla Git repos

Replaces #258. Resolves #254.

If you have a "vanilla" Git repo, then run `scalar repos add` in that repo. Scalar will detect the repo root, register that root with the service, and the maintenance steps will run.

The way we identify a vanilla Git repo (now) is if the object cache dir is not given by config and hence we do all operations on the local `.git` folder. This will not work with Git worktrees, but we will also not discover a `.git` _folder_ (worktrees use a `.git` _file_) so the enlistment will be invalid.

* The `commit-graph` task will generate the incremental commit-graph files using the `--reachable` option. This will be a no-op if the user has not changed their refs.

* The `packfile` task will write a `multi-pack-index` and do the expire/repack logic. To actually do something, the batch-size is decreased if the total pack size is smaller than 2gb. (This was done in #272.)

* The `loose-objects` task will delete loose objects already in pack-files and put them into a new `from-loose` pack. This required no change.

* The `fetch-commits-and-trees` task is renamed the `fetch` task. On a vanilla Git repo, it will simply call `git fetch <remote> --no-update-remote-refs +refs/head/*:refs/hidden/<remote>/*` for each `<remote>` given by `git remote`. This downloads a pack-file with the new objects not reachable form the current refs, but also does not update `refs/remotes/`. The user will see their refs update as normal, but the pack download is much smaller.

* The `config` task (added by #272) runs the necessary Git config.  This step is run in `scalar repos add`, but also in the background with the service. All config options become optional for vanilla Git repos, so a user can opt-out of the settings we recommend.

Scalar.Common/Git/GitProcess.cs

@@ -424,6 +424,23 @@ public Result ForceCheckoutAllFiles()
             return this.InvokeGitInWorkingDirectoryRoot("checkout HEAD -- .", fetchMissingObjects: true);
         }
 
+        public Result BackgroundFetch(string remote)
+        {
+            // By using "--no-update-remote-refs", the user will see their remote refs update
+            // normally when they do a foreground fetch.
+            // By using this refspec, we do not create local refs, but instead store them in the "hidden"
+            // namespace. These refs are never visible to the user (unless they open the .git/refs dir)
+            // but still allow us to run reachability questions like updating the commit-graph.
+            return this.InvokeGitInWorkingDirectoryRoot($"fetch {remote} --quiet --no-update-remote-refs +refs/heads/*:refs/{ScalarConstants.DotGit.Refs.Hidden.Name}/{remote}/*", fetchMissingObjects: true);
+        }
+
+        public string[] GetRemotes()
+        {
+            return this.InvokeGitInWorkingDirectoryRoot("remote", fetchMissingObjects: false)
+                       .Output
+                       .Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
+        }
+
         public Result SparseCheckoutSet(List<string> foldersToSet)
         {
             return this.InvokeGitInWorkingDirectoryRoot(

Scalar.Common/Maintenance/ConfigStep.cs

@@ -83,6 +83,11 @@ public bool TrySetConfig(out string error)
             string expectedHooksPath = Path.Combine(this.Context.Enlistment.WorkingDirectoryRoot, ScalarConstants.DotGit.Hooks.Root);
             expectedHooksPath = Paths.ConvertPathToGitFormat(expectedHooksPath);
 
+            if (!this.UseGvfsProtocol.HasValue)
+            {
+                this.UseGvfsProtocol = this.Context.Enlistment.UsesGvfsProtocol;
+            }
+
             // These settings are required for normal Scalar functionality.
             // They will override any existing local configuration values.
             //
@@ -131,7 +136,9 @@ public bool TrySetConfig(out string error)
                 requiredSettings.Add("http.sslBackend", "schannel");
             }
 
-            if (!this.TrySetConfig(requiredSettings, isRequired: true, out error))
+            // If we do not use the GVFS protocol, then these config settings
+            // are in fact optional.
+            if (!this.TrySetConfig(requiredSettings, isRequired: this.UseGvfsProtocol.Value, out error))
             {
                 error = $"Failed to set some required settings: {error}";
                 this.Context.Tracer.RelatedError(error);

Scalar.Common/Maintenance/FetchCommitsAndTreesStep.cs → Scalar.Common/Maintenance/FetchStep.cs

@@ -9,34 +9,97 @@
 
 namespace Scalar.Common.Maintenance
 {
-    public class FetchCommitsAndTreesStep : GitMaintenanceStep
+    public class FetchStep : GitMaintenanceStep
     {
         private const int IoFailureRetryDelayMS = 50;
         private const int LockWaitTimeMs = 100;
         private const int WaitingOnLockLogThreshold = 50;
         private const string FetchCommitsAndTreesLock = "fetch-commits-trees.lock";
+        private const string FetchTimeFile = "fetch.time";
         private readonly TimeSpan timeBetweenFetches = TimeSpan.FromMinutes(70);
         private readonly TimeSpan timeBetweenFetchesNoCacheServer = TimeSpan.FromDays(1);
+        private readonly bool forceRun;
 
-        public FetchCommitsAndTreesStep(ScalarContext context, GitObjects gitObjects, bool requireCacheLock = true)
+        public FetchStep(
+                    ScalarContext context,
+                    GitObjects gitObjects,
+                    bool requireCacheLock = true,
+                    bool forceRun = false)
             : base(context, requireCacheLock)
         {
             this.GitObjects = gitObjects;
+            this.forceRun = forceRun;
         }
 
         public override string Area => "FetchCommitsAndTreesStep";
 
+        // Used only for vanilla Git repos
+        protected override TimeSpan TimeBetweenRuns => this.timeBetweenFetches;
+
         protected GitObjects GitObjects { get; }
 
-        public bool TryFetchCommitsAndTrees(out string error, GitProcess gitProcess = null)
+        public bool TryFetch(out string error, GitProcess gitProcess = null)
         {
             if (gitProcess == null)
             {
                 gitProcess = new GitProcess(this.Context.Enlistment);
             }
 
+            if (this.Context.Enlistment.UsesGvfsProtocol)
+            {
+                return this.TryFetchUsingGvfsProtocol(gitProcess, out error);
+            }
+            else
+            {
+                return this.TryFetchUsingGitProtocol(gitProcess, out error);
+            }
+
+        }
+
+        protected override void PerformMaintenance()
+        {
+            string error = null;
+
+            this.RunGitCommand(
+                process =>
+                {
+                    this.TryFetch(out error, process);
+                    return null;
+                },
+                nameof(this.TryFetch));
+
+            if (!string.IsNullOrEmpty(error))
+            {
+                this.Context.Tracer.RelatedWarning(
+                    metadata: this.CreateEventMetadata(),
+                    message: $"{nameof(this.TryFetch)} failed with error '{error}'",
+                    keywords: Keywords.Telemetry);
+            }
+        }
+
+        private bool TryFetchUsingGvfsProtocol(GitProcess gitProcess, out string error)
+        {
+            if (!this.TryGetMaxGoodPrefetchPackTimestamp(out long last, out error))
+            {
+                this.Context.Tracer.RelatedError(error);
+                return false;
+            }
+
+            TimeSpan timeBetween = this.GitObjects.IsUsingCacheServer()
+                                    ? this.timeBetweenFetches
+                                    : this.timeBetweenFetchesNoCacheServer;
+
+            DateTime lastDateTime = EpochConverter.FromUnixEpochSeconds(last);
+            DateTime now = DateTime.UtcNow;
+            if (!this.forceRun && now <= lastDateTime + timeBetween)
+            {
+                this.Context.Tracer.RelatedInfo(this.Area + ": Skipping fetch since most-recent fetch ({0}) is too close to now ({1})", lastDateTime, now);
+                error = null;
+                return true;
+            }
+
             // We take our own lock here to keep background and foreground fetches
-            // (i.e. a user running 'scalar maintenance --fetch-commits-and-trees')
+            // (i.e. a user running 'scalar maintenance --task fetch')
             // from running at the same time.
             using (FileBasedLock fetchLock = ScalarPlatform.Instance.CreateFileBasedLock(
                 this.Context.FileSystem,
@@ -63,43 +126,50 @@ public bool TryFetchCommitsAndTrees(out string error, GitProcess gitProcess = nu
             return true;
         }
 
-        protected override void PerformMaintenance()
+        private bool TryFetchUsingGitProtocol(GitProcess gitProcess, out string error)
         {
-            long last;
-            string error = null;
+            this.LastRunTimeFilePath = Path.Combine(this.Context.Enlistment.ScalarLogsRoot, FetchTimeFile);
 
-            if (!this.TryGetMaxGoodPrefetchPackTimestamp(out last, out error))
+            if (!this.forceRun && !this.EnoughTimeBetweenRuns())
             {
-                this.Context.Tracer.RelatedError(error);
-                return;
+                this.Context.Tracer.RelatedInfo($"Skipping {nameof(FetchStep)} due to not enough time between runs");
+                error = null;
+                return true;
             }
 
-            TimeSpan timeBetween = this.GitObjects.IsUsingCacheServer()
-                                    ? this.timeBetweenFetches
-                                    : this.timeBetweenFetchesNoCacheServer;
-
-            DateTime lastDateTime = EpochConverter.FromUnixEpochSeconds(last);
-            DateTime now = DateTime.UtcNow;
-            if (now <= lastDateTime + timeBetween)
+            using (ITracer activity = this.Context.Tracer.StartActivity(nameof(GitProcess.BackgroundFetch), EventLevel.LogAlways))
             {
-                this.Context.Tracer.RelatedInfo(this.Area + ": Skipping fetch since most-recent fetch ({0}) is too close to now ({1})", lastDateTime, now);
-                return;
-            }
+                // Clear hidden refs to avoid arbitrarily large growth
+                string hiddenRefspace = Path.Combine(this.Context.Enlistment.WorkingDirectoryRoot, ScalarConstants.DotGit.Refs.Hidden.Root);
 
-            this.RunGitCommand(
-                process =>
+                if (this.Context.FileSystem.DirectoryExists(hiddenRefspace))
                 {
-                    this.TryFetchCommitsAndTrees(out error, process);
-                    return null;
-                },
-                nameof(this.TryFetchCommitsAndTrees));
+                    this.Context.FileSystem.DeleteDirectory(hiddenRefspace, recursive: true);
+                }
 
-            if (!string.IsNullOrEmpty(error))
-            {
-                this.Context.Tracer.RelatedWarning(
-                    metadata: this.CreateEventMetadata(),
-                    message: $"{nameof(this.TryFetchCommitsAndTrees)} failed with error '{error}'",
-                    keywords: Keywords.Telemetry);
+                string[] remotes = gitProcess.GetRemotes();
+                bool response = true;
+
+                error = "";
+                foreach (string remote in remotes)
+                {
+                    GitProcess.Result result = gitProcess.BackgroundFetch(remote);
+
+                    if (!string.IsNullOrWhiteSpace(result.Errors))
+                    {
+                        error += result.Errors;
+                        activity.RelatedError($"Background fetch from '{remote}' completed with stderr: {result.Errors}");
+                    }
+
+                    if (result.ExitCodeIsFailure)
+                    {
+                        response = false;
+                        // Keep going through other remotes, but the overall result will still be false.
+                    }
+                }
+
+                this.SaveLastRunTimeToFile();
+                return response;
             }
         }
 

Scalar.Common/Maintenance/MaintenanceTasks.cs

@@ -19,7 +19,7 @@ public static string GetVerbTaskName(Task task)
             switch (task)
             {
                 case Task.FetchCommitsAndTrees:
-                    return ScalarConstants.VerbParameters.Maintenance.FetchCommitsAndTreesTaskName;
+                    return ScalarConstants.VerbParameters.Maintenance.FetchTaskName;
                 case Task.LooseObjects:
                     return ScalarConstants.VerbParameters.Maintenance.LooseObjectsTaskName;
                 case Task.PackFiles:

Scalar.Common/Maintenance/PackfileMaintenanceStep.cs

@@ -110,7 +110,7 @@ protected override void PerformMaintenance()
 
                 this.GetPackFilesInfo(out int beforeCount, out long beforeSize, out _, out bool hasKeep);
 
-                if (!hasKeep)
+                if (!hasKeep && this.Context.Enlistment.UsesGvfsProtocol)
                 {
                     activity.RelatedWarning(this.CreateEventMetadata(), "Skipping pack maintenance due to no .keep file.");
                     return;

Scalar.Common/ScalarConstants.cs

@@ -166,6 +166,12 @@ public static class Pack
             public static class Refs
             {
                 public static readonly string Root = Path.Combine(DotGit.Root, "refs");
+
+                public static class Hidden
+                {
+                    public static readonly string Name = "hidden";
+                    public static readonly string Root = Path.Combine(DotGit.Refs.Root, Name);
+                }
             }
         }
 
@@ -194,7 +200,7 @@ public static class Maintenance
             {
                 public const string Task = "task";
 
-                public const string FetchCommitsAndTreesTaskName = "fetch-commits-and-trees";
+                public const string FetchTaskName = "fetch";
                 public const string LooseObjectsTaskName = "loose-objects";
                 public const string PackFilesTaskName = "pack-files";
                 public const string CommitGraphTaskName = "commit-graph";

Scalar.Common/ScalarEnlistment.cs

@@ -43,6 +43,8 @@ private ScalarEnlistment(string enlistmentRoot, string workingDirectory, string
         public override string LocalObjectsRoot { get; protected set; }
         public override string GitPackRoot { get; protected set; }
 
+        public bool UsesGvfsProtocol { get; protected set; }
+
         // These version properties are only used in logging during clone and mount to track version numbers
         public string GitVersion
         {
@@ -186,6 +188,9 @@ public void InitializeCachePaths(string localCacheRoot, string gitObjectsRoot)
             this.LocalCacheRoot = localCacheRoot;
             this.GitObjectsRoot = gitObjectsRoot;
             this.GitPackRoot = Path.Combine(this.GitObjectsRoot, ScalarConstants.DotGit.Objects.Pack.Name);
+
+            // When using the GVFS protocol, we have a different cache location than local objects.
+            this.UsesGvfsProtocol = !this.LocalCacheRoot.Equals(this.LocalObjectsRoot);
         }
 
         public bool TryCreateEnlistmentFolders()

Scalar.FunctionalTests/Categories.cs

@@ -8,6 +8,8 @@ public static class Categories
         public const string WindowsOnly = "WindowsOnly";
         public const string MacOnly = "MacOnly";
 
+        public const string GitRepository = "GitRepository";
+
         public const string NeedsUpdatesForNonVirtualizedMode = "NeedsUpdatesForNonVirtualizedMode";
 
         public static class MacTODO

Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchCommitsAndTreesStepTests.cs → Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchStepTests.cs

@@ -11,22 +11,22 @@ namespace Scalar.FunctionalTests.Tests.EnlistmentPerFixture
 {
     [TestFixture]
     [NonParallelizable]
-    public class FetchCommitsAndTreesStepTests : TestsWithEnlistmentPerFixture
+    public class FetchStepTests : TestsWithEnlistmentPerFixture
     {
         private const string FetchCommitsAndTreesLock = "fetch-commits-trees.lock";
 
         private FileSystemRunner fileSystem;
 
-        public FetchCommitsAndTreesStepTests()
+        public FetchStepTests()
         {
             this.fileSystem = new SystemIORunner();
         }
 
         [TestCase]
         [Category(Categories.MacTODO.TestNeedsToLockFile)]
-        public void FetchCommitsAndTreesCleansUpStaleFetchLock()
+        public void FetchStepCleansUpStaleFetchLock()
         {
-            this.Enlistment.FetchCommitsAndTrees();
+            this.Enlistment.FetchStep();
             string fetchCommitsLockFile = Path.Combine(
                 ScalarHelpers.GetObjectsRootFromGitConfig(this.Enlistment.RepoRoot),
                 "pack",
@@ -42,7 +42,7 @@ public void FetchCommitsAndTreesCleansUpStaleFetchLock()
                 .Count()
                 .ShouldEqual(1, "Incorrect number of .keep files in pack directory");
 
-            this.Enlistment.FetchCommitsAndTrees();
+            this.Enlistment.FetchStep();
             fetchCommitsLockFile.ShouldNotExistOnDisk(this.fileSystem);
         }
     }