Merge pull request #303: Pause maintenance between repos during same …

…task

The `PauseVerb` was added along with a lot of other things in #292. It creates a file storing a timestamp for marking how long to pause maintenance. As noted in #284, this was inadequate. If there are a lot of enlistments and a user runs `scalar pause` in the middle of iterating over those enlistments, the maintenance task will continue to run over all of those enlistments.

Now, stop before triggering a new `scalar run <task>` process. Modify the markdown with the details for how many maintenance tasks completed and whether or not it was halted due to a pause. Here is a formatted version of a trace event for a task during pause:

```
[2020-01-22 17:17:57.7293 -05:00] RunMaintenanceTaskForRepos_MaintenanceSummary 
{
"task":"loose-objects",
"UserId":"S-1-5-21-4082248826-4152932973-4176629329-1001",
"SessionId":1,
"traceMessage":"Maintenance is paused until 1/23/2020 4:14:44 AM.",
"reposInRegistryForUser":11,
"reposSkipped":11,
"reposSuccessfullyRemoved":0,
"repoRemovalFailures":0,
"reposMaintained":0,
"haltedForPause":true
}
```

Scalar.Service/MaintenanceTaskScheduler.cs

@@ -118,30 +118,15 @@ public void ScheduleRecurringTasks()
                 this.taskTimers.Add(new Timer(
                 (state) =>
                 {
-                    if (!schedule.IgnorePause)
-                    {
-                        if (this.repoRegistry.TryGetMaintenanceDelayTime(out DateTime time))
-                        {
-                            if (time.CompareTo(DateTime.Now) > 0)
-                            {
-                                this.tracer.RelatedInfo($"Maintenance is paused until {time}.");
-                                return;
-                            }
-                            else if (!this.repoRegistry.TryRemovePauseFile(out string error))
-                            {
-                                this.tracer.RelatedWarning($"Failed to remove pause file: {error}");
-                            }
-                        }
-                    }
-
                     this.taskQueue.TryEnqueue(
                         new MaintenanceTask(
                             this.tracer,
                             this.fileSystem,
                             this.scalarVerb,
                             this.repoRegistry,
                             this,
-                            schedule.Task));
+                            schedule.Task,
+                            schedule.IgnorePause));
                 },
                 state: null,
                 dueTime: schedule.DueTime,
@@ -157,21 +142,24 @@ internal class MaintenanceTask : IServiceTask
             private readonly IScalarRepoRegistry repoRegistry;
             private readonly IRegisteredUserStore registeredUserStore;
             private readonly MaintenanceTasks.Task task;
+            private readonly bool ignorePause;
 
             public MaintenanceTask(
                 ITracer tracer,
                 PhysicalFileSystem fileSystem,
                 IScalarVerbRunner scalarVerb,
                 IScalarRepoRegistry repoRegistry,
                 IRegisteredUserStore registeredUserStore,
-                MaintenanceTasks.Task task)
+                MaintenanceTasks.Task task,
+                bool ignorePause = true)
             {
                 this.tracer = tracer;
                 this.fileSystem = fileSystem;
                 this.scalarVerb = scalarVerb;
                 this.repoRegistry = repoRegistry;
                 this.registeredUserStore = registeredUserStore;
                 this.task = task;
+                this.ignorePause = ignorePause;
             }
 
             public void Execute()
@@ -212,16 +200,27 @@ private void RunMaintenanceTaskForRepos(UserAndSession registeredUser)
                 int repoRemovalFailures = 0;
                 int reposMaintained = 0;
                 int reposInRegistryForUser = 0;
+                bool maintenancePaused = false;
 
                 string rootPath;
                 string errorMessage;
+                string traceMessage = null;
 
                 IEnumerable<ScalarRepoRegistration> reposForUser = this.repoRegistry.GetRegisteredRepos().Where(
                     x => x.UserId.Equals(registeredUser.UserId, StringComparison.InvariantCultureIgnoreCase));
 
                 foreach (ScalarRepoRegistration repoRegistration in reposForUser)
                 {
                     ++reposInRegistryForUser;
+
+                    if (maintenancePaused || this.IsMaintenancePaused(out traceMessage))
+                    {
+                        metadata[nameof(traceMessage)] = traceMessage;
+                        maintenancePaused = true;
+                        ++reposSkipped;
+                        continue;
+                    }
+
                     rootPath = Path.GetPathRoot(repoRegistration.NormalizedRepoRoot);
 
                     metadata[nameof(repoRegistration.NormalizedRepoRoot)] = repoRegistration.NormalizedRepoRoot;
@@ -281,11 +280,38 @@ private void RunMaintenanceTaskForRepos(UserAndSession registeredUser)
                 metadata.Add(nameof(reposSuccessfullyRemoved), reposSuccessfullyRemoved);
                 metadata.Add(nameof(repoRemovalFailures), repoRemovalFailures);
                 metadata.Add(nameof(reposMaintained), reposMaintained);
+                metadata.Add(nameof(maintenancePaused), maintenancePaused);
                 this.tracer.RelatedEvent(
                     EventLevel.Informational,
                     $"{nameof(this.RunMaintenanceTaskForRepos)}_MaintenanceSummary",
                     metadata);
             }
+
+            private bool IsMaintenancePaused(out string traceMessage)
+            {
+                if (this.ignorePause)
+                {
+                    traceMessage = null;
+                    return false;
+                }
+
+                if (this.repoRegistry.TryGetMaintenanceDelayTime(out DateTime time))
+                {
+                    if (time.CompareTo(DateTime.Now) > 0)
+                    {
+                        traceMessage = $"Maintenance is paused until {time}.";
+                        return true;
+                    }
+                    else if (!this.repoRegistry.TryRemovePauseFile(out string innerError))
+                    {
+                        traceMessage = $"Failed to remove pause file: {innerError}";
+                        return false;
+                    }
+                }
+
+                traceMessage = null;
+                return false;
+            }
         }
 
         private class MaintenanceSchedule