From 6a2f37a973d04550f19ab39c345092f774814634 Mon Sep 17 00:00:00 2001
From: Derrick Stolee <stolee@gmail.com>
Date: Wed, 22 Jan 2020 16:36:08 -0500
Subject: [PATCH] Pause maintenance between repos during same task

Signed-off-by: Derrick Stolee <stolee@gmail.com>
---
 Scalar.Service/MaintenanceTaskScheduler.cs | 62 +++++++++++++++-------
 1 file changed, 44 insertions(+), 18 deletions(-)

diff --git a/Scalar.Service/MaintenanceTaskScheduler.cs b/Scalar.Service/MaintenanceTaskScheduler.cs
index 81fe62dc7c..1089e42d76 100644
--- a/Scalar.Service/MaintenanceTaskScheduler.cs
+++ b/Scalar.Service/MaintenanceTaskScheduler.cs
@@ -118,22 +118,6 @@ public void ScheduleRecurringTasks()
                 this.taskTimers.Add(new Timer(
                 (state) =>
                 {
-                    if (!schedule.IgnorePause)
-                    {
-                        if (this.repoRegistry.TryGetMaintenanceDelayTime(out DateTime time))
-                        {
-                            if (time.CompareTo(DateTime.Now) > 0)
-                            {
-                                this.tracer.RelatedInfo($"Maintenance is paused until {time}.");
-                                return;
-                            }
-                            else if (!this.repoRegistry.TryRemovePauseFile(out string error))
-                            {
-                                this.tracer.RelatedWarning($"Failed to remove pause file: {error}");
-                            }
-                        }
-                    }
-
                     this.taskQueue.TryEnqueue(
                         new MaintenanceTask(
                             this.tracer,
@@ -141,7 +125,8 @@ public void ScheduleRecurringTasks()
                             this.scalarVerb,
                             this.repoRegistry,
                             this,
-                            schedule.Task));
+                            schedule.Task,
+                            schedule.IgnorePause));
                 },
                 state: null,
                 dueTime: schedule.DueTime,
@@ -157,6 +142,7 @@ internal class MaintenanceTask : IServiceTask
             private readonly IScalarRepoRegistry repoRegistry;
             private readonly IRegisteredUserStore registeredUserStore;
             private readonly MaintenanceTasks.Task task;
+            private readonly bool ignorePause;
 
             public MaintenanceTask(
                 ITracer tracer,
@@ -164,7 +150,8 @@ public MaintenanceTask(
                 IScalarVerbRunner scalarVerb,
                 IScalarRepoRegistry repoRegistry,
                 IRegisteredUserStore registeredUserStore,
-                MaintenanceTasks.Task task)
+                MaintenanceTasks.Task task,
+                bool ignorePause = true)
             {
                 this.tracer = tracer;
                 this.fileSystem = fileSystem;
@@ -172,6 +159,7 @@ public MaintenanceTask(
                 this.repoRegistry = repoRegistry;
                 this.registeredUserStore = registeredUserStore;
                 this.task = task;
+                this.ignorePause = ignorePause;
             }
 
             public void Execute()
@@ -212,9 +200,11 @@ private void RunMaintenanceTaskForRepos(UserAndSession registeredUser)
                 int repoRemovalFailures = 0;
                 int reposMaintained = 0;
                 int reposInRegistryForUser = 0;
+                bool maintenancePaused = false;
 
                 string rootPath;
                 string errorMessage;
+                string traceMessage = null;
 
                 IEnumerable<ScalarRepoRegistration> reposForUser = this.repoRegistry.GetRegisteredRepos().Where(
                     x => x.UserId.Equals(registeredUser.UserId, StringComparison.InvariantCultureIgnoreCase));
@@ -222,6 +212,15 @@ private void RunMaintenanceTaskForRepos(UserAndSession registeredUser)
                 foreach (ScalarRepoRegistration repoRegistration in reposForUser)
                 {
                     ++reposInRegistryForUser;
+
+                    if (maintenancePaused || this.IsMaintenancePaused(out traceMessage))
+                    {
+                        metadata[nameof(traceMessage)] = traceMessage;
+                        maintenancePaused = true;
+                        ++reposSkipped;
+                        continue;
+                    }
+
                     rootPath = Path.GetPathRoot(repoRegistration.NormalizedRepoRoot);
 
                     metadata[nameof(repoRegistration.NormalizedRepoRoot)] = repoRegistration.NormalizedRepoRoot;
@@ -281,11 +280,38 @@ private void RunMaintenanceTaskForRepos(UserAndSession registeredUser)
                 metadata.Add(nameof(reposSuccessfullyRemoved), reposSuccessfullyRemoved);
                 metadata.Add(nameof(repoRemovalFailures), repoRemovalFailures);
                 metadata.Add(nameof(reposMaintained), reposMaintained);
+                metadata.Add(nameof(maintenancePaused), maintenancePaused);
                 this.tracer.RelatedEvent(
                     EventLevel.Informational,
                     $"{nameof(this.RunMaintenanceTaskForRepos)}_MaintenanceSummary",
                     metadata);
             }
+
+            private bool IsMaintenancePaused(out string traceMessage)
+            {
+                if (this.ignorePause)
+                {
+                    traceMessage = null;
+                    return false;
+                }
+
+                if (this.repoRegistry.TryGetMaintenanceDelayTime(out DateTime time))
+                {
+                    if (time.CompareTo(DateTime.Now) > 0)
+                    {
+                        traceMessage = $"Maintenance is paused until {time}.";
+                        return true;
+                    }
+                    else if (!this.repoRegistry.TryRemovePauseFile(out string innerError))
+                    {
+                        traceMessage = $"Failed to remove pause file: {innerError}";
+                        return false;
+                    }
+                }
+
+                traceMessage = null;
+                return false;
+            }
         }
 
         private class MaintenanceSchedule