fix(restore): multiple restore requests should be rejected and propos…

…als should not be submitted (#7118) (#7276)

* fix(restore): multiple restore requests should be rejected and proposals should not be submitted

* adding doc defining restore behaviour

* changing message to be more concrete

* making restore return error if operation already running

* making use of GetOngoingTasks

worker/online_restore_ee.go

@@ -62,10 +62,38 @@ func ProcessRestoreRequest(ctx context.Context, req *pb.RestoreRequest) (int, er
 	if err := VerifyBackup(req, &creds, currentGroups); err != nil {
 		return 0, errors.Wrapf(err, "failed to verify backup")
 	}
-
 	if err := FillRestoreCredentials(req.Location, req); err != nil {
 		return 0, errors.Wrapf(err, "cannot fill restore proposal with the right credentials")
 	}
+	// Restore Tracker keeps tracks of ongoing restore operation initiated on the node.
+	// Restore Tracker are node specific. They manage restore operation initiated on the node.
+	// If already initiated on this node, rt.Add() will return already running operation error.
+	restoreId, err := rt.Add()
+	if err != nil {
+		return 0, errors.Wrapf(err, "cannot assign ID to restore operation")
+	}
+
+	// This check if any restore operation running on the node.
+	// Operation initiated on other nodes doesn't have record in the record tracker.
+	// This keeps track if there is an already running restore operation return the error.
+	// IMP: This introduces few corner cases.
+	// Like two concurrent restore operation on different nodes.
+	// Considering Restore as admin operation, solving all those complexities has low gains
+	// than to sacrifice the simplicity.
+	isRestoreRunning := func() bool {
+		tasks := GetOngoingTasks()
+		for _, t := range tasks {
+			if t == opRestore.String() {
+				return true
+			}
+		}
+		return false
+	}
+	if isRestoreRunning() {
+		return 0, errors.Errorf("another restore operation is already running. " +
+			"Please retry later.")
+	}
+
 	req.RestoreTs = State.GetTimestamp(false)
 
 	// TODO: prevent partial restores when proposeRestoreOrSend only sends the restore
@@ -80,10 +108,6 @@ func ProcessRestoreRequest(ctx context.Context, req *pb.RestoreRequest) (int, er
 		}()
 	}
 
-	restoreId, err := rt.Add()
-	if err != nil {
-		return 0, errors.Wrapf(err, "cannot assign ID to restore operation")
-	}
 	go func(restoreId int) {
 		errs := make([]error, 0)
 		for range currentGroups {