From d75b910ef42dac17ee9288d9d0caa891d0869c94 Mon Sep 17 00:00:00 2001 From: Xieql Date: Mon, 23 Oct 2023 21:13:02 +0800 Subject: [PATCH 1/2] backup: init migrate controller Signed-off-by: Xieql --- cmd/fleet-manager/backup/backup.go | 8 + .../backup_restore_migrate_shared.go | 37 +++ .../backup_restore_migrate_shared_test.go | 145 +++++++++ pkg/fleet-manager/migrate_controller.go | 292 ++++++++++++++++++ 4 files changed, 482 insertions(+) create mode 100644 pkg/fleet-manager/migrate_controller.go diff --git a/cmd/fleet-manager/backup/backup.go b/cmd/fleet-manager/backup/backup.go index 6a45d9e83..14efbe128 100644 --- a/cmd/fleet-manager/backup/backup.go +++ b/cmd/fleet-manager/backup/backup.go @@ -37,5 +37,13 @@ func InitControllers(ctx context.Context, opts *options.Options, mgr ctrl.Manage return err } + if err := (&fleet.MigrateManager{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: opts.Concurrency, RecoverPanic: true}); err != nil { + log.Error(err, "unable to create controller", "controller", "Restore") + return err + } + return nil } diff --git a/pkg/fleet-manager/backup_restore_migrate_shared.go b/pkg/fleet-manager/backup_restore_migrate_shared.go index d70cd4a00..bcaf202c4 100644 --- a/pkg/fleet-manager/backup_restore_migrate_shared.go +++ b/pkg/fleet-manager/backup_restore_migrate_shared.go @@ -457,3 +457,40 @@ func syncVeleroRestoreStatus(ctx context.Context, destinationClusters map[Cluste return clusterDetails, nil } + +// isMigrateSourceReady checks if the 'SourceReadyCondition' of a Migrate object is set to 'True'. +func isMigrateSourceReady(migrate *backupapi.Migrate) bool { + for _, condition := range migrate.Status.Conditions { + if condition.Type == backupapi.SourceReadyCondition && condition.Status == "True" { + return true + } + } + return false +} + +// buildVeleroBackupFromMigrate constructs a Velero Backup instance based on the provided migrate specification. +func buildVeleroBackupFromMigrate(migrateSpec *backupapi.MigrateSpec, labels map[string]string, veleroBackupName string) *velerov1.Backup { + // Only consider migrateSpec.Policy.ResourceFilter and migrateSpec.Policy.OrderedResources when building the Velero backup. + backupParam := &backupapi.BackupSpec{} + if migrateSpec.Policy != nil { + backupParam.Policy = &backupapi.BackupPolicy{ + ResourceFilter: migrateSpec.Policy.ResourceFilter, + OrderedResources: migrateSpec.Policy.OrderedResources, + } + } + return buildVeleroBackupInstance(backupParam, labels, veleroBackupName) +} + +// buildVeleroRestoreFromMigrate constructs a Velero Restore instance based on the provided migrate specification. +func buildVeleroRestoreFromMigrate(migrateSpec *backupapi.MigrateSpec, labels map[string]string, veleroBackupName, veleroRestoreName string) *velerov1.Restore { + // Only consider migrateSpec.Policy.NamespaceMapping, migrateSpec.Policy.PreserveNodePorts, and migrateSpec.Policy.MigrateStatus when building the Velero restore. + restoreParam := &backupapi.RestoreSpec{} + if migrateSpec.Policy != nil { + restoreParam.Policy = &backupapi.RestorePolicy{ + NamespaceMapping: migrateSpec.Policy.NamespaceMapping, + PreserveNodePorts: migrateSpec.Policy.PreserveNodePorts, + PreserveStatus: migrateSpec.Policy.MigrateStatus, + } + } + return buildVeleroRestoreInstance(restoreParam, labels, veleroBackupName, veleroRestoreName) +} diff --git a/pkg/fleet-manager/backup_restore_migrate_shared_test.go b/pkg/fleet-manager/backup_restore_migrate_shared_test.go index 3a34553a6..fc3fd60bb 100644 --- a/pkg/fleet-manager/backup_restore_migrate_shared_test.go +++ b/pkg/fleet-manager/backup_restore_migrate_shared_test.go @@ -22,6 +22,7 @@ import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/yaml" backupapi "kurator.dev/kurator/pkg/apis/backups/v1alpha1" @@ -657,3 +658,147 @@ func TestAllRestoreCompleted(t *testing.T) { }) } } + +func TestIsMigrateSourceReady(t *testing.T) { + tests := []struct { + name string + migrate *backupapi.Migrate + expected bool + }{ + { + name: "SourceReadyCondition is True", + migrate: &backupapi.Migrate{ + Status: backupapi.MigrateStatus{ + Conditions: capiv1.Conditions{ + { + Type: backupapi.SourceReadyCondition, + Status: "True", + }, + }, + }, + }, + expected: true, + }, + { + name: "SourceReadyCondition is not True", + migrate: &backupapi.Migrate{ + Status: backupapi.MigrateStatus{ + Conditions: capiv1.Conditions{ + { + Type: backupapi.SourceReadyCondition, + Status: "False", + }, + }, + }, + }, + expected: false, + }, + { + name: "SourceReadyCondition does not exist", + migrate: &backupapi.Migrate{ + Status: backupapi.MigrateStatus{ + Conditions: capiv1.Conditions{ + { + Type: "AnotherCondition", + Status: "True", + }, + }, + }, + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isMigrateSourceReady(tt.migrate) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestBuildVeleroBackupFromMigrate(t *testing.T) { + tests := []struct { + name string + migrateSpec *backupapi.MigrateSpec + labels map[string]string + veleroBackupName string + expected *velerov1.BackupSpec + }{ + { + name: "all values set", + migrateSpec: &backupapi.MigrateSpec{ + Policy: &backupapi.MigratePolicy{ + ResourceFilter: &backupapi.ResourceFilter{ + IncludedNamespaces: []string{"ns1", "ns2"}, + ExcludedResources: []string{"pods"}, + }, + OrderedResources: map[string]string{ + "pods": "ns1/pod1, ns1/pod2, ns1/pod3", + "persistentvolumes": "pv4, pv8", + }, + }, + }, + labels: map[string]string{ + "test": "value", + }, + veleroBackupName: "test-backup", + expected: &velerov1.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + ExcludedResources: []string{"pods"}, + OrderedResources: map[string]string{ + "pods": "ns1/pod1, ns1/pod2, ns1/pod3", + "persistentvolumes": "pv4, pv8", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := buildVeleroBackupFromMigrate(tt.migrateSpec, tt.labels, tt.veleroBackupName) + assert.Equal(t, tt.expected, &result.Spec) + }) + } +} + +func TestBuildVeleroRestoreFromMigrate(t *testing.T) { + tests := []struct { + name string + migrateSpec *backupapi.MigrateSpec + expected *velerov1.RestoreSpec + }{ + { + name: "all values set", + migrateSpec: &backupapi.MigrateSpec{ + Policy: &backupapi.MigratePolicy{ + NamespaceMapping: map[string]string{"src": "dst"}, + MigrateStatus: &backupapi.PreserveStatus{ + IncludedResources: []string{"deployments", "services"}, + ExcludedResources: []string{"pods"}, + }, + PreserveNodePorts: boolPtr(true), + }, + }, + expected: &velerov1.RestoreSpec{ + NamespaceMapping: map[string]string{"src": "dst"}, + RestoreStatus: &velerov1.RestoreStatusSpec{ + IncludedResources: []string{"deployments", "services"}, + ExcludedResources: []string{"pods"}, + }, + PreserveNodePorts: boolPtr(true), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildVeleroRestoreFromMigrate(tt.migrateSpec, nil, "", "") + assert.Equal(t, tt.expected, &got.Spec) + }) + } +} + +func boolPtr(b bool) *bool { + return &b +} diff --git a/pkg/fleet-manager/migrate_controller.go b/pkg/fleet-manager/migrate_controller.go new file mode 100644 index 000000000..de47c822f --- /dev/null +++ b/pkg/fleet-manager/migrate_controller.go @@ -0,0 +1,292 @@ +/* +Copyright Kurator Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleet + +import ( + "context" + "fmt" + "time" + + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/patch" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + backupapi "kurator.dev/kurator/pkg/apis/backups/v1alpha1" +) + +const ( + // MigrationDelay ensures the restore operation happens after the backup. + // This helps avoid potential issues due to timing synchronization. + MigrationDelay = 1 * time.Second +) + +// MigrateManager reconciles a Migrate object +type MigrateManager struct { + client.Client + Scheme *runtime.Scheme +} + +// SetupWithManager sets up the controller with the Manager. +func (m *MigrateManager) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + return ctrl.NewControllerManagedBy(mgr). + For(&backupapi.Migrate{}). + WithOptions(options). + Complete(m) +} + +func (m *MigrateManager) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { + log := ctrl.LoggerFrom(ctx).WithValues("migrate", req.NamespacedName) + + migrate := &backupapi.Migrate{} + + if err := m.Client.Get(ctx, req.NamespacedName, migrate); err != nil { + if apierrors.IsNotFound(err) { + log.Info("migrate does not exist") + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + patchHelper, err := patch.NewHelper(migrate, m.Client) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to init patch helper for migrate %s", req.NamespacedName) + } + defer func() { + if err := patchHelper.Patch(ctx, migrate); err != nil { + reterr = utilerrors.NewAggregate([]error{reterr, errors.Wrapf(err, "failed to patch %s %s", migrate.Name, req.NamespacedName)}) + } + }() + + if !controllerutil.ContainsFinalizer(migrate, MigrateFinalizer) { + controllerutil.AddFinalizer(migrate, MigrateFinalizer) + } + + // Handle deletion + if migrate.GetDeletionTimestamp() != nil { + return m.reconcileDeleteMigrate(ctx, migrate) + } + + // Handle the main reconcile logic + return m.reconcileMigrate(ctx, migrate) +} + +// reconcileMigrate handles the main reconcile logic for a Migrate object. +func (m *MigrateManager) reconcileMigrate(ctx context.Context, migrate *backupapi.Migrate) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + // Update the migrate phase + phase := migrate.Status.Phase + if len(phase) == 0 || phase == backupapi.MigratePhasePending { + migrate.Status.Phase = backupapi.MigratePhaseWaitingForSource + log.Info("Migrate Phase changes", "phase", backupapi.MigratePhaseWaitingForSource) + } + + // The actual migration operation can be divided into two stages + // 1.the backup stage + result, err := m.reconcileMigrateBackup(ctx, migrate) + if err != nil { + return result, err + } + + // 2.the restore stage. + return m.reconcileMigrateRestore(ctx, migrate) +} + +// reconcileMigrateBackup reconcile the backup process during migration. +func (m *MigrateManager) reconcileMigrateBackup(ctx context.Context, migrate *backupapi.Migrate) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + // Fetch details of the source cluster for migration + fleetClusters, err := fetchDestinationClusters(ctx, m.Client, migrate.Namespace, migrate.Spec.SourceCluster) + if err != nil { + log.Error(err, "Failed to fetch source cluster for migration") + return ctrl.Result{}, fmt.Errorf("fetching source cluster: %w", err) + } + var sourceClusterKey ClusterKey + var sourceClusterAccess *fleetCluster + // "migrate.Spec.SourceCluster" must contain one clusters, it is ensured by admission webhook + for key, value := range fleetClusters { + sourceClusterKey = key + sourceClusterAccess = value + } + + // Construct Velero backup instance + migrateLabel := generateVeleroInstanceLabel(MigrateNameLabel, migrate.Name, migrate.Spec.SourceCluster.Fleet) + sourceBackupName := generateVeleroResourceName(sourceClusterKey.Name, MigrateKind, migrate.Namespace, migrate.Name) + sourceBackup := buildVeleroBackupFromMigrate(&migrate.Spec, migrateLabel, sourceBackupName) + + // Sync the Velero backup resource + if err = syncVeleroObj(ctx, sourceClusterAccess, sourceBackup); err != nil { + log.Error(err, "Failed to create backup resource for migration", "backupName", sourceBackupName) + return ctrl.Result{}, fmt.Errorf("creating backup resource: %w", err) + } + + // Get the status of Velero backup resources + veleroBackup := &velerov1.Backup{} + err = getResourceFromClusterClient(ctx, sourceBackupName, VeleroNamespace, *sourceClusterAccess, veleroBackup) + if err != nil { + log.Error(err, "Failed to retrieve backup resource for migration", "backupName", sourceBackupName) + return ctrl.Result{}, fmt.Errorf("retrieving backup status: %w", err) + } + + // Update migration status based on the Velero backup details + currentBackupDetails := &backupapi.BackupDetails{ + ClusterName: sourceClusterKey.Name, + ClusterKind: sourceClusterKey.Kind, + BackupNameInCluster: veleroBackup.Name, + BackupStatusInCluster: &veleroBackup.Status, + } + migrate.Status.SourceClusterStatus = currentBackupDetails + + if veleroBackup.Status.Phase == velerov1.BackupPhaseCompleted { + conditions.MarkTrue(migrate, backupapi.SourceReadyCondition) + } + + return ctrl.Result{}, nil +} + +// reconcileMigrateRestore handles the restore stage of the migration process. +func (m *MigrateManager) reconcileMigrateRestore(ctx context.Context, migrate *backupapi.Migrate) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + // If source cluster's backup resource is not ready, return directly. + if !isMigrateSourceReady(migrate) { + return ctrl.Result{RequeueAfter: RequeueAfter}, nil + } + + targetClusters, err := fetchDestinationClusters(ctx, m.Client, migrate.Namespace, migrate.Spec.TargetClusters) + if err != nil { + log.Error(err, "Failed to fetch target clusters for migration") + return ctrl.Result{}, fmt.Errorf("fetching target clusters: %w", err) + } + + if migrate.Status.Phase != backupapi.MigratePhaseInProgress { + migrate.Status.Phase = backupapi.MigratePhaseInProgress + log.Info("Migrate Phase changes", "phase", backupapi.MigratePhaseInProgress) + // Ensure the restore point is created after the backup. + time.Sleep(MigrationDelay) + } + + // referredBackupName is same in different target clusters velero restore, because the velero backup will sync to current cluster. + // SourceCluster only has one cluster, so the cluster[0].name is the real name of SourceCluster + referredBackupName := generateVeleroResourceName(migrate.Spec.SourceCluster.Clusters[0].Name, MigrateKind, migrate.Namespace, migrate.Name) + restoreLabel := generateVeleroInstanceLabel(MigrateNameLabel, migrate.Name, migrate.Spec.TargetClusters.Fleet) + // Handle create target clusters' velero restore + var tasks []func() error + for clusterKey, clusterAccess := range targetClusters { + // Ensure the velero backup has been sync to current cluster + referredVeleroBackup := &velerov1.Backup{} + if err = getResourceFromClusterClient(ctx, referredBackupName, VeleroNamespace, *clusterAccess, referredVeleroBackup); err != nil { + if apierrors.IsNotFound(err) { + // if not found, requeue with `RequeueAfter` + return ctrl.Result{RequeueAfter: RequeueAfter}, nil + } else { + return ctrl.Result{}, err + } + } + + veleroRestoreName := generateVeleroResourceName(clusterKey.Name, MigrateKind, migrate.Namespace, migrate.Name) + veleroRestore := buildVeleroRestoreFromMigrate(&migrate.Spec, restoreLabel, referredBackupName, veleroRestoreName) + + task := newSyncVeleroTaskFunc(ctx, clusterAccess, veleroRestore) + tasks = append(tasks, task) + } + + g := &multierror.Group{} + for _, task := range tasks { + g.Go(task) + } + + err = g.Wait().ErrorOrNil() + + if err != nil { + log.Error(err, "Error encountered during sync velero restore when migrating") + return ctrl.Result{}, fmt.Errorf("encountered errors during processing: %v", err) + } + + // Collect target clusters backup resource status to current restore + if migrate.Status.TargetClustersStatus == nil { + migrate.Status.TargetClustersStatus = []*backupapi.RestoreDetails{} + } + migrate.Status.TargetClustersStatus, err = syncVeleroRestoreStatus(ctx, targetClusters, migrate.Status.TargetClustersStatus, MigrateKind, migrate.Namespace, migrate.Name) + if err != nil { + log.Error(err, "failed to sync velero restore status for migrate") + return ctrl.Result{}, err + } + + if allRestoreCompleted(migrate.Status.TargetClustersStatus) { + migrate.Status.Phase = backupapi.MigratePhaseCompleted + log.Info("Migrate Phase changes", "phase", backupapi.MigratePhaseCompleted) + return ctrl.Result{}, nil + } else { + return ctrl.Result{RequeueAfter: StatusSyncInterval}, nil + } +} + +func (m *MigrateManager) reconcileDeleteMigrate(ctx context.Context, migrate *backupapi.Migrate) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + shouldRemoveFinalizer := false + defer func() { + if shouldRemoveFinalizer { + controllerutil.RemoveFinalizer(migrate, MigrateFinalizer) + log.Info("Removed finalizer", "migrateName") + } + }() + + // Fetch source clusters + sourceCluster, err := fetchDestinationClusters(ctx, m.Client, migrate.Namespace, migrate.Spec.SourceCluster) + if err != nil { + log.Error(err, "Failed to fetch source clusters when delete migrate") + shouldRemoveFinalizer = true + return ctrl.Result{}, err + } + + // Delete related velero backup instance + backupList := &velerov1.BackupList{} + err = deleteResourcesInClusters(ctx, VeleroNamespace, MigrateNameLabel, migrate.Name, sourceCluster, backupList) + if err != nil { + log.Error(err, "Failed to delete velero backup instances during migrate deletion") + return ctrl.Result{}, err + } + + // Fetch target clusters + targetClusters, err := fetchDestinationClusters(ctx, m.Client, migrate.Namespace, migrate.Spec.TargetClusters) + if err != nil { + log.Error(err, "Failed to fetch target clusters when delete migrate") + shouldRemoveFinalizer = true + return ctrl.Result{}, err + } + + // Delete all related velero restore instance + restoreList := &velerov1.RestoreList{} + err = deleteResourcesInClusters(ctx, VeleroNamespace, MigrateNameLabel, migrate.Name, targetClusters, restoreList) + if err != nil { + log.Error(err, "Failed to delete related velero restore instances during migrate deletion") + return ctrl.Result{}, err + } + + shouldRemoveFinalizer = true + + return ctrl.Result{}, nil +} From 1ac96a0a145fec97a2cce1e0e6b106ef9d121726 Mon Sep 17 00:00:00 2001 From: Xieql Date: Tue, 24 Oct 2023 11:32:54 +0800 Subject: [PATCH 2/2] fix Signed-off-by: Xieql --- pkg/apis/backups/v1alpha1/migrate_type.go | 8 +++---- pkg/fleet-manager/migrate_controller.go | 27 +++++++---------------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/pkg/apis/backups/v1alpha1/migrate_type.go b/pkg/apis/backups/v1alpha1/migrate_type.go index ebcf580fe..a64dac906 100644 --- a/pkg/apis/backups/v1alpha1/migrate_type.go +++ b/pkg/apis/backups/v1alpha1/migrate_type.go @@ -102,11 +102,11 @@ const ( // the controller's validations and therefore will not run. MigratePhaseFailedValidation MigratePhase = "FailedValidation" - // MigratePhaseWaitingForSource means the migrate is currently fetching source cluster resource. - MigratePhaseWaitingForSource MigratePhase = "WaitingForSource" + // MigratePhaseBackupInProgress indicates that the backup phase of the migrate is currently in progress. + MigratePhaseBackupInProgress MigratePhase = "BackupInProgress" - // MigratePhaseInProgress means the migrate is currently executing migrating. - MigratePhaseInProgress MigratePhase = "InProgress" + // MigratePhaseRestoreInProgress indicates that the restore phase of the migrate is currently in progress. + MigratePhaseRestoreInProgress MigratePhase = "RestoreInProgress" // MigratePhaseCompleted means the migrate has run successfully // without errors. diff --git a/pkg/fleet-manager/migrate_controller.go b/pkg/fleet-manager/migrate_controller.go index de47c822f..3c4e3e01d 100644 --- a/pkg/fleet-manager/migrate_controller.go +++ b/pkg/fleet-manager/migrate_controller.go @@ -16,7 +16,6 @@ package fleet import ( "context" "fmt" - "time" "github.com/hashicorp/go-multierror" "github.com/pkg/errors" @@ -34,12 +33,6 @@ import ( backupapi "kurator.dev/kurator/pkg/apis/backups/v1alpha1" ) -const ( - // MigrationDelay ensures the restore operation happens after the backup. - // This helps avoid potential issues due to timing synchronization. - MigrationDelay = 1 * time.Second -) - // MigrateManager reconciles a Migrate object type MigrateManager struct { client.Client @@ -97,8 +90,8 @@ func (m *MigrateManager) reconcileMigrate(ctx context.Context, migrate *backupap // Update the migrate phase phase := migrate.Status.Phase if len(phase) == 0 || phase == backupapi.MigratePhasePending { - migrate.Status.Phase = backupapi.MigratePhaseWaitingForSource - log.Info("Migrate Phase changes", "phase", backupapi.MigratePhaseWaitingForSource) + migrate.Status.Phase = backupapi.MigratePhaseBackupInProgress + log.Info("Migrate Phase changes", "phase", backupapi.MigratePhaseBackupInProgress) } // The actual migration operation can be divided into two stages @@ -160,6 +153,9 @@ func (m *MigrateManager) reconcileMigrateBackup(ctx context.Context, migrate *ba if veleroBackup.Status.Phase == velerov1.BackupPhaseCompleted { conditions.MarkTrue(migrate, backupapi.SourceReadyCondition) + } else { + log.Info("Waiting for source backup to be ready", "sourceBackupName", sourceBackupName) + return ctrl.Result{RequeueAfter: RequeueAfter}, nil } return ctrl.Result{}, nil @@ -169,22 +165,15 @@ func (m *MigrateManager) reconcileMigrateBackup(ctx context.Context, migrate *ba func (m *MigrateManager) reconcileMigrateRestore(ctx context.Context, migrate *backupapi.Migrate) (ctrl.Result, error) { log := ctrl.LoggerFrom(ctx) - // If source cluster's backup resource is not ready, return directly. - if !isMigrateSourceReady(migrate) { - return ctrl.Result{RequeueAfter: RequeueAfter}, nil - } - targetClusters, err := fetchDestinationClusters(ctx, m.Client, migrate.Namespace, migrate.Spec.TargetClusters) if err != nil { log.Error(err, "Failed to fetch target clusters for migration") return ctrl.Result{}, fmt.Errorf("fetching target clusters: %w", err) } - if migrate.Status.Phase != backupapi.MigratePhaseInProgress { - migrate.Status.Phase = backupapi.MigratePhaseInProgress - log.Info("Migrate Phase changes", "phase", backupapi.MigratePhaseInProgress) - // Ensure the restore point is created after the backup. - time.Sleep(MigrationDelay) + if migrate.Status.Phase != backupapi.MigratePhaseBackupInProgress { + migrate.Status.Phase = backupapi.MigratePhaseRestoreInProgress + log.Info("Migrate Phase changes", "phase", backupapi.MigratePhaseRestoreInProgress) } // referredBackupName is same in different target clusters velero restore, because the velero backup will sync to current cluster.