Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

pkg: validate volume number for ebs snapshot restore #5292

Merged
merged 5 commits into from
Sep 27, 2023
Merged

pkg: validate volume number for ebs snapshot restore #5292

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b61fbbf
pkg: validate volume number for ebs snapshot restore
BornChanger Sep 18, 2023
0a120fd
pkg: add unit test
BornChanger Sep 25, 2023
0785eda
pkg: address comments
BornChanger Sep 27, 2023
33d85b4
Merge branch 'master' into ebs_validate_volume_number
ti-chi-bot[bot] Sep 27, 2023
010c96b
Merge branch 'master' into ebs_validate_volume_number
ti-chi-bot[bot] Sep 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 44 additions & 43 deletions pkg/backup/restore/restore_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,38 +338,12 @@ func (rm *restoreManager) readRestoreMetaFromExternalStorage(r *v1alpha1.Restore
}
func (rm *restoreManager) validateRestore(r *v1alpha1.Restore, tc *v1alpha1.TidbCluster) error {
// check tiflash and tikv replicas
tiflashReplicas, tikvReplicas, reason, err := rm.readTiFlashAndTiKVReplicasFromBackupMeta(r)
err := rm.checkTiFlashAndTiKVReplicasFromBackupMeta(r, tc)
if err != nil {
klog.Errorf("read tiflash replica failure with reason %s", reason)
klog.Errorf("check tikv/tiflash failure with reason %v", err)
return err
}

if tc.Spec.TiFlash == nil {
if tiflashReplicas != 0 {
klog.Errorf("tiflash is not configured, backupmeta has %d tiflash", tiflashReplicas)
return fmt.Errorf("tiflash replica missmatched")
}

} else {
if tc.Spec.TiFlash.Replicas != tiflashReplicas {
klog.Errorf("cluster has %d tiflash configured, backupmeta has %d tiflash", tc.Spec.TiFlash.Replicas, tiflashReplicas)
return fmt.Errorf("tiflash replica missmatched")
}
}

if tc.Spec.TiKV == nil {
if tikvReplicas != 0 {
klog.Errorf("tikv is not configured, backupmeta has %d tikv", tikvReplicas)
return fmt.Errorf("tikv replica missmatched")
}

} else {
if tc.Spec.TiKV.Replicas != tikvReplicas {
klog.Errorf("cluster has %d tikv configured, backupmeta has %d tikv", tc.Spec.TiKV.Replicas, tikvReplicas)
return fmt.Errorf("tikv replica missmatched")
}
}

// Check recovery mode is on for EBS br across k8s
if r.Spec.Mode == v1alpha1.RestoreModeVolumeSnapshot && r.Spec.FederalVolumeRestorePhase != v1alpha1.FederalVolumeRestoreFinish && !tc.Spec.RecoveryMode {
klog.Errorf("recovery mode is not set for across k8s EBS snapshot restore")
Expand Down Expand Up @@ -437,27 +411,54 @@ func (rm *restoreManager) checkTiKVEncryption(r *v1alpha1.Restore, tc *v1alpha1.
return nil
}

func (rm *restoreManager) readTiFlashAndTiKVReplicasFromBackupMeta(r *v1alpha1.Restore) (int32, int32, string, error) {
func (rm *restoreManager) checkTiFlashAndTiKVReplicasFromBackupMeta(r *v1alpha1.Restore, tc *v1alpha1.TidbCluster) error {
metaInfo, err := backuputil.GetVolSnapBackupMetaData(r, rm.deps.SecretLister)
if err != nil {
return 0, 0, "GetVolSnapBackupMetaData failed", err
}

var tiflashReplicas, tikvReplicas int32

if metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash == nil {
tiflashReplicas = 0
} else {
tiflashReplicas = metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash.Replicas
klog.Errorf("GetVolSnapBackupMetaData failed")
return err
}

if metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV == nil {
tikvReplicas = 0
} else {
tikvReplicas = metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.Replicas
// Check mismatch of tiflash config
if (metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash == nil ||
metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash.Replicas == 0) &&
tc.Spec.TiFlash != nil && tc.Spec.TiFlash.Replicas > 0 {
klog.Errorf("tiflash is enabled in TC but disabled in backup metadata")
return fmt.Errorf("tiflash replica mismatched")
} else if (tc.Spec.TiFlash == nil || tc.Spec.TiFlash.Replicas == 0) &&
metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash != nil &&
metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash.Replicas > 0 {
klog.Errorf("tiflash is disabled in TC enabled in backup metadata")
return fmt.Errorf("tiflash replica mismatched")
} else if tc.Spec.TiFlash != nil && metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash != nil &&
tc.Spec.TiFlash.Replicas != metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash.Replicas {
klog.Errorf("tiflash number in TC is %d but is %d in backup metadata", tc.Spec.TiFlash.Replicas, metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash.Replicas)
return fmt.Errorf("tiflash replica mismatched")
}

// TiKV node must be there
if tc.Spec.TiKV == nil || tc.Spec.TiKV.Replicas == 0 {
klog.Errorf("ebs snapshot restore doesn't support cluster without tikv nodes")
return fmt.Errorf("restore to no tikv cluster")
} else if metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV == nil ||
metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.Replicas == 0 {
klog.Errorf("backup source tc has no tikv nodes")
return fmt.Errorf("backup source tc has no tivk nodes")
} else if tc.Spec.TiKV.Replicas != metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.Replicas {
klog.Errorf("mismatch tikv replicas, tc has %d, while backup has %d", tc.Spec.TiKV.Replicas, metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV)
return fmt.Errorf("tikv replica mismatch")
}

// Check volume number
if (tc.Spec.TiKV.StorageVolumes == nil && metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.StorageVolumes != nil) ||
BornChanger marked this conversation as resolved.
Show resolved Hide resolved
(tc.Spec.TiKV.StorageVolumes != nil && metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.StorageVolumes == nil) {
klog.Errorf("additional volumes # not match. either tc or backup metadata has no additional volumes, while the other has")
return fmt.Errorf("additional volumes mismatched")
} else if len(tc.Spec.TiKV.StorageVolumes) != len(metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.StorageVolumes) {
klog.Errorf("additional volumes # not match. tc has %d, and backup has %d", len(tc.Spec.TiKV.StorageVolumes), len(metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.StorageVolumes))
return fmt.Errorf("additional volumes mismatched")
}

return tiflashReplicas, tikvReplicas, "", nil
return nil
}

func (rm *restoreManager) readTiKVConfigFromBackupMeta(r *v1alpha1.Restore) (*v1alpha1.TiKVConfigWraper, string, error) {
Expand Down
76 changes: 75 additions & 1 deletion pkg/backup/restore/restore_manager_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,7 @@ func TestInvalidReplicasBRRestoreByEBS(t *testing.T) {
helper.CreateRestore(cases[0].restore)
m := NewRestoreManager(deps)
err := m.Sync(cases[0].restore)
g.Expect(err).Should(MatchError("tikv replica missmatched"))
g.Expect(err).Should(MatchError("tikv replica mismatch"))
})
}

Expand Down Expand Up @@ -598,3 +598,77 @@ func TestInvalidModeBRRestoreByEBS(t *testing.T) {
g.Expect(err).Should(MatchError("recovery mode is off"))
})
}

func TestVolumeNumMismatchBRRestoreByEBS(t *testing.T) {
g := NewGomegaWithT(t)
helper := newHelper(t)
defer helper.Close()
deps := helper.Deps

cases := []struct {
name string
restore *v1alpha1.Restore
}{
{
name: "restore-volume",
restore: &v1alpha1.Restore{
ObjectMeta: metav1.ObjectMeta{
Name: "test-1",
Namespace: "ns-1",
},
Spec: v1alpha1.RestoreSpec{
Type: v1alpha1.BackupTypeFull,
Mode: v1alpha1.RestoreModeVolumeSnapshot,
BR: &v1alpha1.BRConfig{
ClusterNamespace: "ns-1",
Cluster: "cluster-1",
},
StorageProvider: v1alpha1.StorageProvider{
Local: &v1alpha1.LocalStorageProvider{
// Prefix: "prefix",
Volume: corev1.Volume{
Name: "nfs",
VolumeSource: corev1.VolumeSource{
NFS: &corev1.NFSVolumeSource{
Server: "fake-server",
Path: "/tmp",
ReadOnly: true,
},
},
},
VolumeMount: corev1.VolumeMount{
Name: "nfs",
MountPath: "/tmp",
},
},
},
},
Status: v1alpha1.RestoreStatus{},
},
},
}

// Verify invalid tc with mismatch tikv replicas
//generate the restore meta in local nfs, with 3 volumes for each tikv
err := os.WriteFile("/tmp/restoremeta", []byte(testutils.ConstructRestoreTiKVVolumesMetaWithStr()), 0644) //nolint:gosec
g.Expect(err).To(Succeed())

//generate the backup meta in local nfs, tiflash check need backupmeta to validation
err = os.WriteFile("/tmp/backupmeta", []byte(testutils.ConstructRestoreTiKVVolumesMetaWithStr()), 0644) //nolint:gosec
g.Expect(err).To(Succeed())
defer func() {
err = os.Remove("/tmp/restoremeta")
g.Expect(err).To(Succeed())

err = os.Remove("/tmp/backupmeta")
g.Expect(err).To(Succeed())
}()

t.Run(cases[0].name, func(t *testing.T) {
helper.CreateTC(cases[0].restore.Spec.BR.ClusterNamespace, cases[0].restore.Spec.BR.Cluster, true, true)
helper.CreateRestore(cases[0].restore)
m := NewRestoreManager(deps)
err := m.Sync(cases[0].restore)
g.Expect(err).Should(MatchError("additional volumes mismatched"))
})
}
Loading
Loading