Skip to content

Commit

Permalink
pkg: validate volume number for ebs snapshot restore (#5292) (#5310)
Browse files Browse the repository at this point in the history 
Signed-off-by: BornChanger <dawn_catcher@126.com>
Co-authored-by: BornChanger <dawn_catcher@126.com>
  • Loading branch information
@ti-chi-bot @BornChanger
ti-chi-bot and BornChanger committed Sep 27, 2023
1 parent b31d3c6 commit 52e1c7c
Show file tree
Hide file tree
Showing 4 changed files with 317 additions and 45 deletions.
83 changes: 40 additions & 43 deletions pkg/backup/restore/restore_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,38 +338,12 @@ func (rm *restoreManager) readRestoreMetaFromExternalStorage(r *v1alpha1.Restore
}
func (rm *restoreManager) validateRestore(r *v1alpha1.Restore, tc *v1alpha1.TidbCluster) error {
// check tiflash and tikv replicas
tiflashReplicas, tikvReplicas, reason, err := rm.readTiFlashAndTiKVReplicasFromBackupMeta(r)
err := rm.checkTiFlashAndTiKVReplicasFromBackupMeta(r, tc)
if err != nil {
klog.Errorf("read tiflash replica failure with reason %s", reason)
klog.Errorf("check tikv/tiflash failure with reason %v", err)
return err
}

if tc.Spec.TiFlash == nil {
if tiflashReplicas != 0 {
klog.Errorf("tiflash is not configured, backupmeta has %d tiflash", tiflashReplicas)
return fmt.Errorf("tiflash replica missmatched")
}

} else {
if tc.Spec.TiFlash.Replicas != tiflashReplicas {
klog.Errorf("cluster has %d tiflash configured, backupmeta has %d tiflash", tc.Spec.TiFlash.Replicas, tiflashReplicas)
return fmt.Errorf("tiflash replica missmatched")
}
}

if tc.Spec.TiKV == nil {
if tikvReplicas != 0 {
klog.Errorf("tikv is not configured, backupmeta has %d tikv", tikvReplicas)
return fmt.Errorf("tikv replica missmatched")
}

} else {
if tc.Spec.TiKV.Replicas != tikvReplicas {
klog.Errorf("cluster has %d tikv configured, backupmeta has %d tikv", tc.Spec.TiKV.Replicas, tikvReplicas)
return fmt.Errorf("tikv replica missmatched")
}
}

// Check recovery mode is on for EBS br across k8s
if r.Spec.Mode == v1alpha1.RestoreModeVolumeSnapshot && r.Spec.FederalVolumeRestorePhase != v1alpha1.FederalVolumeRestoreFinish && !tc.Spec.RecoveryMode {
klog.Errorf("recovery mode is not set for across k8s EBS snapshot restore")
Expand Down Expand Up @@ -437,27 +411,50 @@ func (rm *restoreManager) checkTiKVEncryption(r *v1alpha1.Restore, tc *v1alpha1.
return nil
}

func (rm *restoreManager) readTiFlashAndTiKVReplicasFromBackupMeta(r *v1alpha1.Restore) (int32, int32, string, error) {
func (rm *restoreManager) checkTiFlashAndTiKVReplicasFromBackupMeta(r *v1alpha1.Restore, tc *v1alpha1.TidbCluster) error {
metaInfo, err := backuputil.GetVolSnapBackupMetaData(r, rm.deps.SecretLister)
if err != nil {
return 0, 0, "GetVolSnapBackupMetaData failed", err
}

var tiflashReplicas, tikvReplicas int32

if metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash == nil {
tiflashReplicas = 0
} else {
tiflashReplicas = metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash.Replicas
klog.Errorf("GetVolSnapBackupMetaData failed")
return err
}

if metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV == nil {
tikvReplicas = 0
} else {
tikvReplicas = metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.Replicas
// Check mismatch of tiflash config
if (metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash == nil ||
metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash.Replicas == 0) &&
tc.Spec.TiFlash != nil && tc.Spec.TiFlash.Replicas > 0 {
klog.Errorf("tiflash is enabled in TC but disabled in backup metadata")
return fmt.Errorf("tiflash replica mismatched")
} else if (tc.Spec.TiFlash == nil || tc.Spec.TiFlash.Replicas == 0) &&
metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash != nil &&
metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash.Replicas > 0 {
klog.Errorf("tiflash is disabled in TC enabled in backup metadata")
return fmt.Errorf("tiflash replica mismatched")
} else if tc.Spec.TiFlash != nil && metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash != nil &&
tc.Spec.TiFlash.Replicas != metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash.Replicas {
klog.Errorf("tiflash number in TC is %d but is %d in backup metadata", tc.Spec.TiFlash.Replicas, metaInfo.KubernetesMeta.TiDBCluster.Spec.TiFlash.Replicas)
return fmt.Errorf("tiflash replica mismatched")
}

// TiKV node must be there
if tc.Spec.TiKV == nil || tc.Spec.TiKV.Replicas == 0 {
klog.Errorf("ebs snapshot restore doesn't support cluster without tikv nodes")
return fmt.Errorf("restore to no tikv cluster")
} else if metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV == nil ||
metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.Replicas == 0 {
klog.Errorf("backup source tc has no tikv nodes")
return fmt.Errorf("backup source tc has no tivk nodes")
} else if tc.Spec.TiKV.Replicas != metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.Replicas {
klog.Errorf("mismatch tikv replicas, tc has %d, while backup has %d", tc.Spec.TiKV.Replicas, metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV)
return fmt.Errorf("tikv replica mismatch")
}

// Check volume number
if len(tc.Spec.TiKV.StorageVolumes) != len(metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.StorageVolumes) {
klog.Errorf("additional volumes # not match. tc has %d, and backup has %d", len(tc.Spec.TiKV.StorageVolumes), len(metaInfo.KubernetesMeta.TiDBCluster.Spec.TiKV.StorageVolumes))
return fmt.Errorf("additional volumes mismatched")
}

return tiflashReplicas, tikvReplicas, "", nil
return nil
}

func (rm *restoreManager) readTiKVConfigFromBackupMeta(r *v1alpha1.Restore) (*v1alpha1.TiKVConfigWraper, string, error) {
Expand Down
76 changes: 75 additions & 1 deletion pkg/backup/restore/restore_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,7 @@ func TestInvalidReplicasBRRestoreByEBS(t *testing.T) {
helper.CreateRestore(cases[0].restore)
m := NewRestoreManager(deps)
err := m.Sync(cases[0].restore)
g.Expect(err).Should(MatchError("tikv replica missmatched"))
g.Expect(err).Should(MatchError("tikv replica mismatch"))
})
}

Expand Down Expand Up @@ -598,3 +598,77 @@ func TestInvalidModeBRRestoreByEBS(t *testing.T) {
g.Expect(err).Should(MatchError("recovery mode is off"))
})
}

func TestVolumeNumMismatchBRRestoreByEBS(t *testing.T) {
g := NewGomegaWithT(t)
helper := newHelper(t)
defer helper.Close()
deps := helper.Deps

cases := []struct {
name string
restore *v1alpha1.Restore
}{
{
name: "restore-volume",
restore: &v1alpha1.Restore{
ObjectMeta: metav1.ObjectMeta{
Name: "test-1",
Namespace: "ns-1",
},
Spec: v1alpha1.RestoreSpec{
Type: v1alpha1.BackupTypeFull,
Mode: v1alpha1.RestoreModeVolumeSnapshot,
BR: &v1alpha1.BRConfig{
ClusterNamespace: "ns-1",
Cluster: "cluster-1",
},
StorageProvider: v1alpha1.StorageProvider{
Local: &v1alpha1.LocalStorageProvider{
// Prefix: "prefix",
Volume: corev1.Volume{
Name: "nfs",
VolumeSource: corev1.VolumeSource{
NFS: &corev1.NFSVolumeSource{
Server: "fake-server",
Path: "/tmp",
ReadOnly: true,
},
},
},
VolumeMount: corev1.VolumeMount{
Name: "nfs",
MountPath: "/tmp",
},
},
},
},
Status: v1alpha1.RestoreStatus{},
},
},
}

// Verify invalid tc with mismatch tikv replicas
//generate the restore meta in local nfs, with 3 volumes for each tikv
err := os.WriteFile("/tmp/restoremeta", []byte(testutils.ConstructRestoreTiKVVolumesMetaWithStr()), 0644) //nolint:gosec
g.Expect(err).To(Succeed())

//generate the backup meta in local nfs, tiflash check need backupmeta to validation
err = os.WriteFile("/tmp/backupmeta", []byte(testutils.ConstructRestoreTiKVVolumesMetaWithStr()), 0644) //nolint:gosec
g.Expect(err).To(Succeed())
defer func() {
err = os.Remove("/tmp/restoremeta")
g.Expect(err).To(Succeed())

err = os.Remove("/tmp/backupmeta")
g.Expect(err).To(Succeed())
}()

t.Run(cases[0].name, func(t *testing.T) {
helper.CreateTC(cases[0].restore.Spec.BR.ClusterNamespace, cases[0].restore.Spec.BR.Cluster, true, true)
helper.CreateRestore(cases[0].restore)
m := NewRestoreManager(deps)
err := m.Sync(cases[0].restore)
g.Expect(err).Should(MatchError("additional volumes mismatched"))
})
}
Loading

0 comments on commit 52e1c7c

Please sign in to comment.