br: backup manifests related to tidb cluster in EBS backup (#5207) (#…

…5215)

pkg/backup/backup/backup_manager.go

@@ -14,9 +14,11 @@
 package backup
 
 import (
+ "bytes"
  "context"
  "encoding/json"
  "fmt"
+ "path"
  "strings"
  "time"
 
@@ -31,27 +33,39 @@ import (
  batchv1 "k8s.io/api/batch/v1"
  corev1 "k8s.io/api/core/v1"
  "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/api/meta"
  "k8s.io/apimachinery/pkg/api/resource"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/cli-runtime/pkg/printers"
  "k8s.io/klog/v2"
  "k8s.io/utils/pointer"
 )
 
 type backupManager struct {
- deps *controller.Dependencies
- backupCleaner BackupCleaner
- backupTracker BackupTracker
- statusUpdater controller.BackupConditionUpdaterInterface
+ deps *controller.Dependencies
+ backupCleaner BackupCleaner
+ backupTracker BackupTracker
+ statusUpdater controller.BackupConditionUpdaterInterface
+ manifestFetchers []ManifestFetcher
 }
 
 // NewBackupManager return backupManager
 func NewBackupManager(deps *controller.Dependencies) backup.BackupManager {
  statusUpdater := controller.NewRealBackupConditionUpdater(deps.Clientset, deps.BackupLister, deps.Recorder)
+ manifestFetchers := []ManifestFetcher{
+ NewTiDBClusterAutoScalerFetcher(deps.TiDBClusterAutoScalerLister),
+ NewTiDBDashboardFetcher(deps.TiDBDashboardLister),
+ NewTiDBInitializerFetcher(deps.TiDBInitializerLister),
+ NewTiDBMonitorFetcher(deps.TiDBMonitorLister),
+ NewTiDBNgMonitoringFetcher(deps.TiDBNGMonitoringLister),
+ }
  return &backupManager{
- deps: deps,
- backupCleaner: NewBackupCleaner(deps, statusUpdater),
- backupTracker: NewBackupTracker(deps, statusUpdater),
- statusUpdater: statusUpdater,
+ deps: deps,
+ backupCleaner: NewBackupCleaner(deps, statusUpdater),
+ backupTracker: NewBackupTracker(deps, statusUpdater),
+ statusUpdater: statusUpdater,
+ manifestFetchers: manifestFetchers,
  }
 }
 
@@ -785,17 +799,92 @@ func (bm *backupManager) saveClusterMetaToExternalStorage(b *v1alpha1.Backup, cs
 }
 
 func (bm *backupManager) volumeSnapshotBackup(b *v1alpha1.Backup, tc *v1alpha1.TidbCluster) (string, error) {
- if s, reason, err := snapshotter.NewSnapshotterForBackup(b.Spec.Mode, bm.deps); err != nil {
+ if err := bm.backupManifests(b, tc); err != nil {
+ return "BackupManifestsFailed", err
+ }
+
+ s, reason, err := snapshotter.NewSnapshotterForBackup(b.Spec.Mode, bm.deps)
+ if err != nil {
  return reason, err
- } else if s != nil {
- csb, reason, err := s.GenerateBackupMetadata(b, tc)
+ }
+
+ csb, reason, err := s.GenerateBackupMetadata(b, tc)
+ if err != nil {
+ return reason, err
+ }
+
+ if reason, err = bm.saveClusterMetaToExternalStorage(b, csb); err != nil {
+ return reason, err
+ }
+ return "", nil
+}
+
+func (bm *backupManager) backupManifests(b *v1alpha1.Backup, tc *v1alpha1.TidbCluster) error {
+ cred := backuputil.GetStorageCredential(b.Namespace, b.Spec.StorageProvider, bm.deps.SecretLister)
+ externalStorage, err := backuputil.NewStorageBackend(b.Spec.StorageProvider, cred)
+ if err != nil {
+ return err
+ }
+
+ manifests := make([]runtime.Object, 0, 4)
+ manifests = append(manifests, tc)
+ for _, fetcher := range bm.manifestFetchers {
+ objects, err := fetcher.ListByTC(tc)
+ if err != nil {
+ return err
+ }
+ manifests = append(manifests, objects...)
+ }
+
+ for _, manifest := range manifests {
+ if err := bm.saveManifest(b, manifest.DeepCopyObject(), externalStorage); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (bm *backupManager) saveManifest(b *v1alpha1.Backup, manifest runtime.Object, externalStorage *backuputil.StorageBackend) error {
+ if manifest.GetObjectKind().GroupVersionKind().Empty() {
+ gvk, err := controller.InferObjectKind(manifest)
  if err != nil {
- return reason, err
+ return err
  }
+ manifest.GetObjectKind().SetGroupVersionKind(gvk)
+ }
+ kind := manifest.GetObjectKind().GroupVersionKind().Kind
+ metadataAccessor, err := meta.Accessor(manifest)
+ if err != nil {
+ return err
+ }
+ namespace := metadataAccessor.GetNamespace()
+ name := metadataAccessor.GetName()
+ // remove managedFields because it is used for k8s internal housekeeping, and we don't need them in backup
+ metadataAccessor.SetManagedFields(nil)
+ klog.Infof("%s/%s get manifest meta, kind: %s, namespace: %s, name: %s", b.Namespace, b.Name, kind, namespace, name)
 
- return bm.saveClusterMetaToExternalStorage(b, csb)
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+ defer cancel()
+
+ filePath := path.Join(constants.ClusterManifests, namespace, kind, fmt.Sprintf("%s.yaml", name))
+ existed, err := externalStorage.Exists(ctx, filePath)
+ if err != nil {
+ return err
  }
- return "", nil
+ if existed {
+ return nil
+ }
+
+ buf := bytes.NewBuffer(make([]byte, 0, 1024))
+ printer := printers.YAMLPrinter{}
+ if err := printer.PrintObj(manifest, buf); err != nil {
+ return err
+ }
+ if err := externalStorage.WriteAll(ctx, filePath, buf.Bytes(), nil); err != nil {
+ return err
+ }
+ klog.Infof("%s/%s upload manifest %s successfully", b.Namespace, b.Name, filePath)
+ return nil
 }
 
 func (bm *backupManager) ensureBackupPVCExist(backup *v1alpha1.Backup) (string, error) {

pkg/backup/backup/backup_manifests.go

@@ -0,0 +1,195 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package backup
+
+import (
+ "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
+ listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/klog/v2"
+)
+
+type ManifestFetcher interface {
+ ListByTC(tc *v1alpha1.TidbCluster) (objects []runtime.Object, err error)
+}
+
+type TiDBDashboardFetcher struct {
+ lister listers.TidbDashboardLister
+}
+
+func NewTiDBDashboardFetcher(lister listers.TidbDashboardLister) ManifestFetcher {
+ return &TiDBDashboardFetcher{
+ lister: lister,
+ }
+}
+
+func (f *TiDBDashboardFetcher) ListByTC(tc *v1alpha1.TidbCluster) (objects []runtime.Object, err error) {
+ emptySelector := labels.NewSelector()
+ dashboards, err := f.lister.List(emptySelector)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, dashboard := range dashboards {
+ for _, cluster := range dashboard.Spec.Clusters {
+ name := cluster.Name
+ namespace := cluster.Namespace
+ if namespace == "" {
+ namespace = dashboard.Namespace
+ }
+ if name == tc.Name && namespace == tc.Namespace {
+ klog.Infof("TiDBDashboard %s/%s matches tc %s/%s",
+ dashboard.Namespace, dashboard.Name, tc.Namespace, tc.Name)
+ objects = append(objects, dashboard)
+ break
+ }
+ }
+ }
+ return
+}
+
+type TiDBMonitorFetcher struct {
+ lister listers.TidbMonitorLister
+}
+
+func NewTiDBMonitorFetcher(lister listers.TidbMonitorLister) ManifestFetcher {
+ return &TiDBMonitorFetcher{
+ lister: lister,
+ }
+}
+
+func (f *TiDBMonitorFetcher) ListByTC(tc *v1alpha1.TidbCluster) (objects []runtime.Object, err error) {
+ emptySelector := labels.NewSelector()
+ monitors, err := f.lister.List(emptySelector)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, monitor := range monitors {
+ for _, cluster := range monitor.Spec.Clusters {
+ name := cluster.Name
+ namespace := cluster.Namespace
+ if namespace == "" {
+ namespace = monitor.Namespace
+ }
+ if name == tc.Name && namespace == tc.Namespace {
+ klog.Infof("TidbMonitor %s/%s matches tc %s/%s",
+ monitor.Namespace, monitor.Name, tc.Namespace, tc.Name)
+ objects = append(objects, monitor)
+ break
+ }
+ }
+ }
+ return
+}
+
+type TiDBClusterAutoScalerFetcher struct {
+ lister listers.TidbClusterAutoScalerLister
+}
+
+func NewTiDBClusterAutoScalerFetcher(lister listers.TidbClusterAutoScalerLister) ManifestFetcher {
+ return &TiDBClusterAutoScalerFetcher{
+ lister: lister,
+ }
+}
+
+func (f *TiDBClusterAutoScalerFetcher) ListByTC(tc *v1alpha1.TidbCluster) (objects []runtime.Object, err error) {
+ emptySelector := labels.NewSelector()
+ autoScalers, err := f.lister.List(emptySelector)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, autoScaler := range autoScalers {
+ name := autoScaler.Spec.Cluster.Name
+ namespace := autoScaler.Spec.Cluster.Namespace
+ if namespace == "" {
+ namespace = autoScaler.Namespace
+ }
+ if name == tc.Name && namespace == tc.Namespace {
+ klog.Infof("TiDBClusterAutoScaler %s/%s matches tc %s/%s",
+ autoScaler.Namespace, autoScaler.Name, tc.Namespace, tc.Name)
+ objects = append(objects, autoScaler)
+ }
+ }
+ return
+}
+
+type TiDBInitializerFetcher struct {
+ lister listers.TidbInitializerLister
+}
+
+func NewTiDBInitializerFetcher(lister listers.TidbInitializerLister) ManifestFetcher {
+ return &TiDBInitializerFetcher{
+ lister: lister,
+ }
+}
+
+func (f *TiDBInitializerFetcher) ListByTC(tc *v1alpha1.TidbCluster) (objects []runtime.Object, err error) {
+ emptySelector := labels.NewSelector()
+ initializers, err := f.lister.List(emptySelector)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, initializer := range initializers {
+ name := initializer.Spec.Clusters.Name
+ namespace := initializer.Spec.Clusters.Namespace
+ if namespace == "" {
+ namespace = initializer.Namespace
+ }
+ if name == tc.Name && namespace == tc.Namespace {
+ klog.Infof("TiDBInitializer %s/%s matches tc %s/%s",
+ initializer.Namespace, initializer.Name, tc.Namespace, tc.Name)
+ objects = append(objects, initializer)
+ }
+ }
+ return
+}
+
+type TiDBNgMonitoringFetcher struct {
+ lister listers.TidbNGMonitoringLister
+}
+
+func NewTiDBNgMonitoringFetcher(lister listers.TidbNGMonitoringLister) ManifestFetcher {
+ return &TiDBNgMonitoringFetcher{
+ lister: lister,
+ }
+}
+
+func (f *TiDBNgMonitoringFetcher) ListByTC(tc *v1alpha1.TidbCluster) (objects []runtime.Object, err error) {
+ emptySelector := labels.NewSelector()
+ monitorings, err := f.lister.List(emptySelector)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, monitoring := range monitorings {
+ for _, cluster := range monitoring.Spec.Clusters {
+ name := cluster.Name
+ namespace := cluster.Namespace
+ if namespace == "" {
+ namespace = monitoring.Namespace
+ }
+ if name == tc.Name && namespace == tc.Namespace {
+ klog.Infof("TidbNGMonitoring %s/%s matches tc %s/%s",
+ monitoring.Namespace, monitoring.Name, tc.Namespace, tc.Name)
+ objects = append(objects, monitoring)
+ break
+ }
+ }
+ }
+ return
+}