Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

check tidb clusters which own builtin StatefulSets only in upgrading #1934

Merged
merged 2 commits into from
Mar 17, 2020
Merged

check tidb clusters which own builtin StatefulSets only in upgrading #1934

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8c45779
Revert "current delete slot annotations check in Advanced Statefulset…
cofyc Mar 16, 2020
f86067c
only check relevant tidb clusters
cofyc Mar 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 26 additions & 24 deletions pkg/upgrader/upgrader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"github.com/pingcap/tidb-operator/pkg/label"
utildiscovery "github.com/pingcap/tidb-operator/pkg/util/discovery"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -58,51 +58,53 @@ var _ Interface = &upgrader{}

// isOwnedByTidbCluster checks if the given object is owned by TidbCluster.
// Schema Kind and Group are checked, Version is ignored.
func isOwnedByTidbCluster(obj metav1.Object) bool {
func isOwnedByTidbCluster(obj metav1.Object) (bool, *metav1.OwnerReference) {
ref := metav1.GetControllerOf(obj)
if ref == nil {
return false
return false, nil
}
gv, err := schema.ParseGroupVersion(ref.APIVersion)
if err != nil {
return false
return false, nil
}
return ref.Kind == v1alpha1.TiDBClusterKind && gv.Group == v1alpha1.SchemeGroupVersion.Group
return ref.Kind == v1alpha1.TiDBClusterKind && gv.Group == v1alpha1.SchemeGroupVersion.Group, ref
}

func (u *upgrader) Upgrade() error {
if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) {
klog.Infof("Upgrader: migrating Kubernetes StatefulSets to Advanced StatefulSets")
// We should not check this, otherwise the controller-manager with
// advanced statefulset cannot be restarted when some clusters have
// delete slot annotations set.
// TODO find a better way
// tcList, err := u.cli.PingcapV1alpha1().TidbClusters(u.ns).List(metav1.ListOptions{})
// if err != nil {
// return err
// }
// for _, tc := range tcList.Items {
// // Existing delete slots annotations must be removed first. This is
// // a safety check to ensure no pods are affected in upgrading
// // process.
// if anns := deleteSlotAnns(&tc); len(anns) > 0 {
// return fmt.Errorf("Upgrader: TidbCluster %s/%s has delete slot annotations %v, please remove them before enabling AdvancedStatefulSet feature", tc.Namespace, tc.Name, anns)
// }
// }
stsList, err := u.kubeCli.AppsV1().StatefulSets(u.ns).List(metav1.ListOptions{})
if err != nil {
return err
}
stsToMigrate := make([]appsv1.StatefulSet, 0)
tidbClusters := make([]*v1alpha1.TidbCluster, 0)
for _, sts := range stsList.Items {
if isOwnedByTidbCluster(&sts) {
if ok, tcRef := isOwnedByTidbCluster(&sts); ok {
stsToMigrate = append(stsToMigrate, sts)
tc, err := u.cli.PingcapV1alpha1().TidbClusters(sts.Namespace).Get(tcRef.Name, metav1.GetOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return err
}
if tc != nil {
tidbClusters = append(tidbClusters, tc)
}
}
}
if len(stsToMigrate) <= 0 {
klog.Infof("Upgrader: found 0 Kubernetes StatefulSets owned by TidbCluster, nothing need to do")
return nil
}
klog.Infof("Upgrader: %d Kubernetes Statfulsets owned by TidbCluster should be migrated to Advanced Statefulsets", len(stsToMigrate))
// Check if relavant TidbClusters have delete slots annotations set.
for _, tc := range tidbClusters {
// Existing delete slots annotations must be removed first. This is
// a safety check to ensure no pods are affected in upgrading
// process.
if anns := deleteSlotAnns(tc); len(anns) > 0 {
return fmt.Errorf("Upgrader: TidbCluster %s/%s has delete slot annotations %v, please remove them before enabling AdvancedStatefulSet feature", tc.Namespace, tc.Name, anns)
}
}
klog.Infof("Upgrader: found %d Kubernetes StatefulSets owned by TidbCluster, trying to migrate one by one", len(stsToMigrate))
for _, sts := range stsToMigrate {
_, err := helper.Upgrade(u.kubeCli, u.asCli, &sts)
Expand All @@ -120,15 +122,15 @@ func (u *upgrader) Upgrade() error {
}
stsList, err := u.asCli.AppsV1().StatefulSets(u.ns).List(metav1.ListOptions{})
if err != nil {
if errors.IsNotFound(err) {
if apierrors.IsNotFound(err) {
klog.Infof("Upgrader: Kubernetes server does't have Advanced StatefulSets resources, skip to revert")
return nil
}
return err
}
stsToMigrate := make([]asappsv1.StatefulSet, 0)
for _, sts := range stsList.Items {
if isOwnedByTidbCluster(&sts) {
if ok, _ := isOwnedByTidbCluster(&sts); ok {
stsToMigrate = append(stsToMigrate, sts)
}
}
Expand Down
206 changes: 141 additions & 65 deletions pkg/upgrader/upgrader_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func TestIsOwnedByTidbCluster(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ok := isOwnedByTidbCluster(&tt.sts)
ok, _ := isOwnedByTidbCluster(&tt.sts)
if tt.wantOK != ok {
t.Errorf("got %v, want %v", ok, tt.wantOK)
}
Expand Down Expand Up @@ -160,10 +160,12 @@ func TestDeleteSlotAnns(t *testing.T) {
}

var (
ownerTCName = "foo"
validOwnerRefs = []metav1.OwnerReference{
{
APIVersion: "pingcap.com/v1alpha1",
Kind: "TidbCluster",
Name: ownerTCName,
Controller: pointer.BoolPtr(true),
},
}
Expand Down Expand Up @@ -314,70 +316,144 @@ func TestUpgrade(t *testing.T) {
},
},
},
// {
// name: "should not upgrade if tc has delete slot annotations",
// tidbClusters: []v1alpha1.TidbCluster{
// {
// ObjectMeta: metav1.ObjectMeta{
// Annotations: map[string]string{
// label.AnnTiDBDeleteSlots: "[1,2]",
// },
// },
// },
// },
// statefulsets: []appsv1.StatefulSet{
// {
// TypeMeta: metav1.TypeMeta{
// Kind: "StatefulSet",
// APIVersion: "apps/v1",
// },
// ObjectMeta: metav1.ObjectMeta{
// Name: "sts1",
// Namespace: "sts",
// OwnerReferences: validOwnerRefs,
// },
// },
// {
// TypeMeta: metav1.TypeMeta{
// Kind: "StatefulSet",
// APIVersion: "apps/v1",
// },
// ObjectMeta: metav1.ObjectMeta{
// Name: "sts2",
// Namespace: "sts",
// OwnerReferences: validOwnerRefs,
// },
// },
// },
// feature: "AdvancedStatefulSet=true",
// ns: metav1.NamespaceAll,
// wantErr: true,
// wantAdvancedStatefulsets: nil,
// wantStatefulsets: []appsv1.StatefulSet{
// {
// TypeMeta: metav1.TypeMeta{
// Kind: "StatefulSet",
// APIVersion: "apps/v1",
// },
// ObjectMeta: metav1.ObjectMeta{
// Name: "sts1",
// Namespace: "sts",
// OwnerReferences: validOwnerRefs,
// },
// },
// {
// TypeMeta: metav1.TypeMeta{
// Kind: "StatefulSet",
// APIVersion: "apps/v1",
// },
// ObjectMeta: metav1.ObjectMeta{
// Name: "sts2",
// Namespace: "sts",
// OwnerReferences: validOwnerRefs,
// },
// },
// },
// },
{
name: "should not upgrade if tc has delete slot annotations",
tidbClusters: []v1alpha1.TidbCluster{
{
ObjectMeta: metav1.ObjectMeta{
Name: ownerTCName,
Namespace: "sts",
Annotations: map[string]string{
label.AnnTiDBDeleteSlots: "[1,2]",
},
},
},
},
statefulsets: []appsv1.StatefulSet{
{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "sts1",
Namespace: "sts",
OwnerReferences: validOwnerRefs,
},
},
{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "sts2",
Namespace: "sts",
OwnerReferences: validOwnerRefs,
},
},
},
feature: "AdvancedStatefulSet=true",
ns: metav1.NamespaceAll,
wantErr: true,
wantAdvancedStatefulsets: nil,
wantStatefulsets: []appsv1.StatefulSet{
{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "sts1",
Namespace: "sts",
OwnerReferences: validOwnerRefs,
},
},
{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "sts2",
Namespace: "sts",
OwnerReferences: validOwnerRefs,
},
},
},
},
{
name: "should upgrade if tc has delete slot annotations but does not own Kubernetes StatefulSets",
tidbClusters: []v1alpha1.TidbCluster{
{
ObjectMeta: metav1.ObjectMeta{
Name: ownerTCName,
Namespace: "sts",
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "bar",
Namespace: "sts",
Annotations: map[string]string{
label.AnnTiDBDeleteSlots: "[1,2]",
},
},
},
},
statefulsets: []appsv1.StatefulSet{
{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "sts1",
Namespace: "sts",
OwnerReferences: validOwnerRefs,
},
},
{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "sts2",
Namespace: "sts",
OwnerReferences: validOwnerRefs,
},
},
},
feature: "AdvancedStatefulSet=true",
ns: metav1.NamespaceAll,
wantErr: false,
wantAdvancedStatefulsets: []asappsv1.StatefulSet{
{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps.pingcap.com/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "sts1",
Namespace: "sts",
OwnerReferences: validOwnerRefs,
},
},
{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps.pingcap.com/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "sts2",
Namespace: "sts",
OwnerReferences: validOwnerRefs,
},
},
},
wantStatefulsets: nil,
},
{
name: "should ignore if sts is not owned by TidbCluster",
tidbClusters: nil,
Expand Down