Skip to content

Commit

Permalink
Orphan pods cleaner (#201)
Browse files Browse the repository at this point in the history
* add ophan pods cleaner

* upgrade duration to 10 minutes
  • Loading branch information
weekface authored and tennix committed Dec 1, 2018
1 parent 1bc6486 commit 94181ea
Show file tree
Hide file tree
Showing 13 changed files with 570 additions and 296 deletions.
9 changes: 9 additions & 0 deletions pkg/controller/tidbcluster/tidb_cluster_control.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/manager"
"github.com/pingcap/tidb-operator/pkg/manager/member"
apiequality "k8s.io/apimachinery/pkg/api/equality"
errorutils "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/tools/record"
Expand All @@ -39,6 +40,7 @@ func NewDefaultTidbClusterControl(
tidbMemberManager manager.Manager,
reclaimPolicyManager manager.Manager,
metaManager manager.Manager,
orphanPodsCleaner member.OrphanPodsCleaner,
recorder record.EventRecorder) ControlInterface {
return &defaultTidbClusterControl{
tcControl,
Expand All @@ -47,6 +49,7 @@ func NewDefaultTidbClusterControl(
tidbMemberManager,
reclaimPolicyManager,
metaManager,
orphanPodsCleaner,
recorder,
}
}
Expand All @@ -58,6 +61,7 @@ type defaultTidbClusterControl struct {
tidbMemberManager manager.Manager
reclaimPolicyManager manager.Manager
metaManager manager.Manager
orphanPodsCleaner member.OrphanPodsCleaner
recorder record.EventRecorder
}

Expand Down Expand Up @@ -92,6 +96,11 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster
return err
}

_, err = tcc.orphanPodsCleaner.Clean(tc)
if err != nil {
return err
}

// PD
err = tcc.pdMemberManager.Sync(tc)
if err != nil {
Expand Down
3 changes: 2 additions & 1 deletion pkg/controller/tidbcluster/tidb_cluster_control_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,8 @@ func newFakeTidbClusterControl() (ControlInterface, *meta.FakeReclaimPolicyManag
tidbMemberManager := mm.NewFakeTiDBMemberManager()
reclaimPolicyManager := meta.NewFakeReclaimPolicyManager()
metaManager := meta.NewFakeMetaManager()
control := NewDefaultTidbClusterControl(tcControl, pdMemberManager, tikvMemberManager, tidbMemberManager, reclaimPolicyManager, metaManager, recorder)
opc := mm.NewFakeOrphanPodsCleaner()
control := NewDefaultTidbClusterControl(tcControl, pdMemberManager, tikvMemberManager, tidbMemberManager, reclaimPolicyManager, metaManager, opc, recorder)

return control, reclaimPolicyManager, pdMemberManager, tikvMemberManager, tidbMemberManager, metaManager
}
Expand Down
5 changes: 5 additions & 0 deletions pkg/controller/tidbcluster/tidb_cluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,11 @@ func NewController(
podInformer.Lister(),
podControl,
),
mm.NewOrphanPodsCleaner(
podInformer.Lister(),
podControl,
pvcInformer.Lister(),
),
recorder,
),
queue: workqueue.NewNamedRateLimitingQueue(
Expand Down
1 change: 1 addition & 0 deletions pkg/controller/tidbcluster/tidb_cluster_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,7 @@ func newFakeTidbClusterController() (*Controller, cache.Indexer, cache.Indexer)
podInformer.Lister(),
podControl,
),
mm.NewFakeOrphanPodsCleaner(),
recorder,
)

Expand Down
3 changes: 1 addition & 2 deletions pkg/label/label_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@ package label
import (
"testing"

"k8s.io/apimachinery/pkg/labels"

. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/labels"
)

func TestLabelNew(t *testing.T) {
Expand Down
109 changes: 109 additions & 0 deletions pkg/manager/member/orphan_pods_cleaner.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package member

import (
"github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/label"
"k8s.io/apimachinery/pkg/api/errors"
corelisters "k8s.io/client-go/listers/core/v1"
)

const (
skipReasonOrphanPodsCleanerIsNotPDOrTiKV = "orphan pods cleaner: member type is not pd or tikv"
skipReasonOrphanPodsCleanerPVCNameIsEmpty = "orphan pods cleaner: pvcName is empty"
skipReasonOrphanPodsCleanerPVCIsFound = "orphan pods cleaner: pvc is found"
)

// OrphanPodsCleaner implements the logic for cleaning the orphan pods(has no pvc)
type OrphanPodsCleaner interface {
Clean(*v1alpha1.TidbCluster) (map[string]string, error)
}

type orphanPodsCleaner struct {
podLister corelisters.PodLister
podControl controller.PodControlInterface
pvcLister corelisters.PersistentVolumeClaimLister
}

// NewOrphanPodsCleaner returns a OrphanPodsCleaner
func NewOrphanPodsCleaner(podLister corelisters.PodLister,
podControl controller.PodControlInterface,
pvcLister corelisters.PersistentVolumeClaimLister) OrphanPodsCleaner {
return &orphanPodsCleaner{podLister, podControl, pvcLister}
}

func (opc *orphanPodsCleaner) Clean(tc *v1alpha1.TidbCluster) (map[string]string, error) {
ns := tc.GetNamespace()
// for unit test
skipReason := map[string]string{}

selector, err := label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).Selector()
if err != nil {
return skipReason, err
}
pods, err := opc.podLister.Pods(ns).List(selector)
if err != nil {
return skipReason, err
}

for _, pod := range pods {
podName := pod.GetName()
l := label.Label(pod.Labels)
if !(l.IsPD() || l.IsTiKV()) {
skipReason[podName] = skipReasonOrphanPodsCleanerIsNotPDOrTiKV
continue
}

var pvcName string
for _, vol := range pod.Spec.Volumes {
if vol.PersistentVolumeClaim != nil {
pvcName = vol.PersistentVolumeClaim.ClaimName
break
}
}
if pvcName == "" {
skipReason[podName] = skipReasonOrphanPodsCleanerPVCNameIsEmpty
continue
}

_, err := opc.pvcLister.PersistentVolumeClaims(ns).Get(pvcName)
if err == nil {
skipReason[podName] = skipReasonOrphanPodsCleanerPVCIsFound
continue
}
if !errors.IsNotFound(err) {
return skipReason, err
}

err = opc.podControl.DeletePod(tc, pod)
if err != nil {
return skipReason, err
}
}

return skipReason, nil
}

type fakeOrphanPodsCleaner struct{}

// NewFakeOrphanPodsCleaner returns a fake orphan pods cleaner
func NewFakeOrphanPodsCleaner() OrphanPodsCleaner {
return &fakeOrphanPodsCleaner{}
}

func (fopc *fakeOrphanPodsCleaner) Clean(_ *v1alpha1.TidbCluster) (map[string]string, error) {
return nil, nil
}
Loading

0 comments on commit 94181ea

Please sign in to comment.