Add more events for tidbcluster and autoscaler

pkg/autoscaler/autoscaler/autoscaler_manager.go

@@ -15,12 +15,14 @@ package autoscaler
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
 	informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions"
 	v1alpha1listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/controller"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -80,7 +82,7 @@ func (am *autoScalerManager) Sync(tac *v1alpha1.TidbClusterAutoScaler) error {
 	if err := am.syncAutoScaling(tc, tac); err != nil {
 		return err
 	}
-	if err := am.syncTidbClusterReplicas(tc, oldTc); err != nil {
+	if err := am.syncTidbClusterReplicas(tac, tc, oldTc); err != nil {
 		return err
 	}
 	return am.syncAutoScalingStatus(tc, oldTc, tac)
@@ -102,7 +104,7 @@ func (am *autoScalerManager) syncAutoScaling(tc *v1alpha1.TidbCluster, tac *v1al
 	return nil
 }
 
-func (am *autoScalerManager) syncTidbClusterReplicas(tc *v1alpha1.TidbCluster, oldTc *v1alpha1.TidbCluster) error {
+func (am *autoScalerManager) syncTidbClusterReplicas(tac *v1alpha1.TidbClusterAutoScaler, tc *v1alpha1.TidbCluster, oldTc *v1alpha1.TidbCluster) error {
 	if tc.Spec.TiDB.Replicas == oldTc.Spec.TiDB.Replicas && tc.Spec.TiKV.Replicas == oldTc.Spec.TiKV.Replicas {
 		return nil
 	}
@@ -111,6 +113,15 @@ func (am *autoScalerManager) syncTidbClusterReplicas(tc *v1alpha1.TidbCluster, o
 	if err != nil {
 		return err
 	}
+	reason := fmt.Sprintf("Successful %s", strings.Title("auto-scaling"))
+	msg := ""
+	if tc.Spec.TiDB.Replicas != oldTc.Spec.TiDB.Replicas {
+		msg = fmt.Sprintf("%s auto-scaling tidb from %d to %d", msg, oldTc.Spec.TiDB.Replicas, tc.Spec.TiDB.Replicas)
+	}
+	if tc.Spec.TiKV.Replicas != oldTc.Spec.TiKV.Replicas {
+		msg = fmt.Sprintf("%s auto-scaling tikv from %d to %d", msg, oldTc.Spec.TiKV.Replicas, tc.Spec.TiKV.Replicas)
+	}
+	am.recorder.Event(tac, corev1.EventTypeNormal, reason, msg)
 	return nil
 }
 

pkg/controller/tidbcluster/tidb_cluster_controller.go

@@ -107,8 +107,8 @@ func NewController(
 	pdScaler := mm.NewPDScaler(pdControl, pvcInformer.Lister(), pvcControl)
 	tikvScaler := mm.NewTiKVScaler(pdControl, pvcInformer.Lister(), pvcControl, podInformer.Lister())
 	pdFailover := mm.NewPDFailover(cli, pdControl, pdFailoverPeriod, podInformer.Lister(), podControl, pvcInformer.Lister(), pvcControl, pvInformer.Lister(), recorder)
-	tikvFailover := mm.NewTiKVFailover(tikvFailoverPeriod)
-	tidbFailover := mm.NewTiDBFailover(tidbFailoverPeriod)
+	tikvFailover := mm.NewTiKVFailover(tikvFailoverPeriod, recorder)
+	tidbFailover := mm.NewTiDBFailover(tidbFailoverPeriod, recorder)
 	pdUpgrader := mm.NewPDUpgrader(pdControl, podControl, podInformer.Lister())
 	tikvUpgrader := mm.NewTiKVUpgrader(pdControl, podControl, podInformer.Lister())
 	tidbUpgrader := mm.NewTiDBUpgrader(tidbControl, podInformer.Lister())
@@ -135,6 +135,7 @@ func NewController(
 				pdUpgrader,
 				autoFailover,
 				pdFailover,
+				recorder,
 			),
 			mm.NewTiKVMemberManager(
 				pdControl,
@@ -150,6 +151,7 @@ func NewController(
 				tikvFailover,
 				tikvScaler,
 				tikvUpgrader,
+				recorder,
 			),
 			mm.NewTiDBMemberManager(
 				setControl,
@@ -163,6 +165,7 @@ func NewController(
 				tidbUpgrader,
 				autoFailover,
 				tidbFailover,
+				recorder,
 			),
 			meta.NewReclaimPolicyManager(
 				pvcInformer.Lister(),

pkg/manager/member/failover.go

@@ -15,6 +15,11 @@ package member
 
 import "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 
+const (
+	unHealthEventReason     = "Unhealthy"
+	unHealthEventMsgPattern = "%s pod[%s] is unhealthy, msg:%s"
+)
+
 // Failover implements the logic for pd/tikv/tidb's failover and recovery.
 type Failover interface {
 	Failover(*v1alpha1.TidbCluster) error

pkg/manager/member/pd_failover.go

@@ -140,8 +140,8 @@ func (pf *pdFailover) tryToMarkAPeerAsFailure(tc *v1alpha1.TidbCluster) error {
 			return err
 		}
 
-		pf.recorder.Eventf(tc, apiv1.EventTypeWarning, "PDMemberMarkedAsFailure",
-			"%s(%s) marked as a failure member", podName, pdMember.ID)
+		msg := fmt.Sprintf("pd member[%s] is unhealthy", pdMember.ID)
+		pf.recorder.Event(tc, apiv1.EventTypeWarning, unHealthEventReason, fmt.Sprintf(unHealthEventMsgPattern, "pd", podName, msg))
 
 		tc.Status.PD.FailureMembers[podName] = v1alpha1.PDFailureMember{
 			PodName:       podName,

pkg/manager/member/pd_failover_test.go

@@ -275,7 +275,7 @@ func TestPDFailoverFailover(t *testing.T) {
 				events := collectEvents(recorder.Events)
 				g.Expect(events).To(HaveLen(2))
 				g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
-				g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) marked as a failure member"))
+				g.Expect(events[1]).To(ContainSubstring("Unhealthy pd pod[test-pd-1] is unhealthy, msg:pd member[12891273174085095651] is unhealthy"))
 			},
 		},
 		{

pkg/manager/member/pd_member_manager.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	v1 "k8s.io/client-go/listers/apps/v1"
 	corelisters "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/klog"
 )
 
@@ -58,6 +59,7 @@ type pdMemberManager struct {
 	pdUpgrader   Upgrader
 	autoFailover bool
 	pdFailover   Failover
+	recorder     record.EventRecorder
 }
 
 // NewPDMemberManager returns a *pdMemberManager
@@ -75,7 +77,8 @@ func NewPDMemberManager(pdControl pdapi.PDControlInterface,
 	pdScaler Scaler,
 	pdUpgrader Upgrader,
 	autoFailover bool,
-	pdFailover Failover) manager.Manager {
+	pdFailover Failover,
+	recorder record.EventRecorder) manager.Manager {
 	return &pdMemberManager{
 		pdControl,
 		setControl,
@@ -91,7 +94,8 @@ func NewPDMemberManager(pdControl pdapi.PDControlInterface,
 		pdScaler,
 		pdUpgrader,
 		autoFailover,
-		pdFailover}
+		pdFailover,
+		recorder}
 }
 
 func (pmm *pdMemberManager) Sync(tc *v1alpha1.TidbCluster) error {

pkg/manager/member/pd_member_manager_test.go

@@ -38,6 +38,7 @@ import (
 	kubeinformers "k8s.io/client-go/informers"
 	kubefake "k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/utils/pointer"
 )
 
@@ -758,7 +759,7 @@ func newFakePDMemberManager() (*pdMemberManager, *controller.FakeStatefulSetCont
 	pdFailover := NewFakePDFailover()
 	pdUpgrader := NewFakePDUpgrader()
 	genericControll := controller.NewFakeGenericControl()
-
+	recorder := record.NewFakeRecorder(100)
 	return &pdMemberManager{
 		pdControl,
 		setControl,
@@ -775,6 +776,7 @@ func newFakePDMemberManager() (*pdMemberManager, *controller.FakeStatefulSetCont
 		pdUpgrader,
 		autoFailover,
 		pdFailover,
+		recorder,
 	}, setControl, svcControl, pdControl, podInformer.Informer().GetIndexer(), pvcInformer.Informer().GetIndexer(), podControl
 }
 

pkg/manager/member/pd_scaler_test.go

@@ -15,11 +15,11 @@ package member
 
 import (
 	"fmt"
-	"github.com/pingcap/kvproto/pkg/pdpb"
 	"testing"
 	"time"
 
 	. "github.com/onsi/gomega"
+	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/controller"
 	"github.com/pingcap/tidb-operator/pkg/label"

pkg/manager/member/tidb_failover.go

@@ -14,21 +14,26 @@
 package member
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/klog"
 )
 
 type tidbFailover struct {
 	tidbFailoverPeriod time.Duration
+	recorder           record.EventRecorder
 }
 
 // NewTiDBFailover returns a tidbFailover instance
-func NewTiDBFailover(failoverPeriod time.Duration) Failover {
+func NewTiDBFailover(failoverPeriod time.Duration, recorder record.EventRecorder) Failover {
 	return &tidbFailover{
 		tidbFailoverPeriod: failoverPeriod,
+		recorder:           recorder,
 	}
 }
 
@@ -59,6 +64,8 @@ func (tf *tidbFailover) Failover(tc *v1alpha1.TidbCluster) error {
 					PodName:   tidbMember.Name,
 					CreatedAt: metav1.Now(),
 				}
+				msg := fmt.Sprintf("tidb[%s] is unhealthy", tidbMember.Name)
+				tf.recorder.Event(tc, corev1.EventTypeWarning, unHealthEventReason, fmt.Sprintf(unHealthEventMsgPattern, "tidb", tidbMember.Name, msg))
 				break
 			}
 		}
@@ -71,7 +78,8 @@ func (tf *tidbFailover) Recover(tc *v1alpha1.TidbCluster) {
 	tc.Status.TiDB.FailureMembers = nil
 }
 
-type fakeTiDBFailover struct{}
+type fakeTiDBFailover struct {
+}
 
 // NewFakeTiDBFailover returns a fake Failover
 func NewFakeTiDBFailover() Failover {

pkg/manager/member/tidb_failover_test.go

@@ -19,6 +19,7 @@ import (
 
 	. "github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/utils/pointer"
 
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
@@ -382,7 +383,8 @@ func TestFakeTiDBFailoverRecover(t *testing.T) {
 }
 
 func newTiDBFailover() Failover {
-	return &tidbFailover{tidbFailoverPeriod: time.Duration(5 * time.Minute)}
+	recorder := record.NewFakeRecorder(100)
+	return &tidbFailover{tidbFailoverPeriod: time.Duration(5 * time.Minute), recorder: recorder}
 }
 
 func newTidbClusterForTiDBFailover() *v1alpha1.TidbCluster {

pkg/manager/member/tidb_member_manager.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	v1 "k8s.io/client-go/listers/apps/v1"
 	corelisters "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/klog"
 	"k8s.io/utils/pointer"
 )
@@ -68,6 +69,7 @@ type tidbMemberManager struct {
 	autoFailover                 bool
 	tidbFailover                 Failover
 	tidbStatefulSetIsUpgradingFn func(corelisters.PodLister, *apps.StatefulSet, *v1alpha1.TidbCluster) (bool, error)
+	recorder                     record.EventRecorder
 }
 
 // NewTiDBMemberManager returns a *tidbMemberManager
@@ -81,7 +83,8 @@ func NewTiDBMemberManager(setControl controller.StatefulSetControlInterface,
 	podLister corelisters.PodLister,
 	tidbUpgrader Upgrader,
 	autoFailover bool,
-	tidbFailover Failover) manager.Manager {
+	tidbFailover Failover,
+	recorder record.EventRecorder) manager.Manager {
 	return &tidbMemberManager{
 		setControl:                   setControl,
 		svcControl:                   svcControl,
@@ -95,6 +98,7 @@ func NewTiDBMemberManager(setControl controller.StatefulSetControlInterface,
 		autoFailover:                 autoFailover,
 		tidbFailover:                 tidbFailover,
 		tidbStatefulSetIsUpgradingFn: tidbStatefulSetIsUpgrading,
+		recorder:                     recorder,
 	}
 }
 

pkg/manager/member/tidb_member_manager_test.go

@@ -38,6 +38,7 @@ import (
 	kubefake "k8s.io/client-go/kubernetes/fake"
 	corelisters "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/utils/pointer"
 )
 
@@ -778,6 +779,7 @@ func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSet
 	tidbUpgrader := NewFakeTiDBUpgrader()
 	tidbFailover := NewFakeTiDBFailover()
 	tidbControl := controller.NewFakeTiDBControl()
+	recorder := record.NewFakeRecorder(100)
 
 	tmm := &tidbMemberManager{
 		setControl,
@@ -793,6 +795,7 @@ func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSet
 		true,
 		tidbFailover,
 		tidbStatefulSetIsUpgrading,
+		recorder,
 	}
 	indexers := &fakeIndexers{
 		pod:    podInformer.Informer().GetIndexer(),

pkg/manager/member/tikv_failover.go

@@ -14,20 +14,24 @@
 package member
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/klog"
 )
 
 type tikvFailover struct {
 	tikvFailoverPeriod time.Duration
+	recorder           record.EventRecorder
 }
 
 // NewTiKVFailover returns a tikv Failover
-func NewTiKVFailover(tikvFailoverPeriod time.Duration) Failover {
-	return &tikvFailover{tikvFailoverPeriod}
+func NewTiKVFailover(tikvFailoverPeriod time.Duration, recorder record.EventRecorder) Failover {
+	return &tikvFailover{tikvFailoverPeriod, recorder}
 }
 
 func (tf *tikvFailover) Failover(tc *v1alpha1.TidbCluster) error {
@@ -62,6 +66,8 @@ func (tf *tikvFailover) Failover(tc *v1alpha1.TidbCluster) error {
 					StoreID:   store.ID,
 					CreatedAt: metav1.Now(),
 				}
+				msg := fmt.Sprintf("store[%s] is Down", store.ID)
+				tf.recorder.Event(tc, corev1.EventTypeWarning, unHealthEventReason, fmt.Sprintf(unHealthEventMsgPattern, "tikv", podName, msg))
 			}
 		}
 	}

pkg/manager/member/tikv_failover_test.go

@@ -20,6 +20,7 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/utils/pointer"
 )
 
@@ -304,5 +305,6 @@ func TestTiKVFailoverFailover(t *testing.T) {
 }
 
 func newFakeTiKVFailover() *tikvFailover {
-	return &tikvFailover{1 * time.Hour}
+	recorder := record.NewFakeRecorder(100)
+	return &tikvFailover{1 * time.Hour, recorder}
 }

pkg/manager/member/tikv_member_manager.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	v1 "k8s.io/client-go/listers/apps/v1"
 	corelisters "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/klog"
 )
 
@@ -62,6 +63,7 @@ type tikvMemberManager struct {
 	tikvScaler                   Scaler
 	tikvUpgrader                 Upgrader
 	tikvStatefulSetIsUpgradingFn func(corelisters.PodLister, pdapi.PDControlInterface, *apps.StatefulSet, *v1alpha1.TidbCluster) (bool, error)
+	recorder                     record.EventRecorder
 }
 
 // NewTiKVMemberManager returns a *tikvMemberManager
@@ -78,7 +80,8 @@ func NewTiKVMemberManager(
 	autoFailover bool,
 	tikvFailover Failover,
 	tikvScaler Scaler,
-	tikvUpgrader Upgrader) manager.Manager {
+	tikvUpgrader Upgrader,
+	recorder record.EventRecorder) manager.Manager {
 	kvmm := tikvMemberManager{
 		pdControl:    pdControl,
 		podLister:    podLister,
@@ -93,6 +96,7 @@ func NewTiKVMemberManager(
 		tikvFailover: tikvFailover,
 		tikvScaler:   tikvScaler,
 		tikvUpgrader: tikvUpgrader,
+		recorder:     recorder,
 	}
 	kvmm.tikvStatefulSetIsUpgradingFn = tikvStatefulSetIsUpgrading
 	return &kvmm