Merge branch 'master' into resume_paused_backup

charts/br-federation/templates/controller-manager-deployment.yaml

@@ -73,6 +73,9 @@ spec:
          {{- if .Values.brFederationManager.leaderRetryPeriod }}
           - -leader-retry-period={{ .Values.brFederationManager.leaderRetryPeriod }}
          {{- end }}
+         {{- if .Values.brFederationManager.leaderResourceLock }}
+          - -leader-resource-lock={{ .Values.controllerManager.leaderResourceLock }}
+         {{- end }}
          {{- if .Values.brFederationManager.kubeClientQPS }}
           - -kube-client-qps={{ .Values.brFederationManager.kubeClientQPS }}
          {{- end }}

charts/br-federation/templates/controller-manager-rbac.yaml

@@ -30,6 +30,9 @@ rules:
 - apiGroups: [""]
   resources: ["endpoints", "events"]
   verbs: ["create", "get", "list", "watch", "update", "delete"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["create", "get", "list", "watch", "update","delete"]
 - apiGroups: ["federation.pingcap.com"]
   resources: ["*"]
   verbs: ["*"]
@@ -73,6 +76,9 @@ rules:
 - apiGroups: [""]
   resources: ["endpoints", "events"]
   verbs: ["create", "get", "list", "watch", "update", "delete"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["create", "get", "list", "watch", "update","delete"]
 - apiGroups: ["federation.pingcap.com"]
   resources: ["*"]
   verbs: ["*"]

charts/br-federation/values.yaml

@@ -47,6 +47,9 @@ brFederationManager:
   # leaderRenewDeadline: 10s
   ## leaderRetryPeriod is the duration the LeaderElector clients should wait between tries of actions
   # leaderRetryPeriod: 2s
+  ## leaderResourceLock is the type of resource object that will be used for locking during leader election
+  ## If using "endpoints" before and want to migrate to "leases", you should migrate to "endpointsleases" first
+  # leaderResourceLock: "leases"
 
   ## number of workers that are allowed to sync concurrently. default 5
   # workers: 5

charts/tidb-operator/templates/controller-manager-deployment.yaml

@@ -106,6 +106,9 @@ spec:
          {{- if .Values.controllerManager.leaderRetryPeriod }}
           - -leader-retry-period={{ .Values.controllerManager.leaderRetryPeriod }}
          {{- end }}
+         {{- if .Values.controllerManager.leaderResourceLock }}
+          - -leader-resource-lock={{ .Values.controllerManager.leaderResourceLock }}
+         {{- end }}
          {{- if .Values.controllerManager.kubeClientQPS }}
           - -kube-client-qps={{ .Values.controllerManager.kubeClientQPS }}
          {{- end }}

charts/tidb-operator/templates/controller-manager-rbac.yaml

@@ -38,6 +38,9 @@ rules:
 - apiGroups: [""]
   resources: ["endpoints","configmaps"]
   verbs: ["create", "get", "list", "watch", "update","delete"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["create", "get", "list", "watch", "update","delete"]
 - apiGroups: [""]
   resources: ["serviceaccounts"]
   verbs: ["create","get","update","delete"]
@@ -195,6 +198,9 @@ rules:
 - apiGroups: [""]
   resources: ["endpoints","configmaps"]
   verbs: ["create", "get", "list", "watch", "update", "delete"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["create", "get", "list", "watch", "update","delete"]
 - apiGroups: [""]
   resources: ["serviceaccounts"]
   verbs: ["create","get","update","delete"]

charts/tidb-operator/values.yaml

@@ -84,6 +84,9 @@ controllerManager:
   # leaderRenewDeadline: 10s
   ## leaderRetryPeriod is the duration the LeaderElector clients should wait between tries of actions
   # leaderRetryPeriod: 2s
+  ## leaderResourceLock is the type of resource object that will be used for locking during leader election
+  ## If using "endpoints" before and want to migrate to "leases", you should migrate to "endpointsleases" first
+  # leaderResourceLock: "leases"
 
   ## number of workers that are allowed to sync concurrently. default 5
   # workers: 5
@@ -149,7 +152,7 @@ controllerManager:
   # kubeClientBurst: 10
 
 scheduler:
-  create: true
+  create: false
   # With rbac.create=false, the user is responsible for creating this account
   # With rbac.create=true, this service account will be created
   # Also see rbac.create and clusterScoped

cmd/br-federation-manager/main.go

@@ -176,8 +176,10 @@ func main() {
 	}
 	// leader election for multiple br-federation-manager instances
 	go wait.Forever(func() {
-		leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{
-			Lock: &resourcelock.EndpointsLock{
+		var lock resourcelock.Interface
+		switch cliCfg.ResourceLock {
+		case resourcelock.EndpointsResourceLock:
+			lock = &resourcelock.EndpointsLock{
 				EndpointsMeta: metav1.ObjectMeta{
 					Namespace: ns,
 					Name:      endPointsName,
@@ -187,7 +189,51 @@ func main() {
 					Identity:      hostName,
 					EventRecorder: &record.FakeRecorder{},
 				},
-			},
+			}
+		case resourcelock.LeasesResourceLock:
+			lock = &resourcelock.LeaseLock{
+				LeaseMeta: metav1.ObjectMeta{
+					Namespace: ns,
+					Name:      endPointsName,
+				},
+				Client: kubeCli.CoordinationV1(),
+				LockConfig: resourcelock.ResourceLockConfig{
+					Identity:      hostName,
+					EventRecorder: &record.FakeRecorder{},
+				},
+			}
+		case resourcelock.EndpointsLeasesResourceLock:
+			lock = &resourcelock.MultiLock{
+				Primary: &resourcelock.EndpointsLock{
+					EndpointsMeta: metav1.ObjectMeta{
+						Namespace: ns,
+						Name:      endPointsName,
+					},
+					Client: kubeCli.CoreV1(),
+					LockConfig: resourcelock.ResourceLockConfig{
+						Identity:      hostName,
+						EventRecorder: &record.FakeRecorder{},
+					},
+				},
+				Secondary: &resourcelock.LeaseLock{
+					LeaseMeta: metav1.ObjectMeta{
+						Namespace: ns,
+						Name:      endPointsName,
+					},
+					Client: kubeCli.CoordinationV1(),
+					LockConfig: resourcelock.ResourceLockConfig{
+						Identity:      hostName,
+						EventRecorder: &record.FakeRecorder{},
+					},
+				},
+			}
+		default:
+			// we don't support configmap lock now
+			klog.Fatalf("only support endpoints, leases or endpointsleases for resource-lock, but got %s", cliCfg.ResourceLock)
+		}
+
+		leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{
+			Lock:          lock,
 			LeaseDuration: cliCfg.LeaseDuration,
 			RenewDeadline: cliCfg.RenewDeadline,
 			RetryPeriod:   cliCfg.RetryPeriod,

cmd/controller-manager/main.go

@@ -221,8 +221,10 @@ func main() {
 	}
 	// leader election for multiple tidb-controller-manager instances
 	go wait.Forever(func() {
-		leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{
-			Lock: &resourcelock.EndpointsLock{
+		var lock resourcelock.Interface
+		switch cliCfg.ResourceLock {
+		case resourcelock.EndpointsResourceLock:
+			lock = &resourcelock.EndpointsLock{
 				EndpointsMeta: metav1.ObjectMeta{
 					Namespace: ns,
 					Name:      endPointsName,
@@ -232,7 +234,51 @@ func main() {
 					Identity:      hostName,
 					EventRecorder: &record.FakeRecorder{},
 				},
-			},
+			}
+		case resourcelock.LeasesResourceLock:
+			lock = &resourcelock.LeaseLock{
+				LeaseMeta: metav1.ObjectMeta{
+					Namespace: ns,
+					Name:      endPointsName,
+				},
+				Client: kubeCli.CoordinationV1(),
+				LockConfig: resourcelock.ResourceLockConfig{
+					Identity:      hostName,
+					EventRecorder: &record.FakeRecorder{},
+				},
+			}
+		case resourcelock.EndpointsLeasesResourceLock:
+			lock = &resourcelock.MultiLock{
+				Primary: &resourcelock.EndpointsLock{
+					EndpointsMeta: metav1.ObjectMeta{
+						Namespace: ns,
+						Name:      endPointsName,
+					},
+					Client: kubeCli.CoreV1(),
+					LockConfig: resourcelock.ResourceLockConfig{
+						Identity:      hostName,
+						EventRecorder: &record.FakeRecorder{},
+					},
+				},
+				Secondary: &resourcelock.LeaseLock{
+					LeaseMeta: metav1.ObjectMeta{
+						Namespace: ns,
+						Name:      endPointsName,
+					},
+					Client: kubeCli.CoordinationV1(),
+					LockConfig: resourcelock.ResourceLockConfig{
+						Identity:      hostName,
+						EventRecorder: &record.FakeRecorder{},
+					},
+				},
+			}
+		default:
+			// we don't support configmap lock now
+			klog.Fatalf("only support endpoints, leases or endpointsleases for resource-lock, but got %s", cliCfg.ResourceLock)
+		}
+
+		leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{
+			Lock:          lock,
 			LeaseDuration: cliCfg.LeaseDuration,
 			RenewDeadline: cliCfg.RenewDeadline,
 			RetryPeriod:   cliCfg.RetryPeriod,

pkg/controller/br_fed_config.go

@@ -16,6 +16,8 @@ package controller
 import (
 	"flag"
 	"time"
+
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
 )
 
 const (
@@ -36,6 +38,7 @@ type BrFedCLIConfig struct {
 	LeaseDuration time.Duration
 	RenewDeadline time.Duration
 	RetryPeriod   time.Duration
+	ResourceLock  string
 	WaitDuration  time.Duration
 	// ResyncDuration is the resync time of informer
 	ResyncDuration time.Duration
@@ -56,6 +59,7 @@ func DefaultBrFedCLIConfig() *BrFedCLIConfig {
 		LeaseDuration:  15 * time.Second,
 		RenewDeadline:  10 * time.Second,
 		RetryPeriod:    2 * time.Second,
+		ResourceLock:   resourcelock.EndpointsResourceLock,
 		WaitDuration:   5 * time.Second,
 		ResyncDuration: 30 * time.Second,
 
@@ -75,6 +79,7 @@ func (c *BrFedCLIConfig) AddFlag(_ *flag.FlagSet) {
 	flag.DurationVar(&c.LeaseDuration, "leader-lease-duration", c.LeaseDuration, "leader-lease-duration is the duration that non-leader candidates will wait to force acquire leadership")
 	flag.DurationVar(&c.RenewDeadline, "leader-renew-deadline", c.RenewDeadline, "leader-renew-deadline is the duration that the acting master will retry refreshing leadership before giving up")
 	flag.DurationVar(&c.RetryPeriod, "leader-retry-period", c.RetryPeriod, "leader-retry-period is the duration the LeaderElector clients should wait between tries of actions")
+	flag.StringVar(&c.ResourceLock, "leader-resource-lock", c.ResourceLock, "The type of resource object that is used for locking during leader election")
 	flag.Float64Var(&c.KubeClientQPS, "kube-client-qps", c.KubeClientQPS, "The maximum QPS to the kubenetes API server from client")
 	flag.IntVar(&c.KubeClientBurst, "kube-client-burst", c.KubeClientBurst, "The maximum burst for throttle to the kubenetes API server from client")
 

pkg/controller/dependences.go

@@ -33,6 +33,7 @@ import (
 	extensionslister "k8s.io/client-go/listers/extensions/v1beta1"
 	networklister "k8s.io/client-go/listers/networking/v1"
 	storagelister "k8s.io/client-go/listers/storage/v1"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -76,6 +77,7 @@ type CLIConfig struct {
 	LeaseDuration         time.Duration
 	RenewDeadline         time.Duration
 	RetryPeriod           time.Duration
+	ResourceLock          string
 	WaitDuration          time.Duration
 	// ResyncDuration is the resync time of informer
 	ResyncDuration time.Duration
@@ -112,6 +114,7 @@ func DefaultCLIConfig() *CLIConfig {
 		LeaseDuration:          15 * time.Second,
 		RenewDeadline:          10 * time.Second,
 		RetryPeriod:            2 * time.Second,
+		ResourceLock:           resourcelock.EndpointsResourceLock,
 		WaitDuration:           5 * time.Second,
 		ResyncDuration:         30 * time.Second,
 		PodHardRecoveryPeriod:  24 * time.Hour,
@@ -151,6 +154,7 @@ func (c *CLIConfig) AddFlag(_ *flag.FlagSet) {
 	flag.DurationVar(&c.LeaseDuration, "leader-lease-duration", c.LeaseDuration, "leader-lease-duration is the duration that non-leader candidates will wait to force acquire leadership")
 	flag.DurationVar(&c.RenewDeadline, "leader-renew-deadline", c.RenewDeadline, "leader-renew-deadline is the duration that the acting master will retry refreshing leadership before giving up")
 	flag.DurationVar(&c.RetryPeriod, "leader-retry-period", c.RetryPeriod, "leader-retry-period is the duration the LeaderElector clients should wait between tries of actions")
+	flag.StringVar(&c.ResourceLock, "leader-resource-lock", c.ResourceLock, "The type of resource object that is used for locking during leader election")
 	flag.Float64Var(&c.KubeClientQPS, "kube-client-qps", c.KubeClientQPS, "The maximum QPS to the kubenetes API server from client")
 	flag.IntVar(&c.KubeClientBurst, "kube-client-burst", c.KubeClientBurst, "The maximum burst for throttle to the kubenetes API server from client")
 }

tests/pkg/fixture/fixture.go

@@ -112,8 +112,8 @@ func GetTidbCluster(ns, name, version string) *v1alpha1.TidbCluster {
 			Helper: &v1alpha1.HelperSpec{
 				Image: pointer.StringPtr(utilimage.HelperImage),
 			},
-			SchedulerName: "tidb-scheduler",
-			Timezone:      "Asia/Shanghai",
+			// SchedulerName: "tidb-scheduler", // use the default k8s scheduler now
+			Timezone: "Asia/Shanghai",
 			Labels: map[string]string{
 				ClusterCustomKey: "value",
 			},
@@ -204,8 +204,8 @@ func GetDMCluster(ns, name, version string) *v1alpha1.DMCluster {
 			Version:         version,
 			ImagePullPolicy: corev1.PullIfNotPresent,
 			PVReclaimPolicy: &deletePVP,
-			SchedulerName:   "tidb-scheduler",
-			Timezone:        "Asia/Shanghai",
+			// SchedulerName:   "tidb-scheduler",
+			Timezone: "Asia/Shanghai",
 			Labels: map[string]string{
 				ClusterCustomKey: "value",
 			},