diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 8f1351d5ce..ac884e8801 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -1611,6 +1611,9 @@ spec: after cluster creation Optional: Defaults to 64' format: int64 type: integer + schedulers-payload: + description: Only used to display + type: object schedulers-v2: description: Schedulers support for loding customized schedulers Immutable, change should be made through pd-ctl after diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index f76834fed6..f1384879c6 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -1943,6 +1943,21 @@ func schema_pkg_apis_pingcap_v1alpha1_PDScheduleConfig(ref common.ReferenceCallb }, }, }, + "schedulers-payload": { + SchemaProps: spec.SchemaProps{ + Description: "Only used to display", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, }, }, }, diff --git a/pkg/apis/pingcap/v1alpha1/pd_config.go b/pkg/apis/pingcap/v1alpha1/pd_config.go index 406fa564e2..bc75c1f659 100644 --- a/pkg/apis/pingcap/v1alpha1/pd_config.go +++ b/pkg/apis/pingcap/v1alpha1/pd_config.go @@ -337,6 +337,10 @@ type PDScheduleConfig struct { // Immutable, change should be made through pd-ctl after cluster creation // +optional Schedulers *PDSchedulerConfigs `toml:"schedulers,omitempty" json:"schedulers-v2,omitempty"` // json v2 is for the sake of compatible upgrade + + // Only used to display + // +optional + SchedulersPayload map[string]string `toml:"schedulers-payload" json:"schedulers-payload,omitempty"` } type PDSchedulerConfigs []PDSchedulerConfig diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index 6f152d33b2..2dda9cf4cd 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -1304,6 +1304,13 @@ func (in *PDScheduleConfig) DeepCopyInto(out *PDScheduleConfig) { } } } + if in.SchedulersPayload != nil { + in, out := &in.SchedulersPayload, &out.SchedulersPayload + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } diff --git a/pkg/manager/member/pd_upgrader.go b/pkg/manager/member/pd_upgrader.go index 24cdea02ab..ae9a5b5397 100644 --- a/pkg/manager/member/pd_upgrader.go +++ b/pkg/manager/member/pd_upgrader.go @@ -74,6 +74,7 @@ func (pu *pdUpgrader) gracefulUpgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Sta if controller.PodWebhookEnabled { setUpgradePartition(newSet, 0) + return nil } setUpgradePartition(newSet, *oldSet.Spec.UpdateStrategy.RollingUpdate.Partition) diff --git a/pkg/manager/member/tikv_upgrader.go b/pkg/manager/member/tikv_upgrader.go index 8283e09209..3d03b507c5 100644 --- a/pkg/manager/member/tikv_upgrader.go +++ b/pkg/manager/member/tikv_upgrader.go @@ -89,6 +89,7 @@ func (tku *tikvUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Stateful if controller.PodWebhookEnabled { setUpgradePartition(newSet, 0) + return nil } setUpgradePartition(newSet, *oldSet.Spec.UpdateStrategy.RollingUpdate.Partition) diff --git a/pkg/webhook/pod/tikv_creater.go b/pkg/webhook/pod/tikv_creater.go index 520cac225a..036e557ca3 100644 --- a/pkg/webhook/pod/tikv_creater.go +++ b/pkg/webhook/pod/tikv_creater.go @@ -14,23 +14,29 @@ package pod import ( + "encoding/json" "fmt" "strings" - "k8s.io/apimachinery/pkg/util/sets" - "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/pdapi" "github.com/pingcap/tidb-operator/pkg/webhook/util" admission "k8s.io/api/admission/v1beta1" core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog" ) const ( - tikvNotBootstrapped = `TiKV cluster not bootstrapped, please start TiKV first"` + tikvNotBootstrapped = `TiKV cluster not bootstrapped, please start TiKV first"` + evictSchedulerLeader = "evict-leader-scheduler" ) +// Payload only used to unmarshal the data from pdapi +type Payload struct { + StoreIdRanges map[string]interface{} `json:"store-id-ranges"` +} + func (pc *PodAdmissionControl) admitCreateTiKVPod(pod *core.Pod, tc *v1alpha1.TidbCluster, pdClient pdapi.PDClient) *admission.AdmissionResponse { name := pod.Name @@ -61,10 +67,9 @@ func (pc *PodAdmissionControl) admitCreateTiKVPod(pod *core.Pod, tc *v1alpha1.Ti return util.ARSuccess() } - schedulerIds := sets.String{} - for _, s := range evictLeaderSchedulers { - id := strings.Split(s, "-")[3] - schedulerIds.Insert(id) + schedulerIds, err := filterLeaderEvictScheduler(evictLeaderSchedulers, pdClient) + if err != nil { + return util.ARFail(err) } // if the pod which is going to be created already have a store and was in evictLeaderSchedulers, @@ -84,3 +89,34 @@ func (pc *PodAdmissionControl) admitCreateTiKVPod(pod *core.Pod, tc *v1alpha1.Ti return util.ARSuccess() } + +// This method is to make compatible between old pdapi version and 4.0 pdapi version. +// To get more detail, see: https://github.com/pingcap/tidb-operator/pull/1831 +func filterLeaderEvictScheduler(evictLeaderSchedulers []string, pdClient pdapi.PDClient) (sets.String, error) { + schedulerIds := sets.String{} + if len(evictLeaderSchedulers) == 1 && evictLeaderSchedulers[0] == evictSchedulerLeader { + c, err := pdClient.GetConfig() + if err != nil { + return schedulerIds, err + } + if c.Schedule != nil && c.Schedule.SchedulersPayload != nil { + v, ok := c.Schedule.SchedulersPayload[evictSchedulerLeader] + if ok { + payload := &Payload{} + err := json.Unmarshal([]byte(v), payload) + if err != nil { + return schedulerIds, err + } + for k := range payload.StoreIdRanges { + schedulerIds.Insert(k) + } + } + } + } else { + for _, s := range evictLeaderSchedulers { + id := strings.Split(s, "-")[3] + schedulerIds.Insert(id) + } + } + return schedulerIds, nil +}