Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make evict leader scheduler compatitable #1831

Merged
merged 9 commits into from
Mar 2, 2020
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions manifests/crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1611,6 +1611,9 @@ spec:
after cluster creation Optional: Defaults to 64'
format: int64
type: integer
schedulers-payload:
description: Only used to display
type: object
schedulers-v2:
description: Schedulers support for loding customized schedulers
Immutable, change should be made through pd-ctl after
Expand Down
15 changes: 15 additions & 0 deletions pkg/apis/pingcap/v1alpha1/openapi_generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions pkg/apis/pingcap/v1alpha1/pd_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,10 @@ type PDScheduleConfig struct {
// Immutable, change should be made through pd-ctl after cluster creation
// +optional
Schedulers *PDSchedulerConfigs `toml:"schedulers,omitempty" json:"schedulers-v2,omitempty"` // json v2 is for the sake of compatible upgrade

// Only used to display
// +optional
SchedulersPayload map[string]string `toml:"schedulers-payload" json:"schedulers-payload,omitempty"`
}

type PDSchedulerConfigs []PDSchedulerConfig
Expand Down
7 changes: 7 additions & 0 deletions pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pkg/manager/member/pd_upgrader.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ func (pu *pdUpgrader) gracefulUpgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Sta

if controller.PodWebhookEnabled {
setUpgradePartition(newSet, 0)
return nil
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

During my test, I found the bug in upgrading and I fixed it.

}

setUpgradePartition(newSet, *oldSet.Spec.UpdateStrategy.RollingUpdate.Partition)
Expand Down
1 change: 1 addition & 0 deletions pkg/manager/member/tikv_upgrader.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ func (tku *tikvUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Stateful

if controller.PodWebhookEnabled {
setUpgradePartition(newSet, 0)
return nil
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto.

}

setUpgradePartition(newSet, *oldSet.Spec.UpdateStrategy.RollingUpdate.Partition)
Expand Down
53 changes: 46 additions & 7 deletions pkg/webhook/pod/tikv_creater.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,29 @@
package pod

import (
"encoding/json"
"fmt"
"strings"

"k8s.io/apimachinery/pkg/util/sets"

"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/pdapi"
"github.com/pingcap/tidb-operator/pkg/webhook/util"
admission "k8s.io/api/admission/v1beta1"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
)

const (
tikvNotBootstrapped = `TiKV cluster not bootstrapped, please start TiKV first"`
tikvNotBootstrapped = `TiKV cluster not bootstrapped, please start TiKV first"`
evictSchedulerLeader = "evict-leader-scheduler"
)

// Payload only used to unmarshal the data from pdapi
type Payload struct {
StoreIdRanges map[string]interface{} `json:"store-id-ranges"`
}

func (pc *PodAdmissionControl) admitCreateTiKVPod(pod *core.Pod, tc *v1alpha1.TidbCluster, pdClient pdapi.PDClient) *admission.AdmissionResponse {

name := pod.Name
Expand Down Expand Up @@ -61,10 +67,9 @@ func (pc *PodAdmissionControl) admitCreateTiKVPod(pod *core.Pod, tc *v1alpha1.Ti
return util.ARSuccess()
}

schedulerIds := sets.String{}
for _, s := range evictLeaderSchedulers {
id := strings.Split(s, "-")[3]
schedulerIds.Insert(id)
schedulerIds, err := filterLeaderEvictScheduler(evictLeaderSchedulers, pdClient)
if err != nil {
return util.ARFail(err)
}

// if the pod which is going to be created already have a store and was in evictLeaderSchedulers,
Expand All @@ -84,3 +89,37 @@ func (pc *PodAdmissionControl) admitCreateTiKVPod(pod *core.Pod, tc *v1alpha1.Ti

return util.ARSuccess()
}

// This method is to make compatible between old pdapi version and 4.0 pdapi version.
// To get more detail, see: https://github.com/pingcap/tidb-operator/pull/1831
func filterLeaderEvictScheduler(evictLeaderSchedulers []string, pdClient pdapi.PDClient) (sets.String, error) {
schedulerIds := sets.String{}
if len(evictLeaderSchedulers) == 1 && evictLeaderSchedulers[0] == evictSchedulerLeader {
c, err := pdClient.GetConfig()
if err != nil {
return schedulerIds, err
}
if c.Schedule != nil {
if c.Schedule.SchedulersPayload == nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's nil, why still need to continue to the following codes?

Copy link
Contributor Author

@Yisaer Yisaer Mar 1, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not necessary, just solve the nil or empty payload with the same logic.

c.Schedule.SchedulersPayload = map[string]string{}
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can just change to:

if c.Schedule != nil && c.Schedule.SchedulersPayload != nil {

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updatd, PTAL

v, ok := c.Schedule.SchedulersPayload[evictSchedulerLeader]
DanielZhangQD marked this conversation as resolved.
Show resolved Hide resolved
if ok {
payload := &Payload{}
err := json.Unmarshal([]byte(v), payload)
if err != nil {
return schedulerIds, err
}
for k := range payload.StoreIdRanges {
schedulerIds.Insert(k)
}
}
}
} else {
for _, s := range evictLeaderSchedulers {
id := strings.Split(s, "-")[3]
schedulerIds.Insert(id)
}
}
return schedulerIds, nil
}