From ba4f7e9dc7fcaa2711d4fa27bdee0b630fd2fc2a Mon Sep 17 00:00:00 2001 From: shuijing198799 <30903849+shuijing198799@users.noreply.github.com> Date: Wed, 8 May 2019 10:58:16 +0800 Subject: [PATCH] Fix bug use shareinformer without copy (#462) * use shareinformer without copy * fix bug use shareinformer without copy in kv pd and svc add webhook Add version command for tkctl (#456) * Add version command for tkctl Signed-off-by: Aylei Add tkctl user manual (#452) * Add basic documents for CLI tool Signed-off-by: Aylei * Add TOC for manual Signed-off-by: Aylei * Document about installation and shell completion Signed-off-by: Aylei * Address review comments * Fix toc anchor link modify code and add cli add tools code and modify code Add tkctl user manual (#452) * Add basic documents for CLI tool Signed-off-by: Aylei * Add TOC for manual Signed-off-by: Aylei * Document about installation and shell completion Signed-off-by: Aylei * Address review comments * Fix toc anchor link add tools code and modify code use shareinformer without copy fix bug use shareinformer without copy in kv pd and svc add webhook Add tkctl user manual (#452) * Add basic documents for CLI tool Signed-off-by: Aylei * Add TOC for manual Signed-off-by: Aylei * Document about installation and shell completion Signed-off-by: Aylei * Address review comments * Fix toc anchor link modify code and add cli add tools code and modify code Add tkctl user manual (#452) * Add basic documents for CLI tool Signed-off-by: Aylei * Add TOC for manual Signed-off-by: Aylei * Document about installation and shell completion Signed-off-by: Aylei * Address review comments * Fix toc anchor link modify code and add cli add tools code and modify code --- .gitignore | 2 + Makefile | 5 +- cmd/webhook/main.go | 84 +++++++ docs/cli-manual.md | 255 ++++++++++++++++++++++ images/tidb-operator/Dockerfile | 1 + pkg/label/label.go | 2 + pkg/manager/member/pd_member_manager.go | 8 +- pkg/manager/member/tidb_member_manager.go | 7 +- pkg/manager/member/tikv_member_manager.go | 8 +- pkg/tkctl/cmd/cmd.go | 4 + pkg/tkctl/cmd/upinfo/upinfo.go | 206 +++++++++++++++++ pkg/tkctl/cmd/version/version.go | 125 +++++++++++ pkg/webhook/route/route.go | 86 ++++++++ pkg/webhook/statefulset/statefulset.go | 81 +++++++ pkg/webhook/util/certs.go | 98 +++++++++ pkg/webhook/util/scheme.go | 39 ++++ pkg/webhook/util/util.go | 76 +++++++ pkg/webhook/webhook.go | 126 +++++++++++ webhook-rbac.yaml | 18 ++ webhook.yaml | 44 ++++ 20 files changed, 1268 insertions(+), 7 deletions(-) create mode 100644 cmd/webhook/main.go create mode 100644 docs/cli-manual.md create mode 100644 pkg/tkctl/cmd/upinfo/upinfo.go create mode 100644 pkg/tkctl/cmd/version/version.go create mode 100644 pkg/webhook/route/route.go create mode 100644 pkg/webhook/statefulset/statefulset.go create mode 100644 pkg/webhook/util/certs.go create mode 100644 pkg/webhook/util/scheme.go create mode 100644 pkg/webhook/util/util.go create mode 100644 pkg/webhook/webhook.go create mode 100644 webhook-rbac.yaml create mode 100644 webhook.yaml diff --git a/.gitignore b/.gitignore index ddbdb8b0f98..bab2459fdf6 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ vendor tests/e2e/e2e.test .orig tkc +tkctl + diff --git a/Makefile b/Makefile index ff399bb8db5..21be7adfffe 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ docker-push: docker docker: build docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest" images/tidb-operator -build: controller-manager scheduler discovery +build: controller-manager scheduler discovery webhook controller-manager: $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-controller-manager cmd/controller-manager/main.go @@ -40,6 +40,9 @@ scheduler: discovery: $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-discovery cmd/discovery/main.go +webhook: + $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-webhook cmd/webhook/main.go + e2e-setup: # ginkgo doesn't work with retool for Go 1.11 @GO111MODULE=on CGO_ENABLED=0 go get github.com/onsi/ginkgo@v1.6.0 diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go new file mode 100644 index 00000000000..69936553817 --- /dev/null +++ b/cmd/webhook/main.go @@ -0,0 +1,84 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "os/signal" + "syscall" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/webhook" + "github.com/pingcap/tidb-operator/pkg/webhook/util" +) + +func main() { + + cli, kubeCli, err := util.GetNewClient() + if err != nil { + glog.Fatalf("failed to get client: %v", err) + } + + ns := os.Getenv("NAMESPACE") + if ns == "" { + glog.Fatalf("fail to get namespace in environment") + } + + svc := os.Getenv("SERVICENAME") + if svc == "" { + glog.Fatalf("fail to get servicename in environment") + } + + // create cert file + cert, err := util.SetupServerCert(ns, svc) + if err != nil { + glog.Fatalf("fail to setup server cert: %v", err) + } + webhookServer := webhook.NewWebHookServer(kubeCli, cli, cert) + + // before start webhook server, create validating-webhook-configuration + err = webhookServer.RegisterWebhook(ns, svc) + if err != nil { + glog.Fatalf("fail to create validaing webhook configuration: %v", err) + } + + sigs := make(chan os.Signal, 1) + done := make(chan bool, 1) + + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-sigs + + // FIXME Consider whether delete the configuration when the service is shutdown. + if err := webhookServer.UnregisterWebhook(); err != nil { + glog.Errorf("fail to delete validating configuration %v", err) + } + + // Graceful shutdown the server + if err := webhookServer.Shutdown(); err != nil { + glog.Errorf("fail to shutdown server %v", err) + } + + done <- true + }() + + if err := webhookServer.Run(); err != nil { + glog.Errorf("stop http server %v", err) + } + + <-done + + glog.Infof("webhook server terminate safely.") +} diff --git a/docs/cli-manual.md b/docs/cli-manual.md new file mode 100644 index 00000000000..59a4af270e4 --- /dev/null +++ b/docs/cli-manual.md @@ -0,0 +1,255 @@ +# The TiDB Kubernetes Contorl(tkctl) User Manual + +> **Disclaimer**: The tkctl CLI tool is currently **Alpha**. The design and sub-commands may change in the future, use at your own risk. + +The TiDB Kubernetes Control(tkctl) is a command line utility for TiDB operators to operate and diagnose their TiDB clusters in kubernetes. + +- [Installation](#installation) + - [Build from Source](#build-from-source) + - [Shell Completion](#shell-completion) + - [Kubernetes Configuration](#kubernetes-configuration) +- [Commands](#commands) + - [tkctl version](#tkctl-version) + - [tkctl list](#tkctl-list) + - [tkctl use](#tkctl-use) + - [tkctl info](#tkctl-info) + - [tkctl get](#tkctl-get-component) + - [tkctl debug](#tkctl-debug-podname) + - [tkctl ctop](#tkctl-ctop) + - [tkctl help](#tkctl-help-command) + - [tkctl options](#tkctl-options) + +# Installation + +You can download the pre-built binary or build `tkctl` from source: + +### Download the Latest Pre-built Binary + +- [MacOS](http://download.pingcap.org/tkctl-darwin-amd64-latest.tgz) +- [Linux](http://download.pingcap.org/tkctl-linux-amd64-latest.tgz) +- [Windows](http://download.pingcap.org/tkctl-windows-amd64-latest.tgz) + +### Build from Source + +```shell +$ git clone https://github.com/pingcap/tidb-operator.git +$ GOOS=${YOUR_GOOS} make cli +$ mv tkctl /usr/local/bin/tkctl +``` + +## Shell Completion + +BASH +```shell +# setup autocomplete in bash into the current shell, bash-completion package should be installed first. +source <(tkctl completion bash) + +# add autocomplete permanently to your bash shell. +echo "if hash tkctl 2>/dev/null; then source <(tkctl completion bash); fi" >> ~/.bashrc +``` + +ZSH +```shell +# setup autocomplete in zsh into the current shell +source <(tkctl completion zsh) + +# add autocomplete permanently to your zsh shell +echo "if hash tkctl 2>/dev/null; then source <(tkctl completion zsh); fi" >> ~/.zshrc +``` + +## Kubernetes Configuration + +`tkctl` reuse the kubeconfig(default to `~/.kube/config`) file to talk with kubernetes cluster. You don't have to set up `kubectl` to use `tkctl`, but make sure you have `~/.kube/config` properly set. You can verify the configuration by executing: + +```shell +$ tkctl version +``` + +If you see the version of tkctl tool and version of TiDB operator installed in target cluster or "No TiDB Controller Manager found, please install one first.", `tkctl` is correctly configured to access your cluster. + +# Commands + +## tkctl version + +This command used to show the version of **tkctl** and **tidb-operator** installed in target cluster. + +Example: +``` +$ tkctl version +Client Version: v1.0.0-beta.1-p2-93-g6598b4d3e75705-dirty +TiDB Controller Manager Version: pingcap/tidb-operator:latest +TiDB Scheduler Version: pingcap/tidb-operator:latest +``` + +## tkctl list + +This command used to list all tidb clusters installed. + +| Flags | Shorthand | Description | +| ----- | --------- | ----------- | +| --all-namespaces | -A | search all namespaces | +| --output | -o | output format, one of [default,json,yaml], the default format is `default` | + +Example: + +``` +$ tkctl list -A +NAMESPACE NAME PD TIKV TIDB AGE +foo demo-cluster 3/3 3/3 2/2 11m +bar demo-cluster 3/3 3/3 1/2 11m +``` + +## tkctl use + +This command used to specify the current TiDB cluster to use, the other commands could omit `--tidbcluster` option and defaults to select current TiDB cluster if there is a current TiDB cluster set. + +Example: + +``` +$ tkctl use --namespace=foo demo-cluster +Tidb cluster switched to foo/demo-cluster +``` + +## tkctl info + +This command used to get the information of TiDB cluster, the current TiDB cluster will be used if exists. + +| Flags | Shorthand | Description | +| ----- | --------- | ----------- | +| --tidb-cluster | -t | select the tidb cluster, default to current TiDB cluster | + +Example: + +``` +$ tkctl info +Name: demo-cluster +Namespace: foo +CreationTimestamp: 2019-04-17 17:33:41 +0800 CST +Overview: + Phase Ready Desired CPU Memory Storage Version + ----- ----- ------- --- ------ ------- ------- + PD: Normal 3 3 200m 1Gi 1Gi pingcap/pd:v2.1.4 + TiKV: Normal 3 3 1000m 2Gi 10Gi pingcap/tikv:v2.1.4 + TiDB Upgrade 1 2 500m 1Gi pingcap/tidb:v2.1.4 +Endpoints(NodePort): + - 172.16.4.158:31441 + - 172.16.4.155:31441 +``` + +## tkctl get [component] + +This is a group of commands used to get the details of TiDB cluster componentes, the current TiDB cluster will be used if exists. + +Available components: `pd`, `tikv`, `tidb`, `volume`, `all`(query all components) + +| Flags | Shorthand | Description | +| ----- | --------- | ----------- | +| --tidb-cluster | -t | select the tidb cluster, default to current TiDB cluster | +| --output | -o | output format, one of [default,json,yaml], the default format is `default` | + +Example: + +``` +$ tkctl get tikv +NAME READY STATUS MEMORY CPU RESTARTS AGE NODE +demo-cluster-tikv-0 2/2 Running 2098Mi/4196Mi 0 3m19s 172.16.4.155 +demo-cluster-tikv-1 2/2 Running 2098Mi/4196Mi 0 4m8s 172.16.4.160 +demo-cluster-tikv-2 2/2 Running 2098Mi/4196Mi 0 4m45s 172.16.4.157 +$ tkctl get volume +tkctl get volume +VOLUME CLAIM STATUS CAPACITY NODE LOCAL +local-pv-d5dad2cf tikv-demo-cluster-tikv-0 Bound 1476Gi 172.16.4.155 /mnt/disks/local-pv56 +local-pv-5ade8580 tikv-demo-cluster-tikv-1 Bound 1476Gi 172.16.4.160 /mnt/disks/local-pv33 +local-pv-ed2ffe50 tikv-demo-cluster-tikv-2 Bound 1476Gi 172.16.4.157 /mnt/disks/local-pv13 +local-pv-74ee0364 pd-demo-cluster-pd-0 Bound 1476Gi 172.16.4.155 /mnt/disks/local-pv46 +local-pv-842034e6 pd-demo-cluster-pd-1 Bound 1476Gi 172.16.4.158 /mnt/disks/local-pv74 +local-pv-e54c122a pd-demo-cluster-pd-2 Bound 1476Gi 172.16.4.156 /mnt/disks/local-pv72 +``` + +## tkctl debug [pod_name] + +This command used to diagnose the Pods of TiDB cluster. It launches a debug container for you which has the nessary troubleshooting tools installed. + +| Flags | Shorthand | Description | +| ----- | --------- | ----------- | +| --image | | specify the docker image of debug container, default to `pingcap/tidb-debug:lastest` | +| --container | -c | select the container to diagnose, default to the first container of target Pod | +| --docker-socket | | specify the docker socket of cluster node, default to `/var/run/docker.sock` | + + +The default image of debug container contains almost all the related tools you may use then diagnosing, however, the image size can be kinda big. You may use `--image=pingcap/tidb-control:latest` if your just need a basic shell, `pd-ctl` and `tidb-ctl`. + +Example: +``` +$ tkctl debug demo-cluster-tikv-0 +# you may have to wait a few seconds or minutes for the debug container running, then you will get the shell prompt +``` + +## tkctl ctop + +`tkctl ctop [pod_name | node/node_name ]` + +This command used to view the real-time stats of target pod or node. Compare to `kubectl top`, `tkctl ctop` provides network and disk stats, which are important for diagnosing TiDB cluster problem. + +| Flags | Shorthand | Description | +| ----- | --------- | ----------- | +| --image | | specify the docker image of ctop, default to `quay.io/vektorlab/ctop:0.7.2` | +| --docker-socket | | specify the docker socket of cluster node, default to `/var/run/docker.sock` | + +Example: + +``` +$ tkctl ctop demo-cluster-tikv-0 +$ tkctl ctop node/172.16.4.155 +``` + +If you don't see the prompt, please wait a few seconds or minutes. + +## tkctl help [command] + +This command used to print the help message of abitrary sub command. + +``` +$ tkctl help debug +``` + +## tkctl options + +This command used to view the global flags of `tkctl`. + +Example: +``` +$ tkctl options +The following options can be passed to any command: + + --alsologtostderr=false: log to standard error as well as files + --as='': Username to impersonate for the operation + --as-group=[]: Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --cache-dir='/Users/alei/.kube/http-cache': Default HTTP cache directory + --certificate-authority='': Path to a cert file for the certificate authority + --client-certificate='': Path to a client certificate file for TLS + --client-key='': Path to a client key file for TLS + --cluster='': The name of the kubeconfig cluster to use + --context='': The name of the kubeconfig context to use + --insecure-skip-tls-verify=false: If true, the server's certificate will not be checked for validity. This will +make your HTTPS connections insecure + --kubeconfig='': Path to the kubeconfig file to use for CLI requests. + --log_backtrace_at=:0: when logging hits line file:N, emit a stack trace + --log_dir='': If non-empty, write log files in this directory + --logtostderr=true: log to standard error instead of files + -n, --namespace='': If present, the namespace scope for this CLI request + --request-timeout='0': The length of time to wait before giving up on a single server request. Non-zero values +should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. + -s, --server='': The address and port of the Kubernetes API server + --stderrthreshold=2: logs at or above this threshold go to stderr + -t, --tidbcluster='': Tidb cluster name + --token='': Bearer token for authentication to the API server + --user='': The name of the kubeconfig user to use + -v, --v=0: log level for V logs + --vmodule=: comma-separated list of pattern=N settings for file-filtered logging +``` +These options are mainly used to talk with the kubernetes cluster, there are two options that used often: + +- `--context`: choose the kubernetes cluster +- `--namespace`: choose the kubernetes namespace + diff --git a/images/tidb-operator/Dockerfile b/images/tidb-operator/Dockerfile index 52679208071..91455a25ea9 100644 --- a/images/tidb-operator/Dockerfile +++ b/images/tidb-operator/Dockerfile @@ -4,3 +4,4 @@ RUN apk add tzdata --no-cache ADD bin/tidb-controller-manager /usr/local/bin/tidb-controller-manager ADD bin/tidb-scheduler /usr/local/bin/tidb-scheduler ADD bin/tidb-discovery /usr/local/bin/tidb-discovery +ADD bin/tidb-webhook /usr/local/bin/tidb-webhook diff --git a/pkg/label/label.go b/pkg/label/label.go index e453027d92d..17ea6c6d951 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -50,6 +50,8 @@ const ( AnnPVCDeferDeleting = "tidb.pingcap.com/pvc-defer-deleting" // AnnPVCPodScheduling is pod scheduling annotation key, it represents whether the pod is scheduling AnnPVCPodScheduling = "tidb.pingcap.com/pod-scheduling" + // AnnTiDBPartition is pod annotation which TiDB pod chould upgrade to + AnnTiDBPartition string = "tidb.pingcap.com/tidb-partition" // PDLabelVal is PD label value PDLabelVal string = "pd" diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index 8d5b0797b91..9e2da56d289 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -96,7 +96,7 @@ func (pmm *pdMemberManager) syncPDServiceForTidbCluster(tc *v1alpha1.TidbCluster tcName := tc.GetName() newSvc := pmm.getNewPDServiceForTidbCluster(tc) - oldSvc, err := pmm.svcLister.Services(ns).Get(controller.PDMemberName(tcName)) + oldSvcTmp, err := pmm.svcLister.Services(ns).Get(controller.PDMemberName(tcName)) if errors.IsNotFound(err) { err = SetServiceLastAppliedConfigAnnotation(newSvc) if err != nil { @@ -108,6 +108,8 @@ func (pmm *pdMemberManager) syncPDServiceForTidbCluster(tc *v1alpha1.TidbCluster return err } + oldSvc := oldSvcTmp.DeepCopy() + equal, err := serviceEqual(newSvc, oldSvc) if err != nil { return err @@ -172,7 +174,7 @@ func (pmm *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClu return err } - oldPDSet, err := pmm.setLister.StatefulSets(ns).Get(controller.PDMemberName(tcName)) + oldPDSetTmp, err := pmm.setLister.StatefulSets(ns).Get(controller.PDMemberName(tcName)) if err != nil && !errors.IsNotFound(err) { return err } @@ -188,6 +190,8 @@ func (pmm *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClu return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for PD cluster running", ns, tcName) } + oldPDSet := oldPDSetTmp.DeepCopy() + if err := pmm.syncTidbClusterStatus(tc, oldPDSet); err != nil { glog.Errorf("failed to sync TidbCluster: [%s/%s]'s status, error: %v", ns, tcName, err) } diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 4a4eb51e769..19b145fb0b1 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -96,7 +96,7 @@ func (tmm *tidbMemberManager) syncTiDBHeadlessServiceForTidbCluster(tc *v1alpha1 tcName := tc.GetName() newSvc := tmm.getNewTiDBHeadlessServiceForTidbCluster(tc) - oldSvc, err := tmm.svcLister.Services(ns).Get(controller.TiDBPeerMemberName(tcName)) + oldSvcTmp, err := tmm.svcLister.Services(ns).Get(controller.TiDBPeerMemberName(tcName)) if errors.IsNotFound(err) { err = SetServiceLastAppliedConfigAnnotation(newSvc) if err != nil { @@ -108,6 +108,8 @@ func (tmm *tidbMemberManager) syncTiDBHeadlessServiceForTidbCluster(tc *v1alpha1 return err } + oldSvc := oldSvcTmp.DeepCopy() + equal, err := serviceEqual(newSvc, oldSvc) if err != nil { return err @@ -131,7 +133,7 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid tcName := tc.GetName() newTiDBSet := tmm.getNewTiDBSetForTidbCluster(tc) - oldTiDBSet, err := tmm.setLister.StatefulSets(ns).Get(controller.TiDBMemberName(tcName)) + oldTiDBSetTemp, err := tmm.setLister.StatefulSets(ns).Get(controller.TiDBMemberName(tcName)) if errors.IsNotFound(err) { err = SetLastAppliedConfigAnnotation(newTiDBSet) if err != nil { @@ -144,6 +146,7 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid tc.Status.TiDB.StatefulSet = &apps.StatefulSetStatus{} return nil } + oldTiDBSet := oldTiDBSetTemp.DeepCopy() if err != nil { return err } diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index e68c19474e2..bbb2ee02b56 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -121,7 +121,7 @@ func (tkmm *tikvMemberManager) syncServiceForTidbCluster(tc *v1alpha1.TidbCluste tcName := tc.GetName() newSvc := tkmm.getNewServiceForTidbCluster(tc, svcConfig) - oldSvc, err := tkmm.svcLister.Services(ns).Get(svcConfig.MemberName(tcName)) + oldSvcTmp, err := tkmm.svcLister.Services(ns).Get(svcConfig.MemberName(tcName)) if errors.IsNotFound(err) { err = SetServiceLastAppliedConfigAnnotation(newSvc) if err != nil { @@ -133,6 +133,8 @@ func (tkmm *tikvMemberManager) syncServiceForTidbCluster(tc *v1alpha1.TidbCluste return err } + oldSvc := oldSvcTmp.DeepCopy() + equal, err := serviceEqual(newSvc, oldSvc) if err != nil { return err @@ -162,7 +164,7 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl return err } - oldSet, err := tkmm.setLister.StatefulSets(ns).Get(controller.TiKVMemberName(tcName)) + oldSetTmp, err := tkmm.setLister.StatefulSets(ns).Get(controller.TiKVMemberName(tcName)) if err != nil && !errors.IsNotFound(err) { return err } @@ -179,6 +181,8 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl return nil } + oldSet := oldSetTmp.DeepCopy() + if err := tkmm.syncTidbClusterStatus(tc, oldSet); err != nil { return err } diff --git a/pkg/tkctl/cmd/cmd.go b/pkg/tkctl/cmd/cmd.go index 5877d92d8b8..ac3f858a2cc 100644 --- a/pkg/tkctl/cmd/cmd.go +++ b/pkg/tkctl/cmd/cmd.go @@ -15,6 +15,7 @@ package cmd import ( "flag" + "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/version" "io" "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/completion" @@ -23,6 +24,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/get" "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/info" "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/list" + "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/upinfo" "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/use" "github.com/pingcap/tidb-operator/pkg/tkctl/config" "github.com/spf13/cobra" @@ -67,6 +69,8 @@ func NewTkcCommand(streams genericclioptions.IOStreams) *cobra.Command { get.NewCmdGet(tkcContext, streams), info.NewCmdInfo(tkcContext, streams), use.NewCmdUse(tkcContext, streams), + version.NewCmdVersion(tkcContext, streams.Out), + upinfo.NewCmdUpInfo(tkcContext, streams), }, }, { diff --git a/pkg/tkctl/cmd/upinfo/upinfo.go b/pkg/tkctl/cmd/upinfo/upinfo.go new file mode 100644 index 00000000000..912c46863e8 --- /dev/null +++ b/pkg/tkctl/cmd/upinfo/upinfo.go @@ -0,0 +1,206 @@ +// Copyright 2019. PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package upinfo + +import ( + "fmt" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/tkctl/config" + "github.com/pingcap/tidb-operator/pkg/tkctl/readable" + "github.com/pingcap/tidb-operator/pkg/util" + "github.com/spf13/cobra" + "io" + apps "k8s.io/api/apps/v1beta1" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/kubernetes" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" +) + +const ( + upinfoLongDesc = ` + Get tidb cluster component upgrade info. + + You can omit --tidbcluster= option by running 'tkc use ', +` + upinfoExample = ` + # get current tidb cluster info (set by tkc use) + tkc upinfo + + # get specified tidb cluster component upgrade info + tkc upinfo -t another-cluster +` + infoUsage = `expected 'upinfo -t CLUSTER_NAME' for the upinfo command or +using 'tkc use' to set tidb cluster first. +` + UPDATED = "updated" + UPDATING = "updating" + WAITING = "waiting" +) + +// UpInfoOptions contains the input to the list command. +type UpInfoOptions struct { + TidbClusterName string + Namespace string + + TcCli *versioned.Clientset + KubeCli *kubernetes.Clientset + + genericclioptions.IOStreams +} + +// NewUpInfoOptions returns a UpInfoOptions +func NewUpInfoOptions(streams genericclioptions.IOStreams) *UpInfoOptions { + return &UpInfoOptions{ + IOStreams: streams, + } +} + +// NewCmdUpInfo creates the upinfo command which show the tidb cluster upgrade detail information +func NewCmdUpInfo(tkcContext *config.TkcContext, streams genericclioptions.IOStreams) *cobra.Command { + o := NewUpInfoOptions(streams) + + cmd := &cobra.Command{ + Use: "upinfo", + Short: "Show tidb upgrade info.", + Example: upinfoExample, + Long: upinfoLongDesc, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.Complete(tkcContext, cmd, args)) + cmdutil.CheckErr(o.Run()) + }, + SuggestFor: []string{"updateinfo", "upgradeinfo"}, + } + + return cmd +} + +func (o *UpInfoOptions) Complete(tkcContext *config.TkcContext, cmd *cobra.Command, args []string) error { + + clientConfig, err := tkcContext.ToTkcClientConfig() + if err != nil { + return err + } + + if tidbClusterName, ok := clientConfig.TidbClusterName(); ok { + o.TidbClusterName = tidbClusterName + } else { + return cmdutil.UsageErrorf(cmd, infoUsage) + } + + namespace, _, err := clientConfig.Namespace() + if err != nil { + return err + } + o.Namespace = namespace + + restConfig, err := clientConfig.RestConfig() + if err != nil { + return err + } + tcCli, err := versioned.NewForConfig(restConfig) + if err != nil { + return err + } + o.TcCli = tcCli + kubeCli, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return err + } + o.KubeCli = kubeCli + + return nil +} + +func (o *UpInfoOptions) Run() error { + + tc, err := o.TcCli.PingcapV1alpha1(). + TidbClusters(o.Namespace). + Get(o.TidbClusterName, metav1.GetOptions{}) + if err != nil { + return err + } + setName := controller.TiDBMemberName(tc.Name) + set, err := o.KubeCli.AppsV1beta1().StatefulSets(o.Namespace).Get(setName, metav1.GetOptions{}) + if err != nil { + return err + } + podList, err := o.KubeCli.CoreV1().Pods(o.Namespace).List(metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s,%s=%s", label.InstanceLabelKey, tc.Name, label.ComponentLabelKey, "tidb"), + }) + if err != nil { + return err + } + msg, err := renderTCUpgradeInfo(tc, set, podList) + if err != nil { + return err + } + fmt.Fprint(o.Out, msg) + return nil +} + +func renderTCUpgradeInfo(tc *v1alpha1.TidbCluster, set *apps.StatefulSet, podList *v1.PodList) (string, error) { + return readable.TabbedString(func(out io.Writer) error { + w := readable.NewPrefixWriter(out) + dbPhase := tc.Status.TiDB.Phase + w.WriteLine(readable.LEVEL_0, "Name:\t%s", tc.Name) + w.WriteLine(readable.LEVEL_0, "Namespace:\t%s", tc.Namespace) + w.WriteLine(readable.LEVEL_0, "CreationTimestamp:\t%s", tc.CreationTimestamp) + w.WriteLine(readable.LEVEL_0, "Statu:\t%s", dbPhase) + { + w.WriteLine(readable.LEVEL_1, "Name\tState\t") + w.WriteLine(readable.LEVEL_1, "----\t-----\t") + { + updateReplicas := set.Spec.UpdateStrategy.RollingUpdate.Partition + + if len(podList.Items) != 0 { + for _, pod := range podList.Items { + var state string + ordinal, err := util.GetOrdinalFromPodName(pod.Name) + if err != nil { + return err + } + if dbPhase == v1alpha1.UpgradePhase { + if (*updateReplicas) < ordinal { + state = UPDATED + } else if (*updateReplicas) == ordinal { + + state = UPDATING + + if pod.Labels[apps.ControllerRevisionHashLabelKey] == tc.Status.TiDB.StatefulSet.UpdateRevision { + if member, exist := tc.Status.TiDB.Members[pod.Name]; exist && member.Health { + state = UPDATED + } + } + + } else { + state = WAITING + } + } else { + state = UPDATED + } + w.WriteLine(readable.LEVEL_1, "%s\t%s\t", pod.Name, state) + } + } else { + w.WriteLine(readable.LEVEL_1, "no resource found") + } + } + } + return nil + }) +} diff --git a/pkg/tkctl/cmd/version/version.go b/pkg/tkctl/cmd/version/version.go new file mode 100644 index 00000000000..c97eee3b429 --- /dev/null +++ b/pkg/tkctl/cmd/version/version.go @@ -0,0 +1,125 @@ +// Copyright 2019. PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +import ( + "fmt" + "io" + + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/tkctl/config" + "github.com/pingcap/tidb-operator/version" + "github.com/spf13/cobra" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/apis/core" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" +) + +const ( + versionExample = ` + # Print the cli version and tidb operator version + tkctl version +` +) + +type VersionOptions struct { + ClientOnly bool + + io.Writer +} + +func NewCmdVersion(tkcContext *config.TkcContext, out io.Writer) *cobra.Command { + options := &VersionOptions{ + Writer: out, + } + cmd := &cobra.Command{ + Use: "version", + Short: "Print the client & server version", + Example: versionExample, + Run: func(_ *cobra.Command, _ []string) { + cmdutil.CheckErr(options.runVersion(tkcContext)) + }, + } + + cmd.Flags().BoolVarP(&options.ClientOnly, "client-only", "c", options.ClientOnly, + "show only client version") + + return cmd +} + +func (o *VersionOptions) runVersion(tkcContext *config.TkcContext) error { + restConfig, err := tkcContext.ToRESTConfig() + if err != nil { + return err + } + kubeCli, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return err + } + + clientVersion := version.Get() + + fmt.Fprintf(o, "Client Version: %s\n", clientVersion) + + if o.ClientOnly { + return nil + } + + controllers, err := kubeCli.AppsV1(). + Deployments(core.NamespaceAll). + List(v1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s,%s=%s", label.ComponentLabelKey, "controller-manager", label.InstanceLabelKey, "tidb-operator"), + }) + if err != nil { + return err + } + schedulers, err := kubeCli.AppsV1(). + Deployments(core.NamespaceAll). + List(v1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s,%s=%s", label.ComponentLabelKey, "scheduler", label.InstanceLabelKey, "tidb-operator"), + }) + if err != nil { + return nil + } + + // TODO: add version endpoint in tidb-controller-manager + // There's no version endpoint of tidb-controller-manager and tidb-scheduler, use image instead + if len(controllers.Items) == 0 { + fmt.Fprintf(o, "No TiDB Controller Manager found, please install one first\n") + } else if len(controllers.Items) == 1 { + fmt.Fprintf(o, "TiDB Controller Manager Version: %s\n", controllers.Items[0].Spec.Template.Spec.Containers[0].Image) + } else { + fmt.Fprintf(o, "TiDB Controller Manager Versions:\n") + for _, item := range controllers.Items { + fmt.Fprintf(o, "\t%s: %s\n", item.Name, item.Spec.Template.Spec.Containers[0].Image) + } + } + if len(schedulers.Items) == 0 { + fmt.Fprintf(o, "No TiDB Scheduler found, please install one first\n") + } else if len(schedulers.Items) == 1 { + fmt.Fprintf(o, "TiDB Scheduler Version:\n") + for _, container := range schedulers.Items[0].Spec.Template.Spec.Containers { + fmt.Fprintf(o, "\t%s: %s\n", container.Name, container.Image) + } + } else { + // warn for multiple scheduler + fmt.Fprintf(o, "WARN: more than one TiDB Scheduler instance found, this is un-supported and may lead to un-expected behavior:\n") + for _, item := range schedulers.Items { + fmt.Fprintf(o, "\t%s\n", item.Name) + } + } + + return nil +} diff --git a/pkg/webhook/route/route.go b/pkg/webhook/route/route.go new file mode 100644 index 00000000000..320fa11a261 --- /dev/null +++ b/pkg/webhook/route/route.go @@ -0,0 +1,86 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package route + +import ( + "encoding/json" + "errors" + "io/ioutil" + "net/http" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/webhook/statefulset" + "github.com/pingcap/tidb-operator/pkg/webhook/util" + "k8s.io/api/admission/v1beta1" +) + +// admitFunc is the type we use for all of our validators +type admitFunc func(v1beta1.AdmissionReview) *v1beta1.AdmissionResponse + +// serve handles the http portion of a request prior to handing to an admit +// function +func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { + + var body []byte + var contentType string + responseAdmissionReview := v1beta1.AdmissionReview{} + requestedAdmissionReview := v1beta1.AdmissionReview{} + deserializer := util.GetCodec() + + // The AdmissionReview that will be returned + if r.Body != nil { + if data, err := ioutil.ReadAll(r.Body); err == nil { + body = data + } else { + responseAdmissionReview.Response = util.ARFail(err) + goto returnData + } + } else { + err := errors.New("request body is nil!") + responseAdmissionReview.Response = util.ARFail(err) + goto returnData + } + + // verify the content type is accurate + contentType = r.Header.Get("Content-Type") + if contentType != "application/json" { + err := errors.New("expect application/json") + responseAdmissionReview.Response = util.ARFail(err) + goto returnData + } + + // The AdmissionReview that was sent to the webhook + if _, _, err := deserializer.Decode(body, nil, &requestedAdmissionReview); err != nil { + responseAdmissionReview.Response = util.ARFail(err) + } else { + // pass to admitFunc + responseAdmissionReview.Response = admit(requestedAdmissionReview) + } + + // Return the same UID + responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID + +returnData: + respBytes, err := json.Marshal(responseAdmissionReview) + if err != nil { + glog.Errorf("%v", err) + } + if _, err := w.Write(respBytes); err != nil { + glog.Errorf("%v", err) + } +} + +func ServeStatefulSets(w http.ResponseWriter, r *http.Request) { + serve(w, r, statefulset.AdmitStatefulSets) +} diff --git a/pkg/webhook/statefulset/statefulset.go b/pkg/webhook/statefulset/statefulset.go new file mode 100644 index 00000000000..e82122c1b39 --- /dev/null +++ b/pkg/webhook/statefulset/statefulset.go @@ -0,0 +1,81 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package statefulset + +import ( + "errors" + "fmt" + "strconv" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/webhook/util" + "k8s.io/api/admission/v1beta1" + apps "k8s.io/api/apps/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func AdmitStatefulSets(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { + glog.Infof("admit statefulsets") + + setResource := metav1.GroupVersionResource{Group: "apps", Version: "v1beta1", Resource: "statefulsets"} + if ar.Request.Resource != setResource { + err := fmt.Errorf("expect resource to be %s", setResource) + glog.Errorf("%v", err) + return util.ARFail(err) + } + + cli, _, err := util.GetNewClient() + if err != nil { + glog.Errorf("failed to get kubernetes Clientset: %v", err) + return util.ARFail(err) + } + + name := ar.Request.Name + namespace := ar.Request.Namespace + + raw := ar.Request.OldObject.Raw + set := apps.StatefulSet{} + deserializer := util.GetCodec() + if _, _, err := deserializer.Decode(raw, nil, &set); err != nil { + glog.Error(err) + return util.ARFail(err) + } + + tc, err := cli.PingcapV1alpha1().TidbClusters(namespace).Get(set.Labels[label.InstanceLabelKey], metav1.GetOptions{}) + if err != nil { + glog.Errorf("fail to fetch tidbcluster info namespace %s clustername(instance) %s err %v", namespace, set.Labels[label.InstanceLabelKey], err) + return util.ARFail(err) + } + + if set.Labels[label.ComponentLabelKey] == "tidb" { + protect, ok := tc.Annotations[label.AnnTiDBPartition] + + if ok { + partition, err := strconv.ParseInt(protect, 10, 32) + if err != nil { + glog.Errorf("fail to convert protect to int namespace %s name %s err %v", namespace, name, err) + return util.ARFail(err) + } + + if (*set.Spec.UpdateStrategy.RollingUpdate.Partition) <= int32(partition) && tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { + glog.Infof("set has been protect by annotations name %s namespace %s", name, namespace) + return util.ARFail(errors.New("protect by annotation")) + } + } + } + + return util.ARSuccess() +} diff --git a/pkg/webhook/util/certs.go b/pkg/webhook/util/certs.go new file mode 100644 index 00000000000..fd8028cd1f4 --- /dev/null +++ b/pkg/webhook/util/certs.go @@ -0,0 +1,98 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "crypto/x509" + "io/ioutil" + "os" + + "github.com/golang/glog" + "k8s.io/client-go/util/cert" +) + +type CertContext struct { + Cert []byte + Key []byte + SigningCert []byte +} + +// Setup the server cert. For example, user apiservers and admission webhooks +// can use the cert to prove their identify to the kube-apiserver +func SetupServerCert(namespaceName, serviceName string) (*CertContext, error) { + certDir, err := ioutil.TempDir("", "create-server-cert") + if err != nil { + glog.Errorf("Failed to create a temp dir for cert generation %v", err) + return nil, err + } + defer os.RemoveAll(certDir) + signingKey, err := cert.NewPrivateKey() + if err != nil { + glog.Errorf("Failed to create CA private key %v", err) + return nil, err + } + signingCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "e2e-server-cert-ca"}, signingKey) + if err != nil { + glog.Errorf("Failed to create CA cert for apiserver %v", err) + return nil, err + } + caCertFile, err := ioutil.TempFile(certDir, "ca.crt") + if err != nil { + glog.Errorf("Failed to create a temp file for ca cert generation %v", err) + return nil, err + } + if err := ioutil.WriteFile(caCertFile.Name(), cert.EncodeCertPEM(signingCert), 0644); err != nil { + glog.Errorf("Failed to write CA cert %v", err) + return nil, err + } + key, err := cert.NewPrivateKey() + if err != nil { + glog.Errorf("Failed to create private key for %v", err) + return nil, err + } + signedCert, err := cert.NewSignedCert( + cert.Config{ + CommonName: serviceName + "." + namespaceName + ".svc", + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }, + key, signingCert, signingKey, + ) + if err != nil { + glog.Errorf("Failed to create cert%v", err) + return nil, err + } + certFile, err := ioutil.TempFile(certDir, "server.crt") + if err != nil { + glog.Errorf("Failed to create a temp file for cert generation %v", err) + return nil, err + } + keyFile, err := ioutil.TempFile(certDir, "server.key") + if err != nil { + glog.Errorf("Failed to create a temp file for key generation %v", err) + return nil, err + } + if err = ioutil.WriteFile(certFile.Name(), cert.EncodeCertPEM(signedCert), 0600); err != nil { + glog.Errorf("Failed to write cert file %v", err) + return nil, err + } + if err = ioutil.WriteFile(keyFile.Name(), cert.EncodePrivateKeyPEM(key), 0644); err != nil { + glog.Errorf("Failed to write key file %v", err) + return nil, err + } + return &CertContext{ + Cert: cert.EncodeCertPEM(signedCert), + Key: cert.EncodePrivateKeyPEM(key), + SigningCert: cert.EncodeCertPEM(signingCert), + }, nil +} diff --git a/pkg/webhook/util/scheme.go b/pkg/webhook/util/scheme.go new file mode 100644 index 00000000000..d655a35940b --- /dev/null +++ b/pkg/webhook/util/scheme.go @@ -0,0 +1,39 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + admissionv1beta1 "k8s.io/api/admission/v1beta1" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() + +func init() { + addToScheme(scheme) +} + +func addToScheme(scheme *runtime.Scheme) { + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(admissionv1beta1.AddToScheme(scheme)) + utilruntime.Must(admissionregistrationv1beta1.AddToScheme(scheme)) +} + +func GetCodec() runtime.Decoder { + return serializer.NewCodecFactory(scheme).UniversalDeserializer() +} diff --git a/pkg/webhook/util/util.go b/pkg/webhook/util/util.go new file mode 100644 index 00000000000..dcb42100561 --- /dev/null +++ b/pkg/webhook/util/util.go @@ -0,0 +1,76 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "crypto/tls" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +func GetNewClient() (versioned.Interface, kubernetes.Interface, error) { + cfg, err := rest.InClusterConfig() + if err != nil { + glog.Errorf("failed to get config: %v", err) + return nil, nil, err + } + + cli, err := versioned.NewForConfig(cfg) + if err != nil { + glog.Errorf("failed to create Clientset: %v", err) + return nil, nil, err + } + + kubeCli, err := kubernetes.NewForConfig(cfg) + if err != nil { + glog.Errorf("failed to get kubernetes Clientset: %v", err) + return nil, nil, err + } + return cli, kubeCli, nil +} + +// toAdmissionResponse is a helper function to create an AdmissionResponse +// with an embedded error +func ARFail(err error) *v1beta1.AdmissionResponse { + return &v1beta1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Message: err.Error(), + Reason: metav1.StatusReasonNotAcceptable, + }, + } +} + +// toAdmissionResponse return allow to action +func ARSuccess() *v1beta1.AdmissionResponse { + return &v1beta1.AdmissionResponse{ + Allowed: true, + } +} + +// config tls cert for server +func ConfigTLS(cert []byte, key []byte) (*tls.Config, error) { + sCert, err := tls.X509KeyPair(cert, key) + if err != nil { + return nil, err + } + return &tls.Config{ + Certificates: []tls.Certificate{sCert}, + }, nil +} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go new file mode 100644 index 00000000000..b3cb904273a --- /dev/null +++ b/pkg/webhook/webhook.go @@ -0,0 +1,126 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package webhook + +import ( + "net/http" + "time" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/webhook/route" + "github.com/pingcap/tidb-operator/pkg/webhook/util" + admissionV1beta1 "k8s.io/api/admissionregistration/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +type WebhookServer struct { + // kubernetes client interface + KubeCli kubernetes.Interface + // operator client interface + Cli versioned.Interface + // cert context ca.crt key server.crt + Context *util.CertContext + // configuration name + ConfigName string + // http server + Server *http.Server +} + +func NewWebHookServer(kubecli kubernetes.Interface, cli versioned.Interface, context *util.CertContext) *WebhookServer { + + http.HandleFunc("/statefulsets", route.ServeStatefulSets) + + sCert, err := util.ConfigTLS(context.Cert, context.Key) + + if err != nil { + glog.Fatalf("failed to create scert file %v", err) + } + + server := &http.Server{ + Addr: ":443", + TLSConfig: sCert, + } + + return &WebhookServer{ + KubeCli: kubecli, + Cli: cli, + Context: context, + ConfigName: "validating-webhook-configuration", + Server: server, + } +} + +func (ws *WebhookServer) Run() error { + return ws.Server.ListenAndServeTLS("", "") +} + +func (ws *WebhookServer) Shutdown() error { + return ws.Server.Shutdown(nil) +} + +func strPtr(s string) *string { return &s } + +func (ws *WebhookServer) RegisterWebhook(namespace string, svcName string) error { + + policyFail := admissionV1beta1.Fail + + _, err := ws.KubeCli.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Create(&admissionV1beta1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: ws.ConfigName, + }, + Webhooks: []admissionV1beta1.Webhook{ + { + Name: "admit-statefulset-webhook.k8s.io", + Rules: []admissionV1beta1.RuleWithOperations{{ + Operations: []admissionV1beta1.OperationType{admissionV1beta1.Update}, + Rule: admissionV1beta1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1beta1"}, + Resources: []string{"statefulsets"}, + }, + }}, + ClientConfig: admissionV1beta1.WebhookClientConfig{ + Service: &admissionV1beta1.ServiceReference{ + Namespace: namespace, + Name: svcName, + Path: strPtr("/statefulsets"), + }, + CABundle: ws.Context.SigningCert, + }, + FailurePolicy: &policyFail, + }, + }, + }) + + if err != nil { + glog.Errorf("registering webhook config %s with namespace %s error %v", ws.ConfigName, namespace, err) + return err + } + + // The webhook configuration is honored in 10s. + time.Sleep(10 * time.Second) + + return nil +} + +func (ws *WebhookServer) UnregisterWebhook() error { + err := ws.KubeCli.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Delete(ws.ConfigName, nil) + if err != nil && !errors.IsNotFound(err) { + glog.Errorf("failed to delete webhook config %v", err) + } + return nil +} diff --git a/webhook-rbac.yaml b/webhook-rbac.yaml new file mode 100644 index 00000000000..c4ed1acee7e --- /dev/null +++ b/webhook-rbac.yaml @@ -0,0 +1,18 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: admission-webhook-example-rbac +subjects: +- kind: ServiceAccount + namespace: pingcap + name: admission-webhook-example-sa +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + namespace: pingcap + name: admission-webhook-example-sa diff --git a/webhook.yaml b/webhook.yaml new file mode 100644 index 00000000000..026f4b3137d --- /dev/null +++ b/webhook.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + name: admission-webhook-example-svc + namespace: pingcap + labels: + app: admission-webhook-example +spec: + ports: + - port: 443 + targetPort: 443 + selector: + app: admission-webhook-example +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admission-webhook-example-deployment + namespace: pingcap + labels: + app: admission-webhook-example +spec: + replicas: 1 + selector: + matchLabels: + app: admission-webhook-example + template: + metadata: + namespace: pingcap + labels: + app: admission-webhook-example + spec: + serviceAccount: admission-webhook-example-sa + containers: + - name: admission-webhook-example + image: hub.pingcap.net/yinliang/pingcap/tidb-operator:latest + imagePullPolicy: Always + command: + - /usr/local/bin/tidb-webhook + env: + - name: NAMESPACE + value: pingcap + - name: SERVICENAME + value: admission-webhook-example-svc