From fdd44d101174652421f5764621807c8ced0e64a4 Mon Sep 17 00:00:00 2001 From: CWen Date: Mon, 25 Mar 2019 10:42:34 +0800 Subject: [PATCH] stability: support more fault injection (#345) * fault-trigger: support more fault injection --- tests/cmd/e2e/config.yaml | 23 --- tests/cmd/stability/main.go | 50 ++++++- tests/fault.go | 136 +++++++++++++----- tests/manifests/e2e-configmap.yaml | 6 + tests/pkg/fault-trigger/api/router.go | 31 +++- tests/pkg/fault-trigger/api/server.go | 34 ++++- tests/pkg/fault-trigger/client/client.go | 108 +++++++++++--- tests/pkg/fault-trigger/client/client_test.go | 79 ++-------- tests/pkg/fault-trigger/manager/etcd.go | 50 ------- .../manager/static_pod_service.go | 105 ++++++++++++++ .../{kubelet.go => systemctl_service.go} | 36 ++++- 11 files changed, 442 insertions(+), 216 deletions(-) delete mode 100644 tests/cmd/e2e/config.yaml delete mode 100644 tests/pkg/fault-trigger/manager/etcd.go create mode 100644 tests/pkg/fault-trigger/manager/static_pod_service.go rename tests/pkg/fault-trigger/manager/{kubelet.go => systemctl_service.go} (61%) diff --git a/tests/cmd/e2e/config.yaml b/tests/cmd/e2e/config.yaml deleted file mode 100644 index 2f35e38f91..0000000000 --- a/tests/cmd/e2e/config.yaml +++ /dev/null @@ -1,23 +0,0 @@ -nodes: - - physical_node: 172.16.4.39 - nodes: - - 172.16.4.171 - - 172.16.4.172 - - 172.16.4.173 - - physical_node: 172.16.4.40 - nodes: - - 172.16.4.174 - - 172.16.4.175 - - 172.16.4.176 -etcds: - - physical_node: 172.16.4.39 - nodes: - - 172.16.4.171 - - 172.16.4.172 - - 172.16.4.173 -apiservers: - - physical_node: 172.16.4.39 - nodes: - - 172.16.4.171 - - 172.16.4.172 - - 172.16.4.173 diff --git a/tests/cmd/stability/main.go b/tests/cmd/stability/main.go index 001973b168..3252fbef77 100644 --- a/tests/cmd/stability/main.go +++ b/tests/cmd/stability/main.go @@ -282,9 +282,57 @@ func main() { glog.Fatal(err) } - time.Sleep(1 * time.Minute) + time.Sleep(30 * time.Second) if err := fa.StartETCD("172.16.4.171"); err != nil { glog.Fatal(err) } + + time.Sleep(10 * time.Second) + + if err := fa.StopKubeAPIServer("172.16.4.171"); err != nil { + glog.Fatal(err) + } + + time.Sleep(30 * time.Second) + + if err := fa.StartKubeAPIServer("172.16.4.171"); err != nil { + glog.Fatal(err) + } + + time.Sleep(10 * time.Second) + + if err := fa.StopKubeScheduler("172.16.4.171"); err != nil { + glog.Fatal(err) + } + + time.Sleep(30 * time.Second) + + if err := fa.StartKubeScheduler("172.16.4.171"); err != nil { + glog.Fatal(err) + } + + time.Sleep(10 * time.Second) + + if err := fa.StopKubeControllerManager("172.16.4.171"); err != nil { + glog.Fatal(err) + } + + time.Sleep(30 * time.Second) + + if err := fa.StartKubeControllerManager("172.16.4.171"); err != nil { + glog.Fatal(err) + } + + time.Sleep(10 * time.Second) + + if err := fa.StopKubelet("172.16.4.171"); err != nil { + glog.Fatal(err) + } + + time.Sleep(30 * time.Second) + + if err := fa.StartKubelet("172.16.4.171"); err != nil { + glog.Fatal(err) + } } diff --git a/tests/fault.go b/tests/fault.go index b77e7527b9..dcffea21cf 100644 --- a/tests/fault.go +++ b/tests/fault.go @@ -11,6 +11,11 @@ import ( "k8s.io/client-go/kubernetes" ) +const ( + startAction = "start" + stopAction = "stop" +) + type FaultTriggerActions interface { StopNode(physicalNode string, node string) error StartNode(physicalNode string, node string) error @@ -18,13 +23,13 @@ type FaultTriggerActions interface { StartETCD(nodes ...string) error StopKubelet(node string) error StartKubelet(node string) error + StopKubeAPIServer(node string) error + StartKubeAPIServer(node string) error + StopKubeControllerManager(node string) error + StartKubeControllerManager(node string) error + StopKubeScheduler(node string) error + StartKubeScheduler(node string) error // TODO: support more faults - // StopKubeAPIServer() error - // StartKubeAPIServer() error - // StopKubeControllerManager() error - // StartKubeControllerManager() error - // StopKubeScheduler() error - // StartKubeScheduler() error // StopKubeProxy(node string) error // StartKubeProxy(node string) error // DiskCorruption(node string) error @@ -97,16 +102,9 @@ func (fa *faultTriggerActions) StopETCD(nodes ...string) error { } for _, node := range nodes { - faultCli := client.NewClient(client.Config{ - Addr: fa.genFaultTriggerAddr(node), - }) - - if err := faultCli.StopETCD(); err != nil { - glog.Errorf("failed to stop etcd %s: %v", node, err) + if err := fa.serviceAction(node, manager.ETCDService, stopAction); err != nil { return err } - - glog.Infof("etcd %s is stopped", node) } return nil @@ -122,16 +120,9 @@ func (fa *faultTriggerActions) StartETCD(nodes ...string) error { } for _, node := range nodes { - faultCli := client.NewClient(client.Config{ - Addr: fa.genFaultTriggerAddr(node), - }) - - if err := faultCli.StartETCD(); err != nil { - glog.Errorf("failed to start etcd %s: %v", node, err) + if err := fa.serviceAction(node, manager.ETCDService, startAction); err != nil { return err } - - glog.Infof("etcd %s is started", node) } return nil @@ -139,32 +130,103 @@ func (fa *faultTriggerActions) StartETCD(nodes ...string) error { // StopKubelet stops the kubelet service. func (fa *faultTriggerActions) StopKubelet(node string) error { - faultCli := client.NewClient(client.Config{ - Addr: fa.genFaultTriggerAddr(node), - }) + return fa.serviceAction(node, manager.KubeletService, stopAction) +} - if err := faultCli.StopKubelet(); err != nil { - glog.Errorf("failed to stop kubelet %s: %v", node, err) - return err - } +// StartKubelet starts the kubelet service. +func (fa *faultTriggerActions) StartKubelet(node string) error { + return fa.serviceAction(node, manager.KubeletService, startAction) +} - glog.Infof("kubelet %s is stopped", node) +// // StopKubeProxy stops the kube-proxy service. +//func (fa *faultTriggerActions) StopKubeProxy(node string) error { +// return fa.serviceAction(node, manager.KubeProxyService, stopAction) +//} +// +//// StartKubeProxy starts the kube-proxy service. +//func (fa *faultTriggerActions) StartKubeProxy(node string) error { +// return fa.serviceAction(node, manager.KubeProxyService, startAction) +//} + +// StopKubeScheduler stops the kube-scheduler service. +func (fa *faultTriggerActions) StopKubeScheduler(node string) error { + return fa.serviceAction(node, manager.KubeSchedulerService, stopAction) +} - return nil +// StartKubeScheduler starts the kube-scheduler service. +func (fa *faultTriggerActions) StartKubeScheduler(node string) error { + return fa.serviceAction(node, manager.KubeSchedulerService, startAction) } -// StartKubelet starts the kubelet service. -func (fa *faultTriggerActions) StartKubelet(node string) error { +// StopKubeControllerManager stops the kube-controller-manager service. +func (fa *faultTriggerActions) StopKubeControllerManager(node string) error { + return fa.serviceAction(node, manager.KubeControllerManagerService, stopAction) +} + +// StartKubeControllerManager starts the kube-controller-manager service. +func (fa *faultTriggerActions) StartKubeControllerManager(node string) error { + return fa.serviceAction(node, manager.KubeControllerManagerService, startAction) +} + +// StopKubeAPIServer stops the apiserver service. +func (fa *faultTriggerActions) StopKubeAPIServer(node string) error { + return fa.serviceAction(node, manager.KubeAPIServerService, stopAction) +} + +// StartKubeAPIServer starts the apiserver service. +func (fa *faultTriggerActions) StartKubeAPIServer(node string) error { + return fa.serviceAction(node, manager.KubeAPIServerService, startAction) +} + +func (fa *faultTriggerActions) serviceAction(node string, serverName string, action string) error { faultCli := client.NewClient(client.Config{ - Addr: node, + Addr: fa.genFaultTriggerAddr(node), }) - if err := faultCli.StartKubelet(); err != nil { - glog.Errorf("failed to start kubelet %s: %v", node, err) + var err error + switch action { + case startAction: + switch serverName { + case manager.KubeletService: + err = faultCli.StartKubelet() + case manager.KubeSchedulerService: + err = faultCli.StartKubeScheduler() + case manager.KubeControllerManagerService: + err = faultCli.StartKubeControllerManager() + case manager.KubeAPIServerService: + err = faultCli.StartKubeAPIServer() + case manager.ETCDService: + err = faultCli.StartETCD() + default: + err = fmt.Errorf("%s %s is not supported", action, serverName) + return err + } + case stopAction: + switch serverName { + case manager.KubeletService: + err = faultCli.StopKubelet() + case manager.KubeSchedulerService: + err = faultCli.StopKubeScheduler() + case manager.KubeControllerManagerService: + err = faultCli.StopKubeControllerManager() + case manager.KubeAPIServerService: + err = faultCli.StopKubeAPIServer() + case manager.ETCDService: + err = faultCli.StopETCD() + default: + err = fmt.Errorf("%s %s is not supported", action, serverName) + } + default: + err = fmt.Errorf("action %s is not supported", action) + return err + } + + if err != nil { + glog.Errorf("failed to %s %s %s: %v", action, serverName, node, err) return err } - glog.Infof("kubelet %s is started", node) + glog.Infof("%s %s %s successfully", action, serverName, node) return nil } diff --git a/tests/manifests/e2e-configmap.yaml b/tests/manifests/e2e-configmap.yaml index 0af85753bb..3f4d76e07f 100644 --- a/tests/manifests/e2e-configmap.yaml +++ b/tests/manifests/e2e-configmap.yaml @@ -28,5 +28,11 @@ data: - 172.16.4.171 - 172.16.4.172 - 172.16.4.173 + controller_manager: + - physical_node: 172.16.4.39 + nodes: + - 172.16.4.171 + - 172.16.4.172 + - 172.16.4.173 diff --git a/tests/pkg/fault-trigger/api/router.go b/tests/pkg/fault-trigger/api/router.go index 9b39afcfa7..bfeeb6916a 100644 --- a/tests/pkg/fault-trigger/api/router.go +++ b/tests/pkg/fault-trigger/api/router.go @@ -13,7 +13,12 @@ package api -import restful "github.com/emicklei/go-restful" +import ( + "fmt" + + restful "github.com/emicklei/go-restful" + "github.com/pingcap/tidb-operator/tests/pkg/fault-trigger/manager" +) const ( // APIPrefix defines a prefix string for fault-trigger api @@ -28,14 +33,26 @@ func (s *Server) newService() *restful.WebService { Produces(restful.MIME_JSON) ws.Route(ws.GET("/vms").To(s.listVMs)) - ws.Route(ws.GET("/vm/{name}/start").To(s.startVM)) - ws.Route(ws.GET("/vm/{name}/stop").To(s.stopVM)) + ws.Route(ws.POST("/vm/{name}/start").To(s.startVM)) + ws.Route(ws.POST("/vm/{name}/stop").To(s.stopVM)) + + ws.Route(ws.POST(fmt.Sprintf("/%s/start", manager.ETCDService)).To(s.startETCD)) + ws.Route(ws.POST(fmt.Sprintf("/%s/stop", manager.ETCDService)).To(s.stopETCD)) + + ws.Route(ws.POST(fmt.Sprintf("/%s/start", manager.KubeletService)).To(s.startKubelet)) + ws.Route(ws.POST(fmt.Sprintf("/%s/stop", manager.KubeletService)).To(s.stopKubelet)) + + ws.Route(ws.POST(fmt.Sprintf("/%s/start", manager.KubeAPIServerService)).To(s.startKubeAPIServer)) + ws.Route(ws.POST(fmt.Sprintf("/%s/stop", manager.KubeAPIServerService)).To(s.stopKubeAPIServer)) - ws.Route(ws.GET("/etcd/start").To(s.startETCD)) - ws.Route(ws.GET("/etcd/stop").To(s.stopETCD)) + ws.Route(ws.POST(fmt.Sprintf("/%s/start", manager.KubeSchedulerService)).To(s.startKubeScheduler)) + ws.Route(ws.POST(fmt.Sprintf("/%s/stop", manager.KubeSchedulerService)).To(s.stopKubeScheduler)) - ws.Route(ws.GET("/kubelet/start").To(s.startKubelet)) - ws.Route(ws.GET("/kubelet/stop").To(s.stopKubelet)) + ws.Route(ws.POST(fmt.Sprintf("/%s/start", manager.KubeControllerManagerService)).To(s.startKubeControllerManager)) + ws.Route(ws.POST(fmt.Sprintf("/%s/stop", manager.KubeControllerManagerService)).To(s.stopKubeControllerManager)) + // TODO: support kube-proxy + // ws.Route(ws.POST(fmt.Sprintf("/%s/start", manager.KubeProxyService)).To(s.startKubeProxy)) + // ws.Route(ws.POST(fmt.Sprintf("/%s/stop", manager.KubeProxyService)).To(s.stopKubeProxy)) return ws } diff --git a/tests/pkg/fault-trigger/api/server.go b/tests/pkg/fault-trigger/api/server.go index 6d0fce8edc..7726fc7f04 100644 --- a/tests/pkg/fault-trigger/api/server.go +++ b/tests/pkg/fault-trigger/api/server.go @@ -1,5 +1,3 @@ -// Copyright 2019 PingCAP, Inc. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -131,6 +129,38 @@ func (s *Server) stopKubelet(req *restful.Request, resp *restful.Response) { s.action(req, resp, s.mgr.StopKubelet, "stopKubelet") } +func (s *Server) startKubeAPIServer(req *restful.Request, resp *restful.Response) { + s.action(req, resp, s.mgr.StartKubeAPIServer, "startKubeAPIServer") +} + +func (s *Server) stopKubeAPIServer(req *restful.Request, resp *restful.Response) { + s.action(req, resp, s.mgr.StopKubeAPIServer, "stopKubeAPIServer") +} + +// func (s *Server) startKubeProxy(req *restful.Request, resp *restful.Response) { +// s.action(req, resp, s.mgr.StartKubeProxy, "startKubeProxy") +// } +// +// func (s *Server) stopKubeProxy(req *restful.Request, resp *restful.Response) { +// s.action(req, resp, s.mgr.StopKubeProxy, "stopKubeProxy") +// } + +func (s *Server) startKubeScheduler(req *restful.Request, resp *restful.Response) { + s.action(req, resp, s.mgr.StartKubeScheduler, "startKubeScheduler") +} + +func (s *Server) stopKubeScheduler(req *restful.Request, resp *restful.Response) { + s.action(req, resp, s.mgr.StopKubeScheduler, "stopKubeScheduler") +} + +func (s *Server) startKubeControllerManager(req *restful.Request, resp *restful.Response) { + s.action(req, resp, s.mgr.StartKubeControllerManager, "startKubeControllerManager") +} + +func (s *Server) stopKubeControllerManager(req *restful.Request, resp *restful.Response) { + s.action(req, resp, s.mgr.StopKubeControllerManager, "stopKubeControllerManager") +} + func (s *Server) action( req *restful.Request, resp *restful.Response, diff --git a/tests/pkg/fault-trigger/client/client.go b/tests/pkg/fault-trigger/client/client.go index 3f7c2348c6..48a1bc5248 100644 --- a/tests/pkg/fault-trigger/client/client.go +++ b/tests/pkg/fault-trigger/client/client.go @@ -1,11 +1,13 @@ package client import ( + "bytes" "encoding/json" "fmt" "io/ioutil" "net/http" + "github.com/golang/glog" "github.com/pingcap/tidb-operator/tests/pkg/fault-trigger/api" "github.com/pingcap/tidb-operator/tests/pkg/fault-trigger/manager" "github.com/pingcap/tidb-operator/tests/pkg/util" @@ -27,6 +29,23 @@ type Client interface { StartKubelet() error // StopKubelet stops the kubelet service StopKubelet() error + // StartKubeAPIServer starts the apiserver service + StartKubeAPIServer() error + // StopKubeAPIServer stops the apiserver service + StopKubeAPIServer() error + StartKubeScheduler() error + // StopKubeScheduler stops the kube-scheduler service + StopKubeScheduler() error + // StartKubeControllerManager starts the kube-controller-manager service + StartKubeControllerManager() error + // StopKubeControllerManager stops the kube-controller-manager service + StopKubeControllerManager() error + // TODO: support controller kube-proxy + // // StartKubeProxy starts the kube-proxy service + // StartKubeProxy() error + // // StopKubeProxy stops the kube-proxy service + // StopKubeProxy() error + // // StartKubeScheduler starts the kube-scheduler service } // client is used to communicate with the fault-trigger @@ -106,10 +125,27 @@ func (c client) get(url string) ([]byte, error) { return body, nil } +func (c *client) post(url string, data []byte) ([]byte, error) { + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("error creating request: %v", err) + } + + req.Header.Set("Content-Type", "application/json") + + _, body, err := c.do(req) + if err != nil { + return nil, err + } + + return body, nil +} + func (c *client) ListVMs() ([]*manager.VM, error) { url := util.GenURL(fmt.Sprintf("%s%s/vms", c.cfg.Addr, api.APIPrefix)) data, err := c.get(url) if err != nil { + glog.Errorf("failed to get %s: %v", url, err) return nil, err } @@ -132,7 +168,8 @@ func (c *client) StartVM(vm *manager.VM) error { } url := util.GenURL(fmt.Sprintf("%s%s/vm/%s/start", c.cfg.Addr, api.APIPrefix, vmName)) - if _, err := c.get(url); err != nil { + if _, err := c.post(url, nil); err != nil { + glog.Errorf("faled to post %s: %v", url, err) return err } @@ -150,7 +187,8 @@ func (c *client) StopVM(vm *manager.VM) error { } url := util.GenURL(fmt.Sprintf("%s%s/vm/%s/stop", c.cfg.Addr, api.APIPrefix, vmName)) - if _, err := c.get(url); err != nil { + if _, err := c.post(url, nil); err != nil { + glog.Errorf("faled to post %s: %v", url, err) return err } @@ -158,35 +196,67 @@ func (c *client) StopVM(vm *manager.VM) error { } func (c *client) StartETCD() error { - url := util.GenURL(fmt.Sprintf("%s%s/etcd/start", c.cfg.Addr, api.APIPrefix)) - if _, err := c.get(url); err != nil { - return err - } - - return nil + return c.startService(manager.ETCDService) } func (c *client) StopETCD() error { - url := util.GenURL(fmt.Sprintf("%s%s/etcd/stop", c.cfg.Addr, api.APIPrefix)) - if _, err := c.get(url); err != nil { - return err - } - - return nil + return c.stopService(manager.ETCDService) } func (c *client) StartKubelet() error { - url := util.GenURL(fmt.Sprintf("%s%s/kubelet/start", c.cfg.Addr, api.APIPrefix)) - if _, err := c.get(url); err != nil { + return c.startService(manager.KubeletService) +} + +func (c *client) StopKubelet() error { + return c.stopService(manager.KubeletService) +} + +func (c *client) StartKubeAPIServer() error { + return c.startService(manager.KubeAPIServerService) +} + +func (c *client) StopKubeAPIServer() error { + return c.stopService(manager.KubeAPIServerService) +} + +// func (c *client) StartKubeProxy() error { +// return c.startService(manager.KubeProxyService) +// } +// +// func (c *client) StopKubeProxy() error { +// return c.stopService(manager.KubeProxyService) +// } + +func (c *client) StartKubeScheduler() error { + return c.startService(manager.KubeSchedulerService) +} + +func (c *client) StopKubeScheduler() error { + return c.stopService(manager.KubeSchedulerService) +} + +func (c *client) StartKubeControllerManager() error { + return c.startService(manager.KubeControllerManagerService) +} + +func (c *client) StopKubeControllerManager() error { + return c.stopService(manager.KubeControllerManagerService) +} + +func (c *client) startService(serviceName string) error { + url := util.GenURL(fmt.Sprintf("%s%s/%s/start", c.cfg.Addr, api.APIPrefix, serviceName)) + if _, err := c.post(url, nil); err != nil { + glog.Errorf("faled to post %s: %v", url, err) return err } return nil } -func (c *client) StopKubelet() error { - url := util.GenURL(fmt.Sprintf("%s%s/kubelet/stop", c.cfg.Addr, api.APIPrefix)) - if _, err := c.get(url); err != nil { +func (c *client) stopService(serviceName string) error { + url := util.GenURL(fmt.Sprintf("%s%s/%s/stop", c.cfg.Addr, api.APIPrefix, serviceName)) + if _, err := c.post(url, nil); err != nil { + glog.Errorf("faled to post %s: %v", url, err) return err } diff --git a/tests/pkg/fault-trigger/client/client_test.go b/tests/pkg/fault-trigger/client/client_test.go index 0cdb497b81..d61d40a3d7 100644 --- a/tests/pkg/fault-trigger/client/client_test.go +++ b/tests/pkg/fault-trigger/client/client_test.go @@ -108,7 +108,7 @@ func TestStopVM(t *testing.T) { g.Expect(err).To(HaveOccurred()) } -func TestStartETCD(t *testing.T) { +func TestStartAndStopService(t *testing.T) { g := NewGomegaWithT(t) resp := &api.Response{ @@ -123,79 +123,16 @@ func TestStartETCD(t *testing.T) { })) defer ts.Close() - cli := NewClient(Config{ - Addr: ts.URL, - }) - - err := cli.StartETCD() - g.Expect(err).NotTo(HaveOccurred()) -} - -func TestStopETCD(t *testing.T) { - g := NewGomegaWithT(t) - - resp := &api.Response{ - Action: "stopETCD", - StatusCode: 200, - Message: "OK", - } - - respJSON, _ := json.Marshal(resp) - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, string(respJSON)) - })) - defer ts.Close() - - cli := NewClient(Config{ - Addr: ts.URL, - }) - - err := cli.StopETCD() - g.Expect(err).NotTo(HaveOccurred()) -} - -func TestStartKubelet(t *testing.T) { - g := NewGomegaWithT(t) - - resp := &api.Response{ - Action: "startKubelet", - StatusCode: 200, - Message: "OK", + cli := &client{ + cfg: Config{ + Addr: ts.URL, + }, + httpCli: http.DefaultClient, } - respJSON, _ := json.Marshal(resp) - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, string(respJSON)) - })) - defer ts.Close() - - cli := NewClient(Config{ - Addr: ts.URL, - }) - - err := cli.StartKubelet() + err := cli.startService(manager.ETCDService) g.Expect(err).NotTo(HaveOccurred()) -} - -func TestStopKubelet(t *testing.T) { - g := NewGomegaWithT(t) - - resp := &api.Response{ - Action: "stopKubelet", - StatusCode: 200, - Message: "OK", - } - - respJSON, _ := json.Marshal(resp) - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, string(respJSON)) - })) - defer ts.Close() - - cli := NewClient(Config{ - Addr: ts.URL, - }) - err := cli.StopKubelet() + err = cli.stopService(manager.ETCDService) g.Expect(err).NotTo(HaveOccurred()) } diff --git a/tests/pkg/fault-trigger/manager/etcd.go b/tests/pkg/fault-trigger/manager/etcd.go deleted file mode 100644 index ef4d644736..0000000000 --- a/tests/pkg/fault-trigger/manager/etcd.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package manager - -import ( - "os/exec" - - "github.com/golang/glog" -) - -// StartETCD starts the etcd service -func (m *Manager) StartETCD() error { - shell := "systemctl start etcd" - cmd := exec.Command("/bin/sh", "-c", shell) - output, err := cmd.CombinedOutput() - if err != nil { - glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) - return err - } - - glog.Info("etcd service is started") - - return nil -} - -// StopETCD stops the etcd service -func (m *Manager) StopETCD() error { - shell := "systemctl stop etcd" - cmd := exec.Command("/bin/sh", "-c", shell) - output, err := cmd.CombinedOutput() - if err != nil { - glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) - return err - } - - glog.Info("etcd service is stopped") - - return nil -} diff --git a/tests/pkg/fault-trigger/manager/static_pod_service.go b/tests/pkg/fault-trigger/manager/static_pod_service.go new file mode 100644 index 0000000000..f30c38e6d1 --- /dev/null +++ b/tests/pkg/fault-trigger/manager/static_pod_service.go @@ -0,0 +1,105 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "fmt" + "os/exec" + + "github.com/golang/glog" +) + +const ( + staticPodPath = "/etc/kubernetes/manifests" + staticPodTmpPath = "/etc/kubernetes/tmp" + kubeAPIServerManifes = "kube-apiserver.yaml" + kubeControllerManagerManifest = "kube-controller-manager.yaml" + kubeSchedulerManifest = "kube-scheduler.yaml" + + KubeAPIServerService = "kube-apiserver" + KubeSchedulerService = "kube-scheduler" + KubeControllerManagerService = "kube-controller-manager" +) + +// StartKubeScheduler starts the kube-scheduler service +func (m *Manager) StartKubeScheduler() error { + return m.startStaticPodService(KubeSchedulerService, kubeSchedulerManifest) +} + +// StopKubeScheduler stops the kube-scheduler service +func (m *Manager) StopKubeScheduler() error { + return m.stopStaticPodService(KubeSchedulerService, kubeSchedulerManifest) +} + +// StartKubeAPIServer starts the apiserver +func (m *Manager) StartKubeAPIServer() error { + return m.startStaticPodService(KubeAPIServerService, kubeAPIServerManifes) +} + +// StopKubeAPIServer stops the apiserver +func (m *Manager) StopKubeAPIServer() error { + return m.stopStaticPodService(KubeAPIServerService, kubeAPIServerManifes) +} + +// // StartKubeProxy starts the kube-proxy service +// func (m *Manager) StartKubeProxy() error { +// return m.startStaticPodService(KubeProxyService, kubeProxyManifest) +// } +// +// // StopKubeProxy stops the kube-proxy service +// func (m *Manager) StopKubeProxy() error { +// return m.stopStaticPodService(KubeProxyService, kubeProxyManifest) +// } + +// StartKubeControllerManager starts the kube-controller-manager service +func (m *Manager) StartKubeControllerManager() error { + return m.startStaticPodService(KubeControllerManagerService, kubeControllerManagerManifest) +} + +// StopKubeControllerManager stops the kube-proxy service +func (m *Manager) StopKubeControllerManager() error { + return m.stopStaticPodService(KubeControllerManagerService, kubeControllerManagerManifest) +} + +func (m *Manager) stopStaticPodService(serviceName string, fileName string) error { + maniest := fmt.Sprintf("%s/%s", staticPodPath, fileName) + shell := fmt.Sprintf("mkdir -p %s && mv %s %s", staticPodTmpPath, maniest, staticPodTmpPath) + + cmd := exec.Command("/bin/sh", "-c", shell) + output, err := cmd.CombinedOutput() + if err != nil { + glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) + return err + } + + glog.Infof("%s is stopped", serviceName) + + return nil +} + +func (m *Manager) startStaticPodService(serviceName string, fileName string) error { + maniest := fmt.Sprintf("%s/%s", staticPodTmpPath, fileName) + shell := fmt.Sprintf("mv %s %s", maniest, staticPodPath) + + cmd := exec.Command("/bin/sh", "-c", shell) + output, err := cmd.CombinedOutput() + if err != nil { + glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) + return err + } + + glog.Infof("%s is started", serviceName) + + return nil +} diff --git a/tests/pkg/fault-trigger/manager/kubelet.go b/tests/pkg/fault-trigger/manager/systemctl_service.go similarity index 61% rename from tests/pkg/fault-trigger/manager/kubelet.go rename to tests/pkg/fault-trigger/manager/systemctl_service.go index 04b6b91f89..309104e46f 100644 --- a/tests/pkg/fault-trigger/manager/kubelet.go +++ b/tests/pkg/fault-trigger/manager/systemctl_service.go @@ -14,14 +14,39 @@ package manager import ( + "fmt" "os/exec" "github.com/golang/glog" ) +const ( + ETCDService = "etcd" + KubeletService = "kubelet" +) + +// StartETCD starts the etcd service +func (m *Manager) StartETCD() error { + return m.systemctlStartService(ETCDService) +} + +// StopETCD stops the etcd service +func (m *Manager) StopETCD() error { + return m.systemctlStopService(ETCDService) +} + // StartKubelet starts the kubelet service func (m *Manager) StartKubelet() error { - shell := "systemctl start kubelet" + return m.systemctlStartService(KubeletService) +} + +// StopKubelet stops the kubelet service +func (m *Manager) StopKubelet() error { + return m.systemctlStopService(KubeletService) +} + +func (m *Manager) systemctlStartService(serviceName string) error { + shell := fmt.Sprintf("systemctl start %s", serviceName) cmd := exec.Command("/bin/sh", "-c", shell) output, err := cmd.CombinedOutput() if err != nil { @@ -29,14 +54,13 @@ func (m *Manager) StartKubelet() error { return err } - glog.Info("kubelet is started") + glog.Infof("%s is started", serviceName) return nil } -// StopKubelet stops the kubelet service -func (m *Manager) StopKubelet() error { - shell := "systemctl stop kubelet" +func (m *Manager) systemctlStopService(serviceName string) error { + shell := fmt.Sprintf("systemctl stop %s", serviceName) cmd := exec.Command("/bin/sh", "-c", shell) output, err := cmd.CombinedOutput() if err != nil { @@ -44,7 +68,7 @@ func (m *Manager) StopKubelet() error { return err } - glog.Info("kubelet is stopped") + glog.Infof("%s is stopped", serviceName) return nil }