diff --git a/tests/actions.go b/tests/actions.go index 35b54b3cbf..4327608063 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -39,11 +39,12 @@ import ( "k8s.io/client-go/kubernetes" ) -func NewOperatorActions(cli versioned.Interface, kubeCli kubernetes.Interface) OperatorActions { +func NewOperatorActions(cli versioned.Interface, kubeCli kubernetes.Interface, logDir string) OperatorActions { return &operatorActions{ cli: cli, kubeCli: kubeCli, pdControl: controller.NewDefaultPDControl(), + logDir: logDir, } } @@ -56,7 +57,7 @@ type OperatorActions interface { DeployOperator(info *OperatorInfo) error CleanOperator(info *OperatorInfo) error UpgradeOperator(info *OperatorInfo) error - DumpAllLogs(info *OperatorInfo, clusterInfo *TidbClusterInfo) error + DumpAllLogs(info *OperatorInfo, clusterInfos []*TidbClusterInfo) error DeployTidbCluster(info *TidbClusterInfo) error CleanTidbCluster(info *TidbClusterInfo) error CheckTidbClusterStatus(info *TidbClusterInfo) error @@ -103,6 +104,7 @@ type operatorActions struct { cli versioned.Interface kubeCli kubernetes.Interface pdControl controller.PDControlInterface + logDir string } var _ = OperatorActions(&operatorActions{}) @@ -219,10 +221,6 @@ func (oa *operatorActions) UpgradeOperator(info *OperatorInfo) error { return nil } -func (oa *operatorActions) DumpAllLogs(info *OperatorInfo, clusterInfo *TidbClusterInfo) error { - return nil -} - func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error { glog.Infof("begin to deploy tidb cluster cluster[%s] namespace[%s]", info.ClusterName, info.Namespace) defer func() { diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index 9f8214ed3b..73b303323c 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -49,7 +49,7 @@ func main() { glog.Fatalf("failed to get kubernetes Clientset: %v", err) } - oa := tests.NewOperatorActions(cli, kubeCli) + oa := tests.NewOperatorActions(cli, kubeCli, "/logDir") operatorInfo := &tests.OperatorInfo{ Namespace: "pingcap", @@ -59,8 +59,14 @@ func main() { SchedulerImage: "gcr.io/google-containers/hyperkube:v1.12.1", LogLevel: "2", } - perror(oa.CleanOperator(operatorInfo)) - perror(oa.DeployOperator(operatorInfo)) + if err := oa.CleanOperator(operatorInfo); err != nil { + oa.DumpAllLogs(operatorInfo, nil) + glog.Fatal(err) + } + if err = oa.DeployOperator(operatorInfo); err != nil { + oa.DumpAllLogs(operatorInfo, nil) + glog.Fatal(err) + } clusterInfo := &tests.TidbClusterInfo{ Namespace: "tidb", @@ -88,17 +94,38 @@ func main() { Args: map[string]string{}, } - perror(oa.CleanTidbCluster(clusterInfo)) - perror(oa.DeployTidbCluster(clusterInfo)) - perror(oa.CheckTidbClusterStatus(clusterInfo)) + if err = oa.CleanTidbCluster(clusterInfo); err != nil { + oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo}) + glog.Fatal(err) + } + if err = oa.DeployTidbCluster(clusterInfo); err != nil { + oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo}) + glog.Fatal(err) + } + if err = oa.CheckTidbClusterStatus(clusterInfo); err != nil { + oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo}) + glog.Fatal(err) + } clusterInfo = clusterInfo.ScaleTiDB(3) - perror(oa.ScaleTidbCluster(clusterInfo)) - perror(oa.CheckTidbClusterStatus(clusterInfo)) + if err := oa.ScaleTidbCluster(clusterInfo); err != nil { + oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo}) + glog.Fatal(err) + } + if err = oa.CheckTidbClusterStatus(clusterInfo); err != nil { + oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo}) + glog.Fatal(err) + } clusterInfo = clusterInfo.UpgradeAll("v2.1.4") - perror(oa.UpgradeTidbCluster(clusterInfo)) - perror(oa.CheckTidbClusterStatus(clusterInfo)) + if err = oa.UpgradeTidbCluster(clusterInfo); err != nil { + oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo}) + glog.Fatal(err) + } + if err = oa.CheckTidbClusterStatus(clusterInfo); err != nil { + oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo}) + glog.Fatal(err) + } restoreClusterInfo := &tests.TidbClusterInfo{ Namespace: "tidb", @@ -126,13 +153,23 @@ func main() { Args: map[string]string{}, } - perror(oa.CleanTidbCluster(restoreClusterInfo)) - perror(oa.DeployTidbCluster(restoreClusterInfo)) - perror(oa.CheckTidbClusterStatus(restoreClusterInfo)) + if err = oa.CleanTidbCluster(restoreClusterInfo); err != nil { + oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo, restoreClusterInfo}) + glog.Fatal(err) + } + if err = oa.DeployTidbCluster(restoreClusterInfo); err != nil { + oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo, restoreClusterInfo}) + glog.Fatal(err) + } + if err = oa.CheckTidbClusterStatus(restoreClusterInfo); err != nil { + oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo, restoreClusterInfo}) + glog.Fatal(err) + } backupCase := backup.NewBackupCase(oa, clusterInfo, restoreClusterInfo) if err := backupCase.Run(); err != nil { + oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo, restoreClusterInfo}) glog.Fatal(err) } } diff --git a/tests/log_dump.go b/tests/log_dump.go new file mode 100644 index 0000000000..97de55c0ad --- /dev/null +++ b/tests/log_dump.go @@ -0,0 +1,110 @@ +package tests + +import ( + "bufio" + "fmt" + "os" + "os/exec" + "path/filepath" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (oa *operatorActions) DumpAllLogs(operatorInfo *OperatorInfo, testClusters []*TidbClusterInfo) error { + logPath := fmt.Sprintf("/%s/%s", oa.logDir, "operator-stability") + if _, err := os.Stat(logPath); os.IsNotExist(err) { + err = os.MkdirAll(logPath, os.ModePerm) + if err != nil { + return err + } + } + + // dump all resources info + resourceLogFile, err := os.Create(filepath.Join(logPath, "resources")) + if err != nil { + return err + } + defer resourceLogFile.Close() + resourceWriter := bufio.NewWriter(resourceLogFile) + dumpLog("kubectl get po -owide -n kube-system", resourceWriter) + dumpLog(fmt.Sprintf("kubectl get po -owide -n %s", operatorInfo.Namespace), resourceWriter) + dumpLog("kubectl get pv", resourceWriter) + dumpLog("kubectl get pv -oyaml", resourceWriter) + dumpedNamespace := map[string]bool{} + for _, testCluster := range testClusters { + if _, exist := dumpedNamespace[testCluster.Namespace]; !exist { + dumpLog(fmt.Sprintf("kubectl get po,pvc,svc,cm,cronjobs,jobs,statefulsets,tidbclusters -owide -n %s", testCluster.Namespace), resourceWriter) + dumpLog(fmt.Sprintf("kubectl get po,pvc,svc,cm,cronjobs,jobs,statefulsets,tidbclusters -n %s -oyaml", testCluster.Namespace), resourceWriter) + dumpedNamespace[testCluster.Namespace] = true + } + } + + // dump operator components's log + operatorPods, err := oa.kubeCli.CoreV1().Pods(operatorInfo.Namespace).List(metav1.ListOptions{}) + if err != nil { + return err + } + for _, pod := range operatorPods.Items { + err := dumpPod(logPath, &pod) + if err != nil { + return err + } + } + + // dump all test clusters's logs + dumpedNamespace = map[string]bool{} + for _, testCluster := range testClusters { + if _, exist := dumpedNamespace[testCluster.Namespace]; !exist { + clusterPodList, err := oa.kubeCli.CoreV1().Pods(testCluster.Namespace).List(metav1.ListOptions{}) + if err != nil { + return err + } + for _, pod := range clusterPodList.Items { + err := dumpPod(logPath, &pod) + if err != nil { + return err + } + } + dumpedNamespace[testCluster.Namespace] = true + } + } + + return nil +} + +func dumpPod(logPath string, pod *corev1.Pod) error { + logFile, err := os.Create(filepath.Join(logPath, fmt.Sprintf("%s-%s.log", pod.Name, pod.Namespace))) + if err != nil { + return err + } + defer logFile.Close() + plogFile, err := os.Create(filepath.Join(logPath, fmt.Sprintf("%s-%s-p.log", pod.Name, pod.Namespace))) + if err != nil { + return err + } + defer plogFile.Close() + + logWriter := bufio.NewWriter(logFile) + plogWriter := bufio.NewWriter(plogFile) + defer logWriter.Flush() + defer plogWriter.Flush() + + for _, c := range pod.Spec.Containers { + dumpLog(fmt.Sprintf("kubectl logs -n %s %s -c %s", pod.Namespace, pod.GetName(), c.Name), logWriter) + dumpLog(fmt.Sprintf("kubectl logs -n %s %s -c %s -p", pod.Namespace, pod.GetName(), c.Name), plogWriter) + } + + return nil +} + +func dumpLog(cmdStr string, writer *bufio.Writer) { + writer.WriteString(fmt.Sprintf("$ %s\n", cmdStr)) + cmd := exec.Command("/bin/sh", "-c", "/usr/local/bin/"+cmdStr) + cmd.Stderr = writer + cmd.Stdout = writer + err := cmd.Run() + if err != nil { + writer.WriteString(err.Error()) + } +} diff --git a/tests/manifests/e2e.yaml b/tests/manifests/e2e.yaml index ed50c4dcc5..77e44a4559 100644 --- a/tests/manifests/e2e.yaml +++ b/tests/manifests/e2e.yaml @@ -30,4 +30,12 @@ spec: image: "" imagePullPolicy: Always command: ["sh", "-c", "/usr/local/bin/e2e"] + volumeMounts: + - mountPath: /logDir + name: logdir + volumes: + - name: logdir + hostPath: + path: /var/log + type: Directory restartPolicy: Never