log dump when test failed

tests/actions.go

@@ -36,11 +36,12 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
-func NewOperatorActions(cli versioned.Interface, kubeCli kubernetes.Interface) OperatorActions {
+func NewOperatorActions(cli versioned.Interface, kubeCli kubernetes.Interface, logDir string) OperatorActions {
 	return &operatorActions{
 		cli:       cli,
 		kubeCli:   kubeCli,
 		pdControl: controller.NewDefaultPDControl(),
+		logDir:    logDir,
 	}
 }
 
@@ -53,7 +54,7 @@ type OperatorActions interface {
 	DeployOperator(info *OperatorInfo) error
 	CleanOperator(info *OperatorInfo) error
 	UpgradeOperator(info *OperatorInfo) error
-	DumpAllLogs(info *OperatorInfo, clusterInfo *TidbClusterInfo) error
+	DumpAllLogs(info *OperatorInfo, clusterInfos []*TidbClusterInfo) error
 	DeployTidbCluster(info *TidbClusterInfo) error
 	CleanTidbCluster(info *TidbClusterInfo) error
 	CheckTidbClusterStatus(info *TidbClusterInfo) error
@@ -99,6 +100,7 @@ type operatorActions struct {
 	cli       versioned.Interface
 	kubeCli   kubernetes.Interface
 	pdControl controller.PDControlInterface
+	logDir    string
 }
 
 type OperatorInfo struct {
@@ -207,10 +209,6 @@ func (oa *operatorActions) UpgradeOperator(info *OperatorInfo) error {
 	return nil
 }
 
-func (oa *operatorActions) DumpAllLogs(info *OperatorInfo, clusterInfo *TidbClusterInfo) error {
-	return nil
-}
-
 func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error {
 	glog.Infof("begin to deploy tidb cluster")
 	defer func() {

tests/cmd/e2e/main.go

@@ -42,7 +42,7 @@ func main() {
 		glog.Fatalf("failed to get kubernetes Clientset: %v", err)
 	}
 
-	oa := tests.NewOperatorActions(cli, kubeCli)
+	oa := tests.NewOperatorActions(cli, kubeCli, "/logDir")
 
 	operatorInfo := &tests.OperatorInfo{
 		Namespace:      "pingcap",
@@ -53,9 +53,11 @@ func main() {
 		LogLevel:       "2",
 	}
 	if err := oa.CleanOperator(operatorInfo); err != nil {
+		oa.DumpAllLogs(operatorInfo, nil)
 		glog.Fatal(err)
 	}
 	if err := oa.DeployOperator(operatorInfo); err != nil {
+		oa.DumpAllLogs(operatorInfo, nil)
 		glog.Fatal(err)
 	}
 
@@ -71,12 +73,15 @@ func main() {
 		Args:             map[string]string{},
 	}
 	if err := oa.CleanTidbCluster(clusterInfo); err != nil {
+		oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo})
 		glog.Fatal(err)
 	}
 	if err := oa.DeployTidbCluster(clusterInfo); err != nil {
+		oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo})
 		glog.Fatal(err)
 	}
 	if err := oa.CheckTidbClusterStatus(clusterInfo); err != nil {
+		oa.DumpAllLogs(operatorInfo, []*tests.TidbClusterInfo{clusterInfo})
 		glog.Fatal(err)
 	}
 

tests/log_dump.go

@@ -0,0 +1,110 @@
+package tests
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (oa *operatorActions) DumpAllLogs(operatorInfo *OperatorInfo, testClusters []*TidbClusterInfo) error {
+	logPath := fmt.Sprintf("/%s/%s", oa.logDir, "operator-stability")
+	if _, err := os.Stat(logPath); os.IsNotExist(err) {
+		err = os.MkdirAll(logPath, os.ModePerm)
+		if err != nil {
+			return err
+		}
+	}
+
+	// dump all resources info
+	resourceLogFile, err := os.Create(filepath.Join(logPath, "resources"))
+	if err != nil {
+		return err
+	}
+	defer resourceLogFile.Close()
+	resourceWriter := bufio.NewWriter(resourceLogFile)
+	dumpLog("kubectl get po -owide -n kube-system", resourceWriter)
+	dumpLog(fmt.Sprintf("kubectl get po -owide -n %s", operatorInfo.Namespace), resourceWriter)
+	dumpLog("kubectl get pv", resourceWriter)
+	dumpLog("kubectl get pv -oyaml", resourceWriter)
+	dumpedNamespace := map[string]bool{}
+	for _, testCluster := range testClusters {
+		if _, exist := dumpedNamespace[testCluster.Namespace]; !exist {
+			dumpLog(fmt.Sprintf("kubectl get po,pvc,svc,cm,cronjobs,jobs,statefulsets,tidbclusters -owide -n %s", testCluster.Namespace), resourceWriter)
+			dumpLog(fmt.Sprintf("kubectl get po,pvc,svc,cm,cronjobs,jobs,statefulsets,tidbclusters -n %s -oyaml", testCluster.Namespace), resourceWriter)
+			dumpedNamespace[testCluster.Namespace] = true
+		}
+	}
+
+	// dump operator components's log
+	operatorPods, err := oa.kubeCli.CoreV1().Pods(operatorInfo.Namespace).List(metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, pod := range operatorPods.Items {
+		err := dumpPod(logPath, &pod)
+		if err != nil {
+			return err
+		}
+	}
+
+	// dump all test clusters's logs
+	dumpedNamespace = map[string]bool{}
+	for _, testCluster := range testClusters {
+		if _, exist := dumpedNamespace[testCluster.Namespace]; !exist {
+			clusterPodList, err := oa.kubeCli.CoreV1().Pods(testCluster.Namespace).List(metav1.ListOptions{})
+			if err != nil {
+				return err
+			}
+			for _, pod := range clusterPodList.Items {
+				err := dumpPod(logPath, &pod)
+				if err != nil {
+					return err
+				}
+			}
+			dumpedNamespace[testCluster.Namespace] = true
+		}
+	}
+
+	return nil
+}
+
+func dumpPod(logPath string, pod *corev1.Pod) error {
+	logFile, err := os.Create(filepath.Join(logPath, fmt.Sprintf("%s-%s.log", pod.Name, pod.Namespace)))
+	if err != nil {
+		return err
+	}
+	defer logFile.Close()
+	plogFile, err := os.Create(filepath.Join(logPath, fmt.Sprintf("%s-%s-p.log", pod.Name, pod.Namespace)))
+	if err != nil {
+		return err
+	}
+	defer plogFile.Close()
+
+	logWriter := bufio.NewWriter(logFile)
+	plogWriter := bufio.NewWriter(plogFile)
+	defer logWriter.Flush()
+	defer plogWriter.Flush()
+
+	for _, c := range pod.Spec.Containers {
+		dumpLog(fmt.Sprintf("kubectl logs -n %s %s -c %s", pod.Namespace, pod.GetName(), c.Name), logWriter)
+		dumpLog(fmt.Sprintf("kubectl logs -n %s %s -c %s -p", pod.Namespace, pod.GetName(), c.Name), plogWriter)
+	}
+
+	return nil
+}
+
+func dumpLog(cmdStr string, writer *bufio.Writer) {
+	writer.WriteString(fmt.Sprintf("$ %s\n", cmdStr))
+	cmd := exec.Command("/bin/sh", "-c", "/usr/local/bin/"+cmdStr)
+	cmd.Stderr = writer
+	cmd.Stdout = writer
+	err := cmd.Run()
+	if err != nil {
+		writer.WriteString(err.Error())
+	}
+}

tests/manifests/e2e.yaml

@@ -30,4 +30,12 @@ spec:
     image: ""
     imagePullPolicy: Always
     command: ["sh", "-c", "/usr/local/bin/e2e"]
+    volumeMounts:
+    - mountPath: /logDir
+      name: logdir
+  volumes:
+  - name: logdir
+    hostPath:
+      path: /var/log
+      type: Directory
   restartPolicy: Never