Merge pull request #20 from replicatedhq/ch20665

Honor timeout in the run collector

.gitignore

@@ -17,6 +17,9 @@ bin
 
 !vendor/**/zz_generated.*
 
+# Vendored files downloaded with go mod
+vendor
+
 # editor and IDE paraphernalia
 .idea
 *.swp

go.mod

@@ -9,21 +9,17 @@ require (
 	github.com/gin-gonic/gin v1.4.0
 	github.com/gizak/termui/v3 v3.1.0
 	github.com/golang/snappy v0.0.1 // indirect
-	github.com/gorilla/websocket v1.4.0
-	github.com/manifoldco/promptui v0.3.2
+	github.com/manifoldco/promptui v0.3.2 // indirect
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/nwaples/rardecode v1.0.0 // indirect
 	github.com/onsi/gomega v1.5.0
 	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
-	github.com/pkg/errors v0.8.1
-	github.com/pmezard/go-difflib v1.0.0
-	github.com/sergi/go-diff v1.0.0
+	github.com/sergi/go-diff v1.0.0 // indirect
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/viper v1.4.0
 	github.com/stretchr/testify v1.3.0
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
-	github.com/xo/dburl v0.0.0-20190203050942-98997a05b24f
-	go.uber.org/zap v1.10.0
+	github.com/xo/dburl v0.0.0-20190203050942-98997a05b24f // indirect
 	golang.org/x/net v0.0.0-20190522155817-f3200d17e092
 	gopkg.in/yaml.v2 v2.2.2
 	k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b

go.sum

@@ -163,6 +163,7 @@ github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
 github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
@@ -444,6 +445,7 @@ gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6d
 gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

pkg/collect/logs.go

@@ -39,7 +39,7 @@ func Logs(logsCollector *troubleshootv1beta1.Logs, redact bool) error {
 		PodLogs: make(map[string][]byte),
 	}
 	for _, pod := range pods {
-		podLogs, err := getPodLogs(client, pod, logsCollector.Limits)
+		podLogs, err := getPodLogs(client, pod, logsCollector.Limits, false)
 		if err != nil {
 			return err
 		}
@@ -81,8 +81,10 @@ func listPodsInSelectors(client *kubernetes.Clientset, namespace string, selecto
 	return pods.Items, nil
 }
 
-func getPodLogs(client *kubernetes.Clientset, pod corev1.Pod, limits *troubleshootv1beta1.LogLimits) (map[string][]byte, error) {
-	podLogOpts := corev1.PodLogOptions{}
+func getPodLogs(client *kubernetes.Clientset, pod corev1.Pod, limits *troubleshootv1beta1.LogLimits, follow bool) (map[string][]byte, error) {
+	podLogOpts := corev1.PodLogOptions{
+		Follow: follow,
+	}
 
 	defaultMaxLines := int64(10000)
 	if limits == nil || limits.MaxLines == 0 {

pkg/collect/run.go

@@ -2,6 +2,7 @@ package collect
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"time"
 
@@ -32,46 +33,69 @@ func Run(runCollector *troubleshootv1beta1.Run, redact bool) error {
 		return err
 	}
 
-	runOutput := &RunOutput{
-		PodLogs: make(map[string][]byte),
+	defer func() {
+		if err := client.CoreV1().Pods(pod.Namespace).Delete(pod.Name, &metav1.DeleteOptions{}); err != nil {
+			fmt.Printf("Failed to delete pod %s: %v\n", pod.Name, err)
+		}
+	}()
+
+	if runCollector.Timeout == "" {
+		return runWithoutTimeout(pod, runCollector, redact)
 	}
 
-	now := time.Now()
-	then := now.Add(time.Duration(20 * time.Second))
+	timeout, err := time.ParseDuration(runCollector.Timeout)
+	if err != nil {
+		return err
+	}
 
-	if runCollector.Timeout != "" {
-		parsedDuration, err := time.ParseDuration(runCollector.Timeout)
-		if err != nil {
-			fmt.Printf("unable to parse time duration %s\n", runCollector.Timeout)
-		} else {
-			then = now.Add(parsedDuration)
-		}
+	runChan := make(chan error, 1)
+	go func() {
+		runChan <- runWithoutTimeout(pod, runCollector, redact)
+	}()
+
+	select {
+	case <-time.After(timeout):
+		return errors.New("timeout")
+	case err := <-runChan:
+		return err
+	}
+}
+
+func runWithoutTimeout(pod *corev1.Pod, runCollector *troubleshootv1beta1.Run, redact bool) error {
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return err
+	}
+
+	client, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return err
 	}
 
 	for {
-		if time.Now().After(then) {
+		status, err := client.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		if status.Status.Phase == "Running" {
 			break
 		}
+		time.Sleep(time.Second * 1)
+	}
 
-		time.Sleep(time.Second)
+	runOutput := &RunOutput{
+		PodLogs: make(map[string][]byte),
 	}
 
 	limits := troubleshootv1beta1.LogLimits{
 		MaxLines: 10000,
 	}
-	podLogs, err := getPodLogs(client, *pod, &limits)
-	if err != nil {
-		return err
-	}
+	podLogs, err := getPodLogs(client, *pod, &limits, true)
 
 	for k, v := range podLogs {
 		runOutput.PodLogs[k] = v
 	}
 
-	if err := client.CoreV1().Pods(pod.Namespace).Delete(pod.Name, &metav1.DeleteOptions{}); err != nil {
-		return err
-	}
-
 	if redact {
 		runOutput, err = runOutput.Redact()
 		if err != nil {