Retry logic for stdin and stdout from pod

Currently receptor has one attempt at opening stdin and stdout streams
from the pod.

Getting an error kills any chance of getting the pod to run. This builds
some retry logic when opening these streams for better stability.

pkg/workceptor/kubernetes.go

@@ -295,7 +295,14 @@ func (kw *kubeUnit) runWorkUsingLogger() {
 		Container: "worker",
 		Follow:    true,
 	})
-	logStream, err := logreq.Stream(kw.ctx)
+	var logStream io.ReadCloser
+	for retries := 0; retries < 3; retries++ {
+		logStream, err = logreq.Stream(kw.ctx)
+		if err != nil {
+			logger.Warning("Issue opening stdout from pod %s, unit %s. Retrying.", kw.pod.Name, kw.unitID)
+			time.Sleep(time.Second * 5)
+		}
+	}
 	if err != nil {
 		errMsg := fmt.Sprintf("Error opening pod stream: %s", err)
 		kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)
@@ -397,10 +404,16 @@ func (kw *kubeUnit) runWorkUsingLogger() {
 		streamWait.Done()
 	} else {
 		go func() {
-			errStdin = exec.Stream(remotecommand.StreamOptions{
-				Stdin: stdin,
-				Tty:   false,
-			})
+			for retries := 0; retries < 3; retries++ {
+				errStdin = exec.Stream(remotecommand.StreamOptions{
+					Stdin: stdin,
+					Tty:   false,
+				})
+				if errStdin != nil {
+					logger.Warning("Could not stream stdin to pod %s, unit %s. Retrying.", kw.pod.Name, kw.unitID)
+					time.Sleep(time.Second * 5)
+				}
+			}
 			if errStdin != nil {
 				logStream.Close()
 			}