Fail gracefully if remote runner connection info is unreliable (#607)

Unless any `--insecure`/`--insecure=false`/`--insecure=true` is
explicitly set.

internal/cmd/root.go

@@ -22,6 +22,7 @@ var (
 	fProject               string
 	fProjectIgnorePatterns []string
 	fRespectGitignore      bool
+	fSkipRunnerFallback    bool
 	fInsecure              bool
 	fLogEnabled            bool
 	fLogFilePath           string
@@ -54,6 +55,9 @@ func Root() *cobra.Command {
 				fProject = envProject
 			}
 
+			// if insecure is explicitly set irrespective of its value, skip runner fallback
+			fSkipRunnerFallback = !cmd.Flags().Changed("insecure")
+
 			fFileMode = cmd.Flags().Changed("chdir") || cmd.Flags().Changed("filename")
 
 			if fFileMode && !cmd.Flags().Changed("allow-unnamed") {

internal/cmd/run.go

@@ -181,26 +181,9 @@ func runCmd() *cobra.Command {
 				client.WrapWithCancelReader(),
 			}
 
-			var runner client.Runner
-
-			if serverAddr == "" {
-				localRunner, err := client.NewLocalRunner(runnerOpts...)
-				if err != nil {
-					return err
-				}
-
-				runner = localRunner
-			} else {
-				remoteRunner, err := client.NewRemoteRunner(
-					cmd.Context(),
-					serverAddr,
-					runnerOpts...,
-				)
-				if err != nil {
-					return err
-				}
-
-				runner = remoteRunner
+			runner, err := client.New(cmd.Context(), serverAddr, fSkipRunnerFallback, runnerOpts)
+			if err != nil {
+				return err
 			}
 
 			for _, task := range runTasks {

internal/cmd/tui.go

@@ -99,26 +99,9 @@ func tuiCmd() *cobra.Command {
 				client.WithProject(proj),
 			)
 
-			if serverAddr != "" {
-				remoteRunner, err := client.NewRemoteRunner(
-					cmd.Context(),
-					serverAddr,
-					runnerOpts...,
-				)
-				if err != nil {
-					return errors.Wrap(err, "failed to create remote runner")
-				}
-
-				runnerClient = remoteRunner
-			} else {
-				localRunner, err := client.NewLocalRunner(
-					runnerOpts...,
-				)
-				if err != nil {
-					return errors.Wrap(err, "failed to create local runner")
-				}
-
-				runnerClient = localRunner
+			runnerClient, err = client.New(cmd.Context(), serverAddr, fSkipRunnerFallback, runnerOpts)
+			if err != nil {
+				return errors.Wrap(err, "failed to create local runner")
 			}
 
 			model := tuiModel{

internal/runner/client/client.go

@@ -68,6 +68,36 @@ type Runner interface {
 	setSettings(settings *RunnerSettings)
 }
 
+func New(context context.Context, serverAddr string, fallbackRunner bool, runnerOpts []RunnerOption) (Runner, error) {
+	if serverAddr != "" {
+		// check if serverAddr points at a healthy server
+		healthy, err := isServerHealthy(context, serverAddr, runnerOpts)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to check health")
+		}
+
+		if !fallbackRunner || healthy {
+			remoteRunner, err := NewRemoteRunner(
+				context,
+				serverAddr,
+				runnerOpts...,
+			)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to create remote runner")
+			}
+
+			return remoteRunner, nil
+		}
+	}
+
+	localRunner, err := NewLocalRunner(runnerOpts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create local runner")
+	}
+
+	return localRunner, nil
+}
+
 func withSettings(applySettings func(settings *RunnerSettings)) RunnerOption {
 	return withSettingsErr(func(settings *RunnerSettings) error {
 		applySettings(settings)

internal/runner/client/client_remote.go

@@ -21,6 +21,7 @@ import (
 	"github.com/stateful/runme/v3/internal/runner"
 	runmetls "github.com/stateful/runme/v3/internal/tls"
 	runnerv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v1"
+	healthv1 "google.golang.org/grpc/health/grpc_health_v1"
 )
 
 type RemoteRunner struct {
@@ -43,6 +44,28 @@ func (r *RemoteRunner) setSettings(rs *RunnerSettings) {
 	r.RunnerSettings = rs
 }
 
+func isServerHealthy(context context.Context, addr string, runnerOpts []RunnerOption) (bool, error) {
+	r := &RemoteRunner{
+		RunnerSettings: &RunnerSettings{},
+	}
+
+	if err := ApplyOptions(r, runnerOpts...); err != nil {
+		return false, err
+	}
+
+	conn, err := getGrpcConnection(context, addr, r)
+	if err != nil {
+		return false, nil
+	}
+
+	resp, err := healthv1.NewHealthClient(conn).Check(context, &healthv1.HealthCheckRequest{})
+	if err != nil || resp.Status != healthv1.HealthCheckResponse_SERVING {
+		return false, nil
+	}
+
+	return true, nil
+}
+
 func NewRemoteRunner(ctx context.Context, addr string, opts ...RunnerOption) (*RemoteRunner, error) {
 	r := &RemoteRunner{
 		RunnerSettings: &RunnerSettings{},
@@ -52,6 +75,21 @@ func NewRemoteRunner(ctx context.Context, addr string, opts ...RunnerOption) (*R
 		return nil, err
 	}
 
+	conn, err := getGrpcConnection(ctx, addr, r)
+	if err != nil {
+		return nil, err
+	}
+
+	r.client = runnerv1.NewRunnerServiceClient(conn)
+
+	if err := r.setupSession(ctx); err != nil {
+		return nil, err
+	}
+
+	return r, nil
+}
+
+func getGrpcConnection(ctx context.Context, addr string, r *RemoteRunner) (*grpc.ClientConn, error) {
 	var creds credentials.TransportCredentials
 
 	if r.insecure {
@@ -70,13 +108,7 @@ func NewRemoteRunner(ctx context.Context, addr string, opts ...RunnerOption) (*R
 		return nil, errors.Wrap(err, "failed to connect to gRPC server")
 	}
 
-	r.client = runnerv1.NewRunnerServiceClient(conn)
-
-	if err := r.setupSession(ctx); err != nil {
-		return nil, err
-	}
-
-	return r, nil
+	return conn, nil
 }
 
 func (r *RemoteRunner) setupSession(ctx context.Context) error {

testdata/script/basic.txtar

@@ -58,6 +58,18 @@ exec runme run hello-python
 stdout 'Hello from Python'
 ! stderr .
 
+env PATH=/opt/homebrew/bin:$PATH
+env RUNME_SERVER_ADDR="123.0.0.9:12345"
+exec runme run hello-python
+stdout 'Hello from Python'
+! stderr .
+
+env PATH=/opt/homebrew/bin:$PATH
+env RUNME_TLS_DIR=/tmp/invalid/tls
+! exec runme run --insecure=false hello-python
+! stdout .
+stderr 'could not execute command: failed to create remote runner: open /tmp/invalid/tls/cert.pem: no such file or directory'
+
 -- README.md --
 ---
 runme: