Check cluster image versions match CLI version

cli/cmd/lib_manager.go

@@ -27,6 +27,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/cortexlabs/cortex/pkg/consts"
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/lib/exit"
 	"github.com/cortexlabs/cortex/pkg/lib/files"
@@ -111,6 +112,11 @@ func pullManager(managerImage string) error {
 }
 
 func runManager(containerConfig *container.Config) (string, *int, error) {
+	containerConfig.Env = append(containerConfig.Env, "CORTEX_CLI_VERSION="+consts.CortexVersion)
+
+	// Add a slight delay before running the command to ensure logs don't start until after the container is attached
+	containerConfig.Cmd[0] = "sleep 0.1 && /root/check_cortex_version.sh && " + containerConfig.Cmd[0]
+
 	docker, err := getDockerClient()
 	if err != nil {
 		return "", nil, err
@@ -161,7 +167,6 @@ func runManager(containerConfig *container.Config) (string, *int, error) {
 		return "", nil, wrapDockerError(err)
 	}
 
-	// There is a slight delay between the container starting at attaching to it, hence the sleep in Cmd
 	logsOutput, err := docker.ContainerAttach(context.Background(), containerInfo.ID, dockertypes.ContainerAttachOptions{
 		Stream: true,
 		Stdout: true,
@@ -216,7 +221,7 @@ func runManagerUpdateCommand(entrypoint string, clusterConfig *clusterconfig.Con
 	containerConfig := &container.Config{
 		Image:        clusterConfig.ImageManager,
 		Entrypoint:   []string{"/bin/bash", "-c"},
-		Cmd:          []string{fmt.Sprintf("sleep 0.1 && eval $(python /root/cluster_config_env.py %s) && %s", mountedConfigPath, entrypoint)},
+		Cmd:          []string{fmt.Sprintf("eval $(python /root/cluster_config_env.py %s) && %s", mountedConfigPath, entrypoint)},
 		Tty:          true,
 		AttachStdout: true,
 		AttachStderr: true,
@@ -246,7 +251,7 @@ func runManagerAccessCommand(entrypoint string, accessConfig clusterconfig.Acces
 	containerConfig := &container.Config{
 		Image:        accessConfig.ImageManager,
 		Entrypoint:   []string{"/bin/bash", "-c"},
-		Cmd:          []string{"sleep 0.1 && " + entrypoint},
+		Cmd:          []string{entrypoint},
 		Tty:          true,
 		AttachStdout: true,
 		AttachStderr: true,

manager/check_cortex_version.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright 2020 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+CORTEX_VERSION=master
+
+if [ "$CORTEX_VERSION" != "$CORTEX_CLI_VERSION" ]; then
+  echo "error: your CLI version ($CORTEX_CLI_VERSION) doesn't match your Cortex manager image version ($CORTEX_VERSION); please update your CLI by following the instructions at https://www.cortex.dev/install, or update your Cortex manager image by modifying the value for \`image_manager\` in your cluster configuration file (e.g. cluster.yaml) and running \`cortex cluster update --config cluster.yaml\` (update other image paths in cluster.yaml as well if necessary)"
+  exit 1
+fi

pkg/lib/configreader/validators.go

@@ -37,7 +37,6 @@ func init() {
 
 func GetFilePathValidator(baseDir string) func(string) (string, error) {
 	return func(val string) (string, error) {
-		val = files.RelToAbsPath(val, baseDir)
 		if err := files.CheckFile(val); err != nil {
 			return "", err
 		}

pkg/lib/files/errors.go

@@ -175,7 +175,7 @@ func ErrorDirDoesNotExist(path string) error {
 func ErrorNotAFile(path string) error {
 	return errors.WithStack(Error{
 		Kind:    ErrNotAFile,
-		message: fmt.Sprintf("%s: not a file path", path),
+		message: fmt.Sprintf("%s: no such file", path),
 	})
 }
 

pkg/lib/files/files.go

@@ -138,7 +138,8 @@ func WriteFile(data []byte, path string) error {
 	return nil
 }
 
-// Returns original path if there was an error
+// e.g. ~/path -> /home/ubuntu/path
+// returns original path if there was an error
 func EscapeTilde(path string) (string, error) {
 	if !(path == "~" || strings.HasPrefix(path, "~/")) {
 		return path, nil
@@ -149,6 +150,9 @@ func EscapeTilde(path string) (string, error) {
 		if err != nil {
 			return path, err
 		}
+		if homeDir == "" || homeDir == "/" {
+			return path, nil
+		}
 		_homeDir = homeDir
 	}
 
@@ -160,6 +164,29 @@ func EscapeTilde(path string) (string, error) {
 	return filepath.Join(_homeDir, path[2:]), nil
 }
 
+// e.g. /home/ubuntu/path -> ~/path
+func ReplacePathWithTilde(absPath string) string {
+	if !strings.HasPrefix(absPath, "/") {
+		return absPath
+	}
+
+	if _homeDir == "" {
+		homeDir, err := homedir.Dir()
+		if err != nil || homeDir == "" || homeDir == "/" {
+			return absPath
+		}
+		_homeDir = homeDir
+	}
+
+	trimmedHomeDir := strings.TrimSuffix(s.EnsurePrefix(_homeDir, "/"), "/")
+
+	if strings.Index(absPath, trimmedHomeDir) == 0 {
+		return "~" + absPath[len(trimmedHomeDir):]
+	}
+
+	return absPath
+}
+
 func TrimDirPrefix(fullPath string, dirPath string) string {
 	if !strings.HasSuffix(dirPath, "/") {
 		dirPath = dirPath + "/"
@@ -175,12 +202,23 @@ func RelToAbsPath(relativePath string, baseDir string) string {
 }
 
 func UserRelToAbsPath(relativePath string) string {
-	cwd, _ := os.Getwd()
+	cwd, err := os.Getwd()
+	if err != nil {
+		return relativePath
+	}
 	return RelToAbsPath(relativePath, cwd)
 }
 
 func PathRelativeToCWD(absPath string) string {
-	cwd, _ := os.Getwd()
+	if !strings.HasPrefix(absPath, "/") {
+		return absPath
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return absPath
+	}
+
 	cwd = s.EnsureSuffix(cwd, "/")
 	return strings.TrimPrefix(absPath, cwd)
 }

pkg/operator/endpoints/errors.go

@@ -103,7 +103,7 @@ func (e Error) Error() string {
 func ErrorAPIVersionMismatch(operatorVersion string, clientVersion string) error {
 	return errors.WithStack(Error{
 		Kind:    ErrAPIVersionMismatch,
-		message: fmt.Sprintf("API version mismatch (Cluster: %s; Client: %s)", operatorVersion, clientVersion),
+		message: fmt.Sprintf("your CLI version (%s) doesn't match your Cortex operator version (%s); please update your cluster by following the instructions at https://www.cortex.dev/cluster-management/update, or update your CLI by following the instructions at https://www.cortex.dev/install", clientVersion, operatorVersion),
 	})
 }
 

pkg/types/clusterconfig/clusterconfig.go

@@ -227,121 +227,141 @@ var UserValidation = &cr.StructValidation{
 		{
 			StructField: "ImagePythonServe",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/python-serve:" + consts.CortexVersion,
+				Default:   "cortexlabs/python-serve:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImagePythonServeGPU",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/python-serve-gpu:" + consts.CortexVersion,
+				Default:   "cortexlabs/python-serve-gpu:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageTFServe",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/tf-serve:" + consts.CortexVersion,
+				Default:   "cortexlabs/tf-serve:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageTFServeGPU",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/tf-serve-gpu:" + consts.CortexVersion,
+				Default:   "cortexlabs/tf-serve-gpu:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageTFAPI",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/tf-api:" + consts.CortexVersion,
+				Default:   "cortexlabs/tf-api:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageONNXServe",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/onnx-serve:" + consts.CortexVersion,
+				Default:   "cortexlabs/onnx-serve:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageONNXServeGPU",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/onnx-serve-gpu:" + consts.CortexVersion,
+				Default:   "cortexlabs/onnx-serve-gpu:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageOperator",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/operator:" + consts.CortexVersion,
+				Default:   "cortexlabs/operator:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageManager",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/manager:" + consts.CortexVersion,
+				Default:   "cortexlabs/manager:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageDownloader",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/downloader:" + consts.CortexVersion,
+				Default:   "cortexlabs/downloader:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageRequestMonitor",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/request-monitor:" + consts.CortexVersion,
+				Default:   "cortexlabs/request-monitor:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageClusterAutoscaler",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/cluster-autoscaler:" + consts.CortexVersion,
+				Default:   "cortexlabs/cluster-autoscaler:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageMetricsServer",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/metrics-server:" + consts.CortexVersion,
+				Default:   "cortexlabs/metrics-server:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageNvidia",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/nvidia:" + consts.CortexVersion,
+				Default:   "cortexlabs/nvidia:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageFluentd",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/fluentd:" + consts.CortexVersion,
+				Default:   "cortexlabs/fluentd:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageStatsd",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/statsd:" + consts.CortexVersion,
+				Default:   "cortexlabs/statsd:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageIstioProxy",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/istio-proxy:" + consts.CortexVersion,
+				Default:   "cortexlabs/istio-proxy:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageIstioPilot",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/istio-pilot:" + consts.CortexVersion,
+				Default:   "cortexlabs/istio-pilot:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageIstioCitadel",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/istio-citadel:" + consts.CortexVersion,
+				Default:   "cortexlabs/istio-citadel:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageIstioGalley",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/istio-galley:" + consts.CortexVersion,
+				Default:   "cortexlabs/istio-galley:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 		// Extra keys that exist in the cluster config file
@@ -364,6 +384,29 @@ var UserValidation = &cr.StructValidation{
 	},
 }
 
+func validateImageVersion(image string) (string, error) {
+	if !strings.HasPrefix(image, "cortexlabs/") && !strings.HasPrefix(image, "cortexlabsdev/") {
+		return image, nil
+	}
+
+	var tag string
+
+	if colonIndex := strings.LastIndex(image, ":"); colonIndex != -1 {
+		tag = image[colonIndex+1:]
+	}
+
+	// in docker, missing tag implies "latest"
+	if tag == "" {
+		tag = "latest"
+	}
+
+	if tag != consts.CortexVersion {
+		return "", ErrorImageVersionMismatch(image, tag)
+	}
+
+	return image, nil
+}
+
 var Validation = &cr.StructValidation{
 	StructFieldValidations: append(UserValidation.StructFieldValidations,
 		&cr.StructFieldValidation{
@@ -395,7 +438,8 @@ var AccessValidation = &cr.StructValidation{
 		{
 			StructField: "ImageManager",
 			StringValidation: &cr.StringValidation{
-				Default: "cortexlabs/manager:" + consts.CortexVersion,
+				Default:   "cortexlabs/manager:" + consts.CortexVersion,
+				Validator: validateImageVersion,
 			},
 		},
 	},