From 433ac0ba08d3d1f08e51de74960a854da3652b48 Mon Sep 17 00:00:00 2001
From: vishal <vishalbollu@users.noreply.github.com>
Date: Thu, 17 Jun 2021 21:57:45 -0400
Subject: [PATCH 01/28] Use cortexlabs's version kubexit

---
 images/kubexit/Dockerfile | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/images/kubexit/Dockerfile b/images/kubexit/Dockerfile
index 387a3f0265..ba77644143 100644
--- a/images/kubexit/Dockerfile
+++ b/images/kubexit/Dockerfile
@@ -1 +1,14 @@
-FROM miguelvr/kubexit:0.3.2-patch
+FROM golang:1.14.1-alpine3.11 AS builder
+RUN apk update && apk upgrade && \
+    apk add bash git openssh
+
+RUN mkdir /tmp/kubexit
+RUN git clone -b v0.1.0-cortex --depth 1  https://github.com/cortexlabs/kubexit.git /tmp/kubexit
+
+WORKDIR /tmp/kubexit
+RUN CGO_ENABLED=0 GOOS=linux go build -o kubexit ./cmd/kubexit
+
+FROM alpine:3.11
+RUN apk --no-cache add ca-certificates tzdata
+COPY --from=builder /tmp/kubexit/kubexit /bin/
+ENTRYPOINT ["kubexit"]
\ No newline at end of file

From a30641bad1ee0a5908274db42a00df528654ab3d Mon Sep 17 00:00:00 2001
From: vishal <vishalbollu@users.noreply.github.com>
Date: Thu, 17 Jun 2021 22:14:59 -0400
Subject: [PATCH 02/28] Add newline

---
 images/kubexit/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/images/kubexit/Dockerfile b/images/kubexit/Dockerfile
index ba77644143..57feb01e25 100644
--- a/images/kubexit/Dockerfile
+++ b/images/kubexit/Dockerfile
@@ -11,4 +11,4 @@ RUN CGO_ENABLED=0 GOOS=linux go build -o kubexit ./cmd/kubexit
 FROM alpine:3.11
 RUN apk --no-cache add ca-certificates tzdata
 COPY --from=builder /tmp/kubexit/kubexit /bin/
-ENTRYPOINT ["kubexit"]
\ No newline at end of file
+ENTRYPOINT ["kubexit"]

From 3c754a975768554ed31161523cbea773220ee331 Mon Sep 17 00:00:00 2001
From: vishal <vishalbollu@users.noreply.github.com>
Date: Fri, 18 Jun 2021 09:19:34 -0400
Subject: [PATCH 03/28] Add comment describing where the dockerfile is from

---
 images/kubexit/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/images/kubexit/Dockerfile b/images/kubexit/Dockerfile
index 57feb01e25..9851d0d2fc 100644
--- a/images/kubexit/Dockerfile
+++ b/images/kubexit/Dockerfile
@@ -1,3 +1,4 @@
+# This file has been adapted from https://github.com/cortexlabs/kubexit
 FROM golang:1.14.1-alpine3.11 AS builder
 RUN apk update && apk upgrade && \
     apk add bash git openssh

From 46609de8402435a92331e520de40a1d25dd131f0 Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Fri, 18 Jun 2021 18:07:07 +0300
Subject: [PATCH 04/28] WIP on the addition of arm instances

---
 build/images.sh                           | 13 +++++++++++++
 images/dequeuer/Dockerfile                |  4 +++-
 images/enqueuer/Dockerfile                |  4 +++-
 images/proxy/Dockerfile                   |  4 +++-
 pkg/lib/aws/ec2.go                        | 17 -----------------
 pkg/types/clusterconfig/cluster_config.go |  8 --------
 pkg/types/clusterconfig/errors.go         |  8 --------
 7 files changed, 22 insertions(+), 36 deletions(-)

diff --git a/build/images.sh b/build/images.sh
index ca07560b3b..340ac5cf7f 100644
--- a/build/images.sh
+++ b/build/images.sh
@@ -50,6 +50,19 @@ non_dev_images=(
   "kubexit"
 )
 
+# for linux/amd64 and linux/arm64
+multi_arch_images=(
+  "proxy"
+  "async-gateway"
+  "enqueuer"
+  "dequeuer"
+  "fluent-bit"
+  "prometheus-statsd-exporter" # remove it once a new cluster is spun up
+  "prometheus-node-exporter"
+  "kube-rbac-proxy"
+  "kubexit"
+)
+
 all_images=(
   "${dev_images[@]}"
   "${non_dev_images[@]}"
diff --git a/images/dequeuer/Dockerfile b/images/dequeuer/Dockerfile
index 5c3e7c31f7..24671721cb 100644
--- a/images/dequeuer/Dockerfile
+++ b/images/dequeuer/Dockerfile
@@ -1,3 +1,5 @@
+ARG TARGETARCH, TARGETOS
+
 # Build the manager binary
 FROM golang:1.15 as builder
 
@@ -17,7 +19,7 @@ COPY pkg/workloads pkg/workloads
 COPY cmd/dequeuer cmd/dequeuer
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o dequeuer ./cmd/dequeuer
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -o dequeuer ./cmd/dequeuer
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/images/enqueuer/Dockerfile b/images/enqueuer/Dockerfile
index 392d3d90fa..04de1f6952 100644
--- a/images/enqueuer/Dockerfile
+++ b/images/enqueuer/Dockerfile
@@ -1,3 +1,5 @@
+ARG TARGETARCH, TARGETOS
+
 # Build the manager binary
 FROM golang:1.15 as builder
 
@@ -16,7 +18,7 @@ COPY pkg/workloads pkg/workloads
 COPY cmd/enqueuer cmd/enqueuer
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o enqueuer ./cmd/enqueuer
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -o enqueuer ./cmd/enqueuer
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/images/proxy/Dockerfile b/images/proxy/Dockerfile
index 2573e4e64d..eb4e1a4534 100644
--- a/images/proxy/Dockerfile
+++ b/images/proxy/Dockerfile
@@ -1,3 +1,5 @@
+ARG TARGETARCH, TARGETOS
+
 # Build the manager binary
 FROM golang:1.15 as builder
 
@@ -15,7 +17,7 @@ COPY cmd/proxy cmd/proxy
 WORKDIR /workspace/cmd/proxy
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o /workspace/bin/proxy main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH}  GO111MODULE=on go build -a -o /workspace/bin/proxy main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/pkg/lib/aws/ec2.go b/pkg/lib/aws/ec2.go
index 8ae55b0cb8..7270818204 100644
--- a/pkg/lib/aws/ec2.go
+++ b/pkg/lib/aws/ec2.go
@@ -103,23 +103,6 @@ func ParseInstanceType(instanceType string) (ParsedInstanceType, error) {
 	}, nil
 }
 
-func IsARMInstance(instanceType string) (bool, error) {
-	parsedType, err := ParseInstanceType(instanceType)
-	if err != nil {
-		return false, err
-	}
-
-	if parsedType.Family == "a" {
-		return true, nil
-	}
-
-	if parsedType.Capabilities.Has("g") {
-		return true, nil
-	}
-
-	return false, nil
-}
-
 func IsAMDGPUInstance(instanceType string) (bool, error) {
 	parsedType, err := ParseInstanceType(instanceType)
 	if err != nil {
diff --git a/pkg/types/clusterconfig/cluster_config.go b/pkg/types/clusterconfig/cluster_config.go
index 6049aeeaf8..01cd3f5caa 100644
--- a/pkg/types/clusterconfig/cluster_config.go
+++ b/pkg/types/clusterconfig/cluster_config.go
@@ -1443,14 +1443,6 @@ func validateInstanceType(instanceType string) (string, error) {
 		return "", ErrorInstanceTypeNotSupportedByCortex(instanceType)
 	}
 
-	isARM, err := aws.IsARMInstance(instanceType)
-	if err != nil {
-		return "", err
-	}
-	if isARM {
-		return "", ErrorARMInstancesNotSupported(instanceType)
-	}
-
 	isAMDGPU, err := aws.IsAMDGPUInstance(instanceType)
 	if err != nil {
 		return "", err
diff --git a/pkg/types/clusterconfig/errors.go b/pkg/types/clusterconfig/errors.go
index c6c13113ba..f719e7eff4 100644
--- a/pkg/types/clusterconfig/errors.go
+++ b/pkg/types/clusterconfig/errors.go
@@ -46,7 +46,6 @@ const (
 	ErrSpotPriceGreaterThanTargetOnDemand     = "clusterconfig.spot_price_greater_than_target_on_demand"
 	ErrSpotPriceGreaterThanMaxPrice           = "clusterconfig.spot_price_greater_than_max_price"
 	ErrInstanceTypeNotSupportedByCortex       = "clusterconfig.instance_type_not_supported_by_cortex"
-	ErrARMInstancesNotSupported               = "clusterconfig.arm_instances_not_supported"
 	ErrAMDGPUInstancesNotSupported            = "clusterconfig.amd_gpu_instances_not_supported"
 	ErrAtLeastOneInstanceDistribution         = "clusterconfig.at_least_one_instance_distribution"
 	ErrNoCompatibleSpotInstanceFound          = "clusterconfig.no_compatible_spot_instance_found"
@@ -203,13 +202,6 @@ func ErrorInstanceTypeNotSupportedByCortex(instanceType string) error {
 	})
 }
 
-func ErrorARMInstancesNotSupported(instanceType string) error {
-	return errors.WithStack(&errors.Error{
-		Kind:    ErrARMInstancesNotSupported,
-		Message: fmt.Sprintf("ARM-based instances (including %s) are not supported by cortex", instanceType),
-	})
-}
-
 func ErrorAMDGPUInstancesNotSupported(instanceType string) error {
 	return errors.WithStack(&errors.Error{
 		Kind:    ErrAMDGPUInstancesNotSupported,

From 1ddc52b4d8d0785961222fba1189bc66e57345cf Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Fri, 18 Jun 2021 18:43:40 +0300
Subject: [PATCH 05/28] Add target os/arch to async gateway

---
 images/async-gateway/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/images/async-gateway/Dockerfile b/images/async-gateway/Dockerfile
index 6eae9ad24f..94ec94c6f5 100644
--- a/images/async-gateway/Dockerfile
+++ b/images/async-gateway/Dockerfile
@@ -1,3 +1,5 @@
+ARG TARGETARCH, TARGETOS
+
 FROM golang:1.15 as builder
 
 COPY go.mod go.sum /workspace/
@@ -10,7 +12,7 @@ COPY pkg/async-gateway pkg/async-gateway
 COPY pkg/types pkg/types
 COPY cmd/async-gateway cmd/async-gateway
 
-RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -installsuffix cgo -o async-gateway ./cmd/async-gateway
+RUN GO111MODULE=on CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -installsuffix cgo -o async-gateway ./cmd/async-gateway
 
 FROM alpine:3.12
 

From 6069f8b5090edb1b254305a90d04be2df4a8a77e Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Fri, 18 Jun 2021 21:13:19 +0300
Subject: [PATCH 06/28] Add Amr64 AMI images

---
 build/generate_ami_mapping.go |  13 ++--
 manager/generate_eks.py       |  40 ++++++++++--
 manager/manifests/ami.json    | 115 ++++++++++++++++++++--------------
 3 files changed, 113 insertions(+), 55 deletions(-)

diff --git a/build/generate_ami_mapping.go b/build/generate_ami_mapping.go
index 14a8ab25d3..83e39df95f 100644
--- a/build/generate_ami_mapping.go
+++ b/build/generate_ami_mapping.go
@@ -235,11 +235,15 @@ func main() {
 		fmt.Print(region)
 		sess := session.New(&aws.Config{Region: aws.String(region)})
 		svc := ec2.New(sess)
-		cpuAMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf("amazon-eks-node-%s-v*", k8sVersion))
+		cpuAmd64AMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf("amazon-eks-node-%s-v*", k8sVersion))
 		if err != nil {
 			log.Fatal(err.Error())
 		}
-		acceleratedAMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf("amazon-eks-gpu-node-%s-v*", k8sVersion))
+		cpuArm64AMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf("amazon-eks-arm64-node-%s-v*", k8sVersion))
+		if err != nil {
+			log.Fatal(err.Error())
+		}
+		acceleratedAmd64AMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf("amazon-eks-gpu-node-%s-v*", k8sVersion))
 		if err != nil {
 			log.Fatal(err.Error())
 		}
@@ -248,8 +252,9 @@ func main() {
 			k8sVersionMap[k8sVersion][region] = map[string]string{}
 		}
 		k8sVersionMap[k8sVersion][region] = map[string]string{
-			"cpu":         cpuAMI,
-			"accelerated": acceleratedAMI,
+			"cpu_amd64":         cpuAmd64AMI,
+			"cpu_arm64":         cpuArm64AMI,
+			"accelerated_amd64": acceleratedAmd64AMI,
 		}
 		fmt.Println(" ✓")
 	}
diff --git a/manager/generate_eks.py b/manager/generate_eks.py
index c54c18a216..046208147d 100644
--- a/manager/generate_eks.py
+++ b/manager/generate_eks.py
@@ -13,13 +13,34 @@
 # limitations under the License.
 
 import json
-import sys
 import click
 
+from collections import namedtuple
+import re
 import yaml
 
 K8S_VERSION = "1.18"
 
+ParsedInstanceType = namedtuple(
+    "ParsedInstanceType", ["family", "generation", "capabilities", "size"]
+)
+
+
+def parse_instance_type(instance_type: str) -> ParsedInstanceType:
+    parts = instance_type.split(".")
+    if len(parts) != 2:
+        raise ValueError(f"unexpected invalid instance type: {instance_type}")
+
+    prefix = parts[0]
+    size = parts[1]
+
+    family = re.search("[a-z]*", prefix.lower()).group()
+    generation = re.sub("\D", "", prefix.lower())
+    capabilities = prefix[len(family) + len(generation) :]
+
+    return ParsedInstanceType(family, generation, capabilities, size)
+
+
 # kubelet config schema:
 # https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go
 def default_nodegroup(cluster_config):
@@ -132,7 +153,8 @@ def apply_gpu_settings(nodegroup):
 
 
 def is_gpu(instance_type):
-    return instance_type.startswith("g") or instance_type.startswith("p")
+    parsed_instance_type = parse_instance_type(instance_type)
+    return parsed_instance_type.family in ["g", "p"]
 
 
 def apply_inf_settings(nodegroup, config):
@@ -155,7 +177,8 @@ def apply_inf_settings(nodegroup, config):
 
 
 def is_inf(instance_type):
-    return instance_type.startswith("inf")
+    parsed_instance_type = parse_instance_type(instance_type)
+    return parsed_instance_type.family == "inf"
 
 
 def get_inf_resources(instance_type):
@@ -170,6 +193,11 @@ def get_inf_resources(instance_type):
     return num_chips, f"{128 * num_chips}Mi"
 
 
+def is_arm64(instance_type: str):
+    parsed_instance_type = parse_instance_type(instance_type)
+    return parsed_instance_type.family == "a" or "g" in parsed_instance_type.capabilities
+
+
 def get_all_worker_nodegroups(ami_map: dict, cluster_config: dict) -> list:
     """
     Gets all node groups in EKS-dict format.
@@ -220,8 +248,10 @@ def get_empty_eks_nodegroup(name: str) -> dict:
 
 def get_ami(ami_map: dict, instance_type: str) -> str:
     if is_gpu(instance_type) or is_inf(instance_type):
-        return ami_map["accelerated"]
-    return ami_map["cpu"]
+        return ami_map["accelerated_amd64"]
+    if is_arm64(instance_type):
+        return ami_map["cpu_arm64"]
+    return ami_map["cpu_amd64"]
 
 
 @click.command()
diff --git a/manager/manifests/ami.json b/manager/manifests/ami.json
index aa3e2ffdfd..9c10c9de5f 100644
--- a/manager/manifests/ami.json
+++ b/manager/manifests/ami.json
@@ -1,96 +1,119 @@
 {
 	"1.18": {
 		"af-south-1": {
-			"accelerated": "ami-09dbb6301ecd4c8ed",
-			"cpu": "ami-01fda598d7d31ca34"
+			"accelerated_amd64": "ami-09dbb6301ecd4c8ed",
+			"cpu_amd64": "ami-01fda598d7d31ca34",
+			"cpu_arm64": "ami-08556b4f3d2523237"
 		},
 		"ap-east-1": {
-			"accelerated": "ami-092643f8396c00e6d",
-			"cpu": "ami-05baab8ab39f975cc"
+			"accelerated_amd64": "ami-092643f8396c00e6d",
+			"cpu_amd64": "ami-05baab8ab39f975cc",
+			"cpu_arm64": "ami-0d0efa0b348fc62be"
 		},
 		"ap-northeast-1": {
-			"accelerated": "ami-07075c31664644d56",
-			"cpu": "ami-0d8366db514b71362"
+			"accelerated_amd64": "ami-07075c31664644d56",
+			"cpu_amd64": "ami-0d8366db514b71362",
+			"cpu_arm64": "ami-0b57af18f786a8723"
 		},
 		"ap-northeast-2": {
-			"accelerated": "ami-0b0ccf6e451bc71c0",
-			"cpu": "ami-0f2595b147de819e6"
+			"accelerated_amd64": "ami-0b0ccf6e451bc71c0",
+			"cpu_amd64": "ami-0f2595b147de819e6",
+			"cpu_arm64": "ami-0e9c6c02a5a2b3108"
 		},
 		"ap-northeast-3": {
-			"accelerated": "ami-0cc1736088413e3d3",
-			"cpu": "ami-07526d5726fefb0e3"
+			"accelerated_amd64": "ami-0cc1736088413e3d3",
+			"cpu_amd64": "ami-07526d5726fefb0e3",
+			"cpu_arm64": "ami-054c13b9aef254d3c"
 		},
 		"ap-south-1": {
-			"accelerated": "ami-0c068f5bfde3d2537",
-			"cpu": "ami-018ad103064da8b95"
+			"accelerated_amd64": "ami-0c068f5bfde3d2537",
+			"cpu_amd64": "ami-018ad103064da8b95",
+			"cpu_arm64": "ami-086a8ff4ed7eec145"
 		},
 		"ap-southeast-1": {
-			"accelerated": "ami-0fe1513e385d2398c",
-			"cpu": "ami-0df554c21406efafb"
+			"accelerated_amd64": "ami-0fe1513e385d2398c",
+			"cpu_amd64": "ami-0df554c21406efafb",
+			"cpu_arm64": "ami-0cfd1a3fd0befa360"
 		},
 		"ap-southeast-2": {
-			"accelerated": "ami-0104bf0b5b2fef4af",
-			"cpu": "ami-04560040fa5006d03"
+			"accelerated_amd64": "ami-0104bf0b5b2fef4af",
+			"cpu_amd64": "ami-04560040fa5006d03",
+			"cpu_arm64": "ami-0407c312d26bb79bd"
 		},
 		"ca-central-1": {
-			"accelerated": "ami-03da7af698418621b",
-			"cpu": "ami-047e794bde5d7210b"
+			"accelerated_amd64": "ami-03da7af698418621b",
+			"cpu_amd64": "ami-047e794bde5d7210b",
+			"cpu_arm64": "ami-0bd73de29f7b06de6"
 		},
 		"eu-central-1": {
-			"accelerated": "ami-0d9666606ff989a98",
-			"cpu": "ami-04d9a86ec8fd40b1f"
+			"accelerated_amd64": "ami-0d9666606ff989a98",
+			"cpu_amd64": "ami-04d9a86ec8fd40b1f",
+			"cpu_arm64": "ami-06849dbd130ba315a"
 		},
 		"eu-north-1": {
-			"accelerated": "ami-00dbfd663385ce54b",
-			"cpu": "ami-031ebcd37827da3ce"
+			"accelerated_amd64": "ami-00dbfd663385ce54b",
+			"cpu_amd64": "ami-031ebcd37827da3ce",
+			"cpu_arm64": "ami-07fed4d9d487f8ba8"
 		},
 		"eu-south-1": {
-			"accelerated": "ami-0495fadf45bb9435e",
-			"cpu": "ami-0de202af1558dda27"
+			"accelerated_amd64": "ami-0495fadf45bb9435e",
+			"cpu_amd64": "ami-0de202af1558dda27",
+			"cpu_arm64": "ami-0c4b9dff31b0f4a2f"
 		},
 		"eu-west-1": {
-			"accelerated": "ami-0288c9a9037e0d365",
-			"cpu": "ami-01ea01b6f3b80adfa"
+			"accelerated_amd64": "ami-0288c9a9037e0d365",
+			"cpu_amd64": "ami-01ea01b6f3b80adfa",
+			"cpu_arm64": "ami-07baa2ef15a15b140"
 		},
 		"eu-west-2": {
-			"accelerated": "ami-046ace414ecb0894b",
-			"cpu": "ami-064d5de87f6381ad1"
+			"accelerated_amd64": "ami-046ace414ecb0894b",
+			"cpu_amd64": "ami-064d5de87f6381ad1",
+			"cpu_arm64": "ami-0f43b63ae39a176b5"
 		},
 		"eu-west-3": {
-			"accelerated": "ami-0167e363b586e57c3",
-			"cpu": "ami-034f14ae8f5582c2b"
+			"accelerated_amd64": "ami-0167e363b586e57c3",
+			"cpu_amd64": "ami-034f14ae8f5582c2b",
+			"cpu_arm64": "ami-0f9f72a42176ecb78"
 		},
 		"me-south-1": {
-			"accelerated": "ami-0adb2677502f6a201",
-			"cpu": "ami-0ee3de9dbfa430ba0"
+			"accelerated_amd64": "ami-0adb2677502f6a201",
+			"cpu_amd64": "ami-0ee3de9dbfa430ba0",
+			"cpu_arm64": "ami-0ac6cc848dc354d0e"
 		},
 		"sa-east-1": {
-			"accelerated": "ami-00b89eb1afb4d1bc3",
-			"cpu": "ami-062970bb42bd2a893"
+			"accelerated_amd64": "ami-00b89eb1afb4d1bc3",
+			"cpu_amd64": "ami-062970bb42bd2a893",
+			"cpu_arm64": "ami-04011d161dcca2764"
 		},
 		"us-east-1": {
-			"accelerated": "ami-0defcae078e1e5047",
-			"cpu": "ami-0c29dd87e87fb4dfd"
+			"accelerated_amd64": "ami-0defcae078e1e5047",
+			"cpu_amd64": "ami-0c29dd87e87fb4dfd",
+			"cpu_arm64": "ami-020da68433ccd915e"
 		},
 		"us-east-2": {
-			"accelerated": "ami-0884b0ef7c1f7e7fc",
-			"cpu": "ami-0170c40fb8fc775fd"
+			"accelerated_amd64": "ami-0884b0ef7c1f7e7fc",
+			"cpu_amd64": "ami-0170c40fb8fc775fd",
+			"cpu_arm64": "ami-081909c0f3aaa7ff7"
 		},
 		"us-gov-east-1": {
-			"accelerated": "ami-07653f09bdcedd6e9",
-			"cpu": "ami-02e81a5a67eb65e67"
+			"accelerated_amd64": "ami-07653f09bdcedd6e9",
+			"cpu_amd64": "ami-02e81a5a67eb65e67",
+			"cpu_arm64": "ami-06e2a5be334ca6bef"
 		},
 		"us-gov-west-1": {
-			"accelerated": "ami-082b687837e601b3f",
-			"cpu": "ami-0531b9d822dfe1630"
+			"accelerated_amd64": "ami-082b687837e601b3f",
+			"cpu_amd64": "ami-0531b9d822dfe1630",
+			"cpu_arm64": "ami-0492cd68a612a51b8"
 		},
 		"us-west-1": {
-			"accelerated": "ami-055013041042c12f5",
-			"cpu": "ami-00d244e1909bc2e94"
+			"accelerated_amd64": "ami-055013041042c12f5",
+			"cpu_amd64": "ami-00d244e1909bc2e94",
+			"cpu_arm64": "ami-0f3bf36928134da10"
 		},
 		"us-west-2": {
-			"accelerated": "ami-042223ea760b6b965",
-			"cpu": "ami-035810617acd47976"
+			"accelerated_amd64": "ami-042223ea760b6b965",
+			"cpu_amd64": "ami-035810617acd47976",
+			"cpu_arm64": "ami-0b9104051172588d9"
 		}
 	}
 }

From 463f193c756d1d52e2af869586cfc3979580df66 Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Fri, 18 Jun 2021 22:24:45 +0300
Subject: [PATCH 07/28] Build kube-rbac-proxy for amd64/arm64

---
 images/kube-rbac-proxy/Dockerfile | 15 ++++++++++++++-
 images/kubexit/Dockerfile         |  8 ++++----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/images/kube-rbac-proxy/Dockerfile b/images/kube-rbac-proxy/Dockerfile
index 1ffeaecb1d..e9d51ccd9d 100644
--- a/images/kube-rbac-proxy/Dockerfile
+++ b/images/kube-rbac-proxy/Dockerfile
@@ -1 +1,14 @@
-FROM quay.io/coreos/kube-rbac-proxy:v0.4.1
+# This file has been adapted from https://github.com/brancz/kube-rbac-proxy
+ARG TARGETARCH, TARGETOS
+
+FROM golang:1.11 AS builder
+RUN git clone -b v0.4.1 --depth 1 https://github.com/brancz/kube-rbac-proxy /go/src/github.com/brancz/kube-rbac-proxy
+WORKDIR /go/src/github.com/brancz/kube-rbac-proxy
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build --installsuffix cgo -o kube-rbac-proxy github.com/brancz/kube-rbac-proxy \
+    && cp kube-rbac-proxy /usr/local/bin
+
+FROM alpine:3.8
+RUN apk add -U --no-cache ca-certificates && rm -rf /var/cache/apk/*
+COPY --from=builder /usr/local/bin/kube-rbac-proxy .
+ENTRYPOINT ["./kube-rbac-proxy"]
+EXPOSE 8080
diff --git a/images/kubexit/Dockerfile b/images/kubexit/Dockerfile
index 9851d0d2fc..77987ca179 100644
--- a/images/kubexit/Dockerfile
+++ b/images/kubexit/Dockerfile
@@ -1,13 +1,13 @@
 # This file has been adapted from https://github.com/cortexlabs/kubexit
-FROM golang:1.14.1-alpine3.11 AS builder
-RUN apk update && apk upgrade && \
-    apk add bash git openssh
+ARG TARGETARCH, TARGETOS
+
+FROM golang:1.14 AS builder
 
 RUN mkdir /tmp/kubexit
 RUN git clone -b v0.1.0-cortex --depth 1  https://github.com/cortexlabs/kubexit.git /tmp/kubexit
 
 WORKDIR /tmp/kubexit
-RUN CGO_ENABLED=0 GOOS=linux go build -o kubexit ./cmd/kubexit
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o kubexit ./cmd/kubexit
 
 FROM alpine:3.11
 RUN apk --no-cache add ca-certificates tzdata

From cef68f868af9f944c9e50abd12b8944283a3d372 Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Mon, 21 Jun 2021 20:29:35 +0300
Subject: [PATCH 08/28] Add kube-proxy to eks template and update vpc-cni

---
 manager/generate_eks.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/manager/generate_eks.py b/manager/generate_eks.py
index 046208147d..204c407818 100644
--- a/manager/generate_eks.py
+++ b/manager/generate_eks.py
@@ -341,7 +341,11 @@ def generate_eks(
         "addons": [
             {
                 "name": "vpc-cni",
-                "version": "1.7.10",
+                "version": "1.8.0",
+            },
+            {
+                "name": "kube-proxy",
+                "version": "1.18.8",
             },
         ],
     }

From fcb4e5ad09b5acb21372cad73f3dc64d439ba73f Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Mon, 21 Jun 2021 22:54:13 +0300
Subject: [PATCH 09/28] Makefile changes

---
 Makefile              |  4 +++
 build/build-image.sh  |  7 ++++-
 build/build-images.sh | 15 +++++-----
 build/images.sh       | 35 +++++++++++-----------
 build/push-image.sh   |  9 +++++-
 build/push-images.sh  | 15 +++++-----
 dev/registry.sh       | 69 ++++++++++++++++++++++++++++++-------------
 dev/util.sh           | 10 +++++++
 8 files changed, 107 insertions(+), 57 deletions(-)

diff --git a/Makefile b/Makefile
index 1461188697..902ba822d2 100644
--- a/Makefile
+++ b/Makefile
@@ -127,11 +127,15 @@ async-gateway-update:
 # docker images
 images-all:
 	@./dev/registry.sh update all
+images-all-multi-arch:
+	@./dev/registry.sh update all --include-arm64-arch
 images-all-skip-push:
 	@./dev/registry.sh update all --skip-push
 
 images-dev:
 	@./dev/registry.sh update dev
+images-dev-multi-arch:
+	@./dev/registry.sh update dev --include-arm64-arch
 images-dev-skip-push:
 	@./dev/registry.sh update dev --skip-push
 
diff --git a/build/build-image.sh b/build/build-image.sh
index 42fcee6933..522d16edb3 100755
--- a/build/build-image.sh
+++ b/build/build-image.sh
@@ -22,8 +22,13 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 CORTEX_VERSION=master
 
 image=$1
+multi_arch=$2
 
 if [ "$image" == "inferentia" ]; then
   aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 790709498068.dkr.ecr.us-west-2.amazonaws.com
 fi
-docker build "$ROOT" -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION}
+if [ "$multi_arch" == "false" ]; then
+  docker build "$ROOT" -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION}
+else
+  docker buildx build $ROOT -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION} --platform linux/amd64,linux/arm64
+fi
diff --git a/build/build-images.sh b/build/build-images.sh
index 7615d38e50..05f69a2656 100755
--- a/build/build-images.sh
+++ b/build/build-images.sh
@@ -22,11 +22,10 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 source $ROOT/build/images.sh
 source $ROOT/dev/util.sh
 
-# if parallel utility is installed, the docker build commands will be parallelized
-if command -v parallel &> /dev/null && [ -n "${NUM_BUILD_PROCS+set}" ] && [ "$NUM_BUILD_PROCS" != "1" ]; then
-  ROOT=$ROOT SHELL=$(type -p /bin/bash) parallel --will-cite --halt now,fail=1 --eta --jobs $NUM_BUILD_PROCS $ROOT/build/build-image.sh {} ::: "${all_images[@]}"
-else
-  for image in "${all_images[@]}"; do
-    $ROOT/build/build-image.sh $image
-  done
-fi
+for image in "${all_images[@]}"; do
+  multi_arch_image="false"
+  if in_array $image "multi_arch_images"; then
+    multi_arch_image="true"
+  fi
+  $ROOT/build/build-image.sh $image $multi_arch_image
+done
diff --git a/build/images.sh b/build/images.sh
index 340ac5cf7f..b19b91442e 100644
--- a/build/images.sh
+++ b/build/images.sh
@@ -21,27 +21,27 @@ set -euo pipefail
 
 dev_images=(
   "manager"
-  "proxy"
-  "async-gateway"
-  "enqueuer"
+  # "proxy"
+  # "async-gateway"
+  # "enqueuer"
   "dequeuer"
 )
 
 non_dev_images=(
-  "cluster-autoscaler"
-  "operator"
-  "controller-manager"
-  "istio-proxy"
-  "istio-pilot"
-  "fluent-bit"
-  "prometheus"
-  "prometheus-config-reloader"
-  "prometheus-operator"
-  "prometheus-statsd-exporter"
-  "prometheus-dcgm-exporter"
-  "prometheus-kube-state-metrics"
-  "prometheus-node-exporter"
-  "kube-rbac-proxy"
+  # "cluster-autoscaler"
+  # "operator"
+  # "controller-manager"
+  # "istio-proxy"
+  # "istio-pilot"
+  # "fluent-bit"
+  # "prometheus"
+  # "prometheus-config-reloader"
+  # "prometheus-operator"
+  # "prometheus-statsd-exporter"
+  # "prometheus-dcgm-exporter"
+  # "prometheus-kube-state-metrics"
+  # "prometheus-node-exporter"
+  # "kube-rbac-proxy"
   "grafana"
   "event-exporter"
   "metrics-server"
@@ -57,7 +57,6 @@ multi_arch_images=(
   "enqueuer"
   "dequeuer"
   "fluent-bit"
-  "prometheus-statsd-exporter" # remove it once a new cluster is spun up
   "prometheus-node-exporter"
   "kube-rbac-proxy"
   "kubexit"
diff --git a/build/push-image.sh b/build/push-image.sh
index b87ccec98f..8f37c8bb9d 100755
--- a/build/push-image.sh
+++ b/build/push-image.sh
@@ -17,10 +17,17 @@
 
 set -euo pipefail
 
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
+
 CORTEX_VERSION=master
 
 host=$1
 image=$2
+multi_arch=$2
 
 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-docker push $host/cortexlabs/${image}:${CORTEX_VERSION}
+if [ "$multi_arch" == "false" ]; then
+  docker push $host/cortexlabs/${image}:${CORTEX_VERSION}
+else
+  docker buildx build $ROOT -f $ROOT/images/$image/Dockerfile $host/cortexlabs/${image}:${CORTEX_VERSION} --platform linux/amd64,linux/arm64 --push
+fi
diff --git a/build/push-images.sh b/build/push-images.sh
index 59ebf75b71..6546686341 100755
--- a/build/push-images.sh
+++ b/build/push-images.sh
@@ -24,11 +24,10 @@ source $ROOT/dev/util.sh
 
 host=$1
 
-# if parallel utility is installed, the docker push commands will be parallelized
-if command -v parallel &> /dev/null && [ -n "${NUM_BUILD_PROCS+set}" ] && [ "$NUM_BUILD_PROCS" != "1" ]; then
-  ROOT=$ROOT DOCKER_USERNAME=$DOCKER_USERNAME DOCKER_PASSWORD=$DOCKER_PASSWORD SHELL=$(type -p /bin/bash) parallel --will-cite --halt now,fail=1 --eta --jobs $NUM_BUILD_PROCS $ROOT/build/push-image.sh $host {} ::: "${all_images[@]}"
-else
-  for image in "${all_images[@]}"; do
-    $ROOT/build/push-image.sh $host $image
-  done
-fi
+for image in "${all_images[@]}"; do
+  multi_arch_image="false"
+  if in_array $image "multi_arch_images"; then
+    multi_arch_image="true"
+  fi
+  $ROOT/build/push-image.sh $host $image $multi_arch_image
+done
diff --git a/dev/registry.sh b/dev/registry.sh
index f8e7312dc3..2ad57151da 100755
--- a/dev/registry.sh
+++ b/dev/registry.sh
@@ -23,8 +23,6 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 source $ROOT/build/images.sh
 source $ROOT/dev/util.sh
 
-images_with_builders="operator proxy async-gateway enqueuer dequeuer controller-manager"
-
 if [ -f "$ROOT/dev/config/env.sh" ]; then
   source $ROOT/dev/config/env.sh
 fi
@@ -33,6 +31,7 @@ AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID:-}
 AWS_REGION=${AWS_REGION:-}
 
 skip_push="false"
+include_arm64_arch="false"
 positional_args=()
 while [[ $# -gt 0 ]]; do
   key="$1"
@@ -41,6 +40,10 @@ while [[ $# -gt 0 ]]; do
     skip_push="true"
     shift
     ;;
+    --include-arm64-arch)
+    include_arm64_arch="true"
+    shift
+    ;;
     *)
     positional_args+=("$1")
     shift
@@ -120,15 +123,6 @@ function build() {
   green_echo "Built $image:$tag\n"
 }
 
-function cache_builder() {
-  local image=$1
-  local dir="${ROOT}/images/${image}"
-
-  blue_echo "Building $image-builder..."
-  docker build $ROOT -f $dir/Dockerfile -t cortexlabs/$image-builder:$CORTEX_VERSION --target builder
-  green_echo "Built $image-builder\n"
-}
-
 function push() {
   if [ "$skip_push" = "true" ]; then
     return
@@ -154,6 +148,40 @@ function build_and_push() {
   push $image $tag
 }
 
+function build_and_push_multi_arch() {
+  local image=$1
+  local include_arm64_arch=$2
+  local dir="${ROOT}/images/${image}"
+
+  set -euo pipefail  # necessary since this is called in a new shell by parallel
+
+  registry_login
+  if [ ! -n "$AWS_ACCOUNT_ID" ] || [ ! -n "$AWS_REGION" ]; then
+    echo "AWS_ACCOUNT_ID or AWS_REGION env vars not found"
+    exit 1
+  fi
+
+  tag=$CORTEX_VERSION
+  if [ "$include_arm64_arch" = "true" ]; then
+    blue_echo "Building and pushing $image:$tag with amd64/arm64 arch support..."
+  else
+    blue_echo "Building and pushing $image:$tag with amd64 arch support only..."
+  fi
+
+  platforms="--platform linux/amd64"
+  if [ "$include_arm64_arch" = "true" ]; then
+    platforms+=",linux/arm64"
+  fi
+
+  docker buildx build $ROOT -f $dir/Dockerfile -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag $platforms --push
+
+  if [ "$include_arm64_arch" = "true" ]; then
+    green_echo "Built and pushed $image:$tag with amd64/arm64 arch support..."
+  else
+    green_echo "Built and pushed $image:$tag with amd64 arch support only..."
+  fi
+}
+
 function cleanup_local() {
   echo "cleaning local repositories..."
   docker container prune -f
@@ -218,10 +246,11 @@ elif [ "$cmd" = "create" ]; then
 # usage: registry.sh update-single IMAGE
 elif [ "$cmd" = "update-single" ]; then
   image=$sub_cmd
-  if [[ " $images_with_builders " =~ " $image " ]]; then
-    cache_builder $image
+  if ! in_array $image "multi_arch_images"; then
+    build_and_push $image
+  else
+    build_and_push_multi_arch $image $include_arm64_arch
   fi
-  build_and_push $image
 
 # usage: registry.sh update all|dev|api
 # if parallel utility is installed, the docker build commands will be parallelized
@@ -236,17 +265,15 @@ elif [ "$cmd" = "update" ]; then
     images_to_build+=( "${dev_images[@]}" )
   fi
 
-  for image in $images_with_builders; do
-    if [[ " ${images_to_build[@]} " =~ " $image " ]]; then
-      cache_builder $image
-    fi
-  done
-
   if command -v parallel &> /dev/null && [ -n "${NUM_BUILD_PROCS+set}" ] && [ "$NUM_BUILD_PROCS" != "1" ]; then
     is_registry_logged_in=$is_registry_logged_in ROOT=$ROOT registry_push_url=$registry_push_url SHELL=$(type -p /bin/bash) parallel --will-cite --halt now,fail=1 --eta --jobs $NUM_BUILD_PROCS build_and_push "{}" ::: "${images_to_build[@]}"
   else
     for image in "${images_to_build[@]}"; do
-      build_and_push $image
+      if in_array $image "multi_arch_images"; then
+        build_and_push $image
+      else
+        build_and_push_multi_arch $image $include_arm64_arch
+      fi
     done
   fi
 
diff --git a/dev/util.sh b/dev/util.sh
index 9595ecc381..1d3fcea532 100755
--- a/dev/util.sh
+++ b/dev/util.sh
@@ -45,3 +45,13 @@ function join_by() {
   done
   echo
 }
+
+# Check if array contains item [$1: item, $2: array name]
+function in_array() {
+    local needle="$1" item
+    local -n arrref="$2"
+    for item in "${arrref[@]}"; do
+        [[ "${item}" == "${needle}" ]] && return 0
+    done
+    return 1
+}

From ba9edcea30e29385ca07ac2ae0c2e8010d7e0806 Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Mon, 21 Jun 2021 22:54:56 +0300
Subject: [PATCH 10/28] Uncomment images

---
 build/images.sh | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/build/images.sh b/build/images.sh
index b19b91442e..32a3f1b661 100644
--- a/build/images.sh
+++ b/build/images.sh
@@ -21,27 +21,27 @@ set -euo pipefail
 
 dev_images=(
   "manager"
-  # "proxy"
-  # "async-gateway"
-  # "enqueuer"
+  "proxy"
+  "async-gateway"
+  "enqueuer"
   "dequeuer"
 )
 
 non_dev_images=(
-  # "cluster-autoscaler"
-  # "operator"
-  # "controller-manager"
-  # "istio-proxy"
-  # "istio-pilot"
-  # "fluent-bit"
-  # "prometheus"
-  # "prometheus-config-reloader"
-  # "prometheus-operator"
-  # "prometheus-statsd-exporter"
-  # "prometheus-dcgm-exporter"
-  # "prometheus-kube-state-metrics"
-  # "prometheus-node-exporter"
-  # "kube-rbac-proxy"
+  "cluster-autoscaler"
+  "operator"
+  "controller-manager"
+  "istio-proxy"
+  "istio-pilot"
+  "fluent-bit"
+  "prometheus"
+  "prometheus-config-reloader"
+  "prometheus-operator"
+  "prometheus-statsd-exporter"
+  "prometheus-dcgm-exporter"
+  "prometheus-kube-state-metrics"
+  "prometheus-node-exporter"
+  "kube-rbac-proxy"
   "grafana"
   "event-exporter"
   "metrics-server"

From 5672309165bf727d145c0fbe6051ad2536b84765 Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 00:01:05 +0300
Subject: [PATCH 11/28] WIP arm instances

---
 .circleci/config.yml                          |  4 ++++
 .../hello-world/cortex_cpu_arm64.yaml         | 16 +++++++++++++++
 test/apis/realtime/hello-world/sample.json    |  1 +
 test/e2e/tests/aws/test_realtime.py           | 20 +++++++++++++++++++
 4 files changed, 41 insertions(+)
 create mode 100644 test/apis/realtime/hello-world/cortex_cpu_arm64.yaml
 create mode 100644 test/apis/realtime/hello-world/sample.json

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 17b42d7c24..33a62c24f0 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -174,6 +174,10 @@ jobs:
                 instance_type: inf1.xlarge
                 min_instances: 1
                 max_instances: 2
+              - name: arm
+                instance_type: a1.large
+                min_instances: 1
+                max_instances: 2
             EOF
       - run:
           name: Create/Update AWS User policy
diff --git a/test/apis/realtime/hello-world/cortex_cpu_arm64.yaml b/test/apis/realtime/hello-world/cortex_cpu_arm64.yaml
new file mode 100644
index 0000000000..7c953a8f13
--- /dev/null
+++ b/test/apis/realtime/hello-world/cortex_cpu_arm64.yaml
@@ -0,0 +1,16 @@
+- name: hello-world
+  kind: RealtimeAPI
+  pod:
+    port: 8080
+    max_concurrency: 1
+    containers:
+    - name: api
+      image: infrastructureascode/hello-world
+      readiness_probe:
+        http_get:
+          path: "/healthz"
+          port: 8080
+      compute:
+        cpu: 200m
+        mem: 128Mi
+  node_groups: ["arm"]
diff --git a/test/apis/realtime/hello-world/sample.json b/test/apis/realtime/hello-world/sample.json
new file mode 100644
index 0000000000..0967ef424b
--- /dev/null
+++ b/test/apis/realtime/hello-world/sample.json
@@ -0,0 +1 @@
+{}
diff --git a/test/e2e/tests/aws/test_realtime.py b/test/e2e/tests/aws/test_realtime.py
index 71c4b2fd13..e2a3656dfa 100644
--- a/test/e2e/tests/aws/test_realtime.py
+++ b/test/e2e/tests/aws/test_realtime.py
@@ -32,6 +32,12 @@
         "extra_path": "",
     },
 ]
+TEST_APIS_ARM = [
+    {
+        "name": "realtime/hello-world",
+        "extra_path": "",
+    },
+]
 TEST_APIS_GPU = [
     {
         "name": "realtime/image-classifier-resnet50",
@@ -57,6 +63,20 @@ def test_realtime_api(printer: Callable, config: Dict, client: cx.Client, api: D
         extra_path=api["extra_path"],
     )
 
+@pytest.mark.usefixtures("client")
+@pytest.mark.parametrize("api", TEST_APIS_ARM)
+def test_realtime_api(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]):
+
+    printer(f"testing {api['name']}")
+    e2e.tests.test_realtime_api(
+        printer=printer,
+        client=client,
+        api=api["name"],
+        timeout=config["global"]["realtime_deploy_timeout"],
+        api_config_name="cortex_cpu_arm64.yaml",
+        extra_path=api["extra_path"],
+    )
+
 
 @pytest.mark.usefixtures("client")
 @pytest.mark.parametrize("api", TEST_APIS_GPU)

From d0e37b81c700cc4f0e01e2b0908439150d0a5f4b Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 00:24:45 +0300
Subject: [PATCH 12/28] Simplify registry.sh script

---
 dev/registry.sh | 88 ++++++++++++++++++++++++-------------------------
 1 file changed, 43 insertions(+), 45 deletions(-)

diff --git a/dev/registry.sh b/dev/registry.sh
index 2ad57151da..a546d31c9c 100755
--- a/dev/registry.sh
+++ b/dev/registry.sh
@@ -23,6 +23,9 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 source $ROOT/build/images.sh
 source $ROOT/dev/util.sh
 
+images_with_builders="operator proxy async-gateway enqueuer dequeuer controller-manager kube-rbac-proxy kubexit"
+images_that_can_run_locally="operator"
+
 if [ -f "$ROOT/dev/config/env.sh" ]; then
   source $ROOT/dev/config/env.sh
 fi
@@ -108,53 +111,48 @@ function create_ecr_repository() {
 
 ### HELPERS ###
 
-function build() {
+function cache_builder() {
   local image=$1
-  local tag=$2
+  local include_arm64_arch=$2
   local dir="${ROOT}/images/${image}"
 
-  tag_args=""
-  if [ -n "$AWS_ACCOUNT_ID" ] && [ -n "$AWS_REGION" ]; then
-    tag_args+=" -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag"
+  if ! in_array $image "multi_arch_images"; then
+    include_arm64_arch="false"
   fi
 
-  blue_echo "Building $image:$tag..."
-  docker build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag $tag_args
-  green_echo "Built $image:$tag\n"
-}
+  tag=$CORTEX_VERSION
 
-function push() {
-  if [ "$skip_push" = "true" ]; then
-    return
+  if [ "$include_arm64_arch" = "true" ]; then
+    blue_echo "Building $image-builder with amd64/arm64 arch support..."
+  else
+    blue_echo "Building $image-builder with amd64 arch support only..."
   fi
 
-  registry_login
+  platforms="--platform linux/amd64"
+  if [ "$include_arm64_arch" = "true" ]; then
+    platforms+=",linux/arm64"
+  fi
 
-  local image=$1
-  local tag=$2
+  docker buildx build $ROOT -f $dir/Dockerfile -t cortexlabs/$image-builder:$tag $platforms --target builder
 
-  blue_echo "Pushing $image:$tag..."
-  docker push $registry_push_url/cortexlabs/$image:$tag
-  green_echo "Pushed $image:$tag\n"
+  if [ "$include_arm64_arch" = "true" ]; then
+    green_echo "Built $image-builder with amd64/arm64 arch support..."
+  else
+    green_echo "Built $image-builder with amd64 arch support only..."
+  fi
 }
 
 function build_and_push() {
   local image=$1
-
-  set -euo pipefail  # necessary since this is called in a new shell by parallel
-
-  tag=$CORTEX_VERSION
-  build $image $tag
-  push $image $tag
-}
-
-function build_and_push_multi_arch() {
-  local image=$1
   local include_arm64_arch=$2
   local dir="${ROOT}/images/${image}"
 
   set -euo pipefail  # necessary since this is called in a new shell by parallel
 
+  if ! in_array $image "multi_arch_images"; then
+    include_arm64_arch="false"
+  fi
+
   registry_login
   if [ ! -n "$AWS_ACCOUNT_ID" ] || [ ! -n "$AWS_REGION" ]; then
     echo "AWS_ACCOUNT_ID or AWS_REGION env vars not found"
@@ -180,12 +178,19 @@ function build_and_push_multi_arch() {
   else
     green_echo "Built and pushed $image:$tag with amd64 arch support only..."
   fi
+
+  blue_echo "Exporting $image:$tag to local docker..."
+  if [[ " $images_that_can_run_locally " =~ " $image " ]] && [[ "$include_arm64_arch" == "false" ]]; then
+    docker buildx build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag $platforms --load
+  fi
+  green_echo "Exported $image:$tag to local docker..."
 }
 
 function cleanup_local() {
   echo "cleaning local repositories..."
   docker container prune -f
   docker image prune -f
+  docker buildx prune -f
 }
 
 function cleanup_ecr() {
@@ -223,14 +228,6 @@ function validate_env() {
   fi
 }
 
-# export functions for parallel command
-export -f build_and_push
-export -f push
-export -f build
-export -f blue_echo
-export -f green_echo
-export -f registry_login
-
 # validate environment is correctly set on env.sh
 validate_env
 
@@ -246,11 +243,10 @@ elif [ "$cmd" = "create" ]; then
 # usage: registry.sh update-single IMAGE
 elif [ "$cmd" = "update-single" ]; then
   image=$sub_cmd
-  if ! in_array $image "multi_arch_images"; then
-    build_and_push $image
-  else
-    build_and_push_multi_arch $image $include_arm64_arch
+  if [[ " $images_with_builders " =~ " $image " ]]; then
+    cache_builder $image $include_arm64_arch
   fi
+  build_and_push $image $include_arm64_arch
 
 # usage: registry.sh update all|dev|api
 # if parallel utility is installed, the docker build commands will be parallelized
@@ -265,15 +261,17 @@ elif [ "$cmd" = "update" ]; then
     images_to_build+=( "${dev_images[@]}" )
   fi
 
+  for image in $images_with_builders; do
+    if [[ " ${images_to_build[@]} " =~ " $image " ]]; then
+      cache_builder $image $include_arm64_arch
+    fi
+  done
+
   if command -v parallel &> /dev/null && [ -n "${NUM_BUILD_PROCS+set}" ] && [ "$NUM_BUILD_PROCS" != "1" ]; then
     is_registry_logged_in=$is_registry_logged_in ROOT=$ROOT registry_push_url=$registry_push_url SHELL=$(type -p /bin/bash) parallel --will-cite --halt now,fail=1 --eta --jobs $NUM_BUILD_PROCS build_and_push "{}" ::: "${images_to_build[@]}"
   else
     for image in "${images_to_build[@]}"; do
-      if in_array $image "multi_arch_images"; then
-        build_and_push $image
-      else
-        build_and_push_multi_arch $image $include_arm64_arch
-      fi
+      build_and_push $image $include_arm64_arch
     done
   fi
 

From ba7cf0a5339f3e99cdb66202853257fcf1037159 Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 00:33:52 +0300
Subject: [PATCH 13/28] Changes to make-ci-* cmds

---
 build/build-image.sh                | 13 +++++++------
 build/build-images.sh               |  6 +++---
 build/push-image.sh                 | 13 +++++++------
 build/push-images.sh                |  6 +++---
 test/e2e/tests/aws/test_realtime.py |  1 +
 5 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/build/build-image.sh b/build/build-image.sh
index 522d16edb3..dba2334d26 100755
--- a/build/build-image.sh
+++ b/build/build-image.sh
@@ -22,13 +22,14 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 CORTEX_VERSION=master
 
 image=$1
-multi_arch=$2
+include_arm64_arch=$2
+
+platforms="--platform linux/amd64"
+if [ "$include_arm64_arch" == "false" ]; then
+  platforms+=",linux/arm64"
+fi
 
 if [ "$image" == "inferentia" ]; then
   aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 790709498068.dkr.ecr.us-west-2.amazonaws.com
 fi
-if [ "$multi_arch" == "false" ]; then
-  docker build "$ROOT" -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION}
-else
-  docker buildx build $ROOT -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION} --platform linux/amd64,linux/arm64
-fi
+docker buildx build $ROOT -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION} $platforms
diff --git a/build/build-images.sh b/build/build-images.sh
index 05f69a2656..7e35cfba1b 100755
--- a/build/build-images.sh
+++ b/build/build-images.sh
@@ -23,9 +23,9 @@ source $ROOT/build/images.sh
 source $ROOT/dev/util.sh
 
 for image in "${all_images[@]}"; do
-  multi_arch_image="false"
+  include_arm64_arch="false"
   if in_array $image "multi_arch_images"; then
-    multi_arch_image="true"
+    include_arm64_arch="true"
   fi
-  $ROOT/build/build-image.sh $image $multi_arch_image
+  $ROOT/build/build-image.sh $image $include_arm64_arch
 done
diff --git a/build/push-image.sh b/build/push-image.sh
index 8f37c8bb9d..4a52f3b270 100755
--- a/build/push-image.sh
+++ b/build/push-image.sh
@@ -23,11 +23,12 @@ CORTEX_VERSION=master
 
 host=$1
 image=$2
-multi_arch=$2
+include_arm64_arch=$2
 
-echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-if [ "$multi_arch" == "false" ]; then
-  docker push $host/cortexlabs/${image}:${CORTEX_VERSION}
-else
-  docker buildx build $ROOT -f $ROOT/images/$image/Dockerfile $host/cortexlabs/${image}:${CORTEX_VERSION} --platform linux/amd64,linux/arm64 --push
+platforms="--platform linux/amd64"
+if [ "$include_arm64_arch" == "false" ]; then
+  platforms+=",linux/arm64"
 fi
+
+echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+docker buildx build $ROOT -f $ROOT/images/$image/Dockerfile $host/cortexlabs/${image}:${CORTEX_VERSION} $platforms --push
diff --git a/build/push-images.sh b/build/push-images.sh
index 6546686341..b298c2120d 100755
--- a/build/push-images.sh
+++ b/build/push-images.sh
@@ -25,9 +25,9 @@ source $ROOT/dev/util.sh
 host=$1
 
 for image in "${all_images[@]}"; do
-  multi_arch_image="false"
+  include_arm64_arch="false"
   if in_array $image "multi_arch_images"; then
-    multi_arch_image="true"
+    include_arm64_arch="true"
   fi
-  $ROOT/build/push-image.sh $host $image $multi_arch_image
+  $ROOT/build/push-image.sh $host $image $include_arm64_arch
 done
diff --git a/test/e2e/tests/aws/test_realtime.py b/test/e2e/tests/aws/test_realtime.py
index e2a3656dfa..62b1e4638c 100644
--- a/test/e2e/tests/aws/test_realtime.py
+++ b/test/e2e/tests/aws/test_realtime.py
@@ -63,6 +63,7 @@ def test_realtime_api(printer: Callable, config: Dict, client: cx.Client, api: D
         extra_path=api["extra_path"],
     )
 
+
 @pytest.mark.usefixtures("client")
 @pytest.mark.parametrize("api", TEST_APIS_ARM)
 def test_realtime_api(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]):

From 3dd47a9e6f3d98034e65eed71cf5fe58485195b8 Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 00:39:31 +0300
Subject: [PATCH 14/28] Fixes and nits

---
 .circleci/config.yml |  2 +-
 Makefile             |  1 -
 dev/registry.sh      | 11 +++--------
 3 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 33a62c24f0..991a2b25ad 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -121,7 +121,7 @@ jobs:
       - checkout
       - setup_remote_docker
       - install-go
-      - run: sudo apt update && sudo apt install parallel -y
+      - run: docker buildx create --name builder --platform linux/amd64,linux/arm64 --use && docker buildx inspect --bootstrap
       - run: sudo pip install awscli
       - run: make ci-build-cli
       - return-if-not-deployed-branch
diff --git a/Makefile b/Makefile
index 902ba822d2..50bb13633f 100644
--- a/Makefile
+++ b/Makefile
@@ -155,7 +155,6 @@ tools:
 	@go get -u -v github.com/kyoh86/looppointer/cmd/looppointer
 	@go get -u -v github.com/VojtechVitek/rerun/cmd/rerun
 	@go get -u -v github.com/go-delve/delve/cmd/dlv
-	@if [[ "$$OSTYPE" == "darwin"* ]]; then brew install parallel; elif [[ "$$OSTYPE" == "linux"* ]]; then sudo apt-get install -y parallel; else echo "your operating system is not supported"; fi
 	@python3 -m pip install aiohttp black 'pydoc-markdown>=3.0.0,<4.0.0' boto3 pyyaml
 	@python3 -m pip install -e test/e2e
 
diff --git a/dev/registry.sh b/dev/registry.sh
index a546d31c9c..3c2bf79495 100755
--- a/dev/registry.sh
+++ b/dev/registry.sh
@@ -249,7 +249,6 @@ elif [ "$cmd" = "update-single" ]; then
   build_and_push $image $include_arm64_arch
 
 # usage: registry.sh update all|dev|api
-# if parallel utility is installed, the docker build commands will be parallelized
 elif [ "$cmd" = "update" ]; then
   images_to_build=()
 
@@ -267,13 +266,9 @@ elif [ "$cmd" = "update" ]; then
     fi
   done
 
-  if command -v parallel &> /dev/null && [ -n "${NUM_BUILD_PROCS+set}" ] && [ "$NUM_BUILD_PROCS" != "1" ]; then
-    is_registry_logged_in=$is_registry_logged_in ROOT=$ROOT registry_push_url=$registry_push_url SHELL=$(type -p /bin/bash) parallel --will-cite --halt now,fail=1 --eta --jobs $NUM_BUILD_PROCS build_and_push "{}" ::: "${images_to_build[@]}"
-  else
-    for image in "${images_to_build[@]}"; do
-      build_and_push $image $include_arm64_arch
-    done
-  fi
+  for image in "${images_to_build[@]}"; do
+    build_and_push $image $include_arm64_arch
+  done
 
   cleanup_local
 

From 26e07581e44232bf8fe1f7356ef044bb803678df Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 01:20:26 +0300
Subject: [PATCH 15/28] Fixes

---
 Makefile           |  3 +++
 dev/registry.sh    | 11 ++++++-----
 manager/install.sh |  2 +-
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 50bb13633f..915936afe0 100644
--- a/Makefile
+++ b/Makefile
@@ -142,6 +142,9 @@ images-dev-skip-push:
 images-manager-skip-push:
 	@./dev/registry.sh update-single manager --skip-push
 
+images-clean-cache:
+	@./dev/registry.sh clean-cache
+
 registry-create:
 	@./dev/registry.sh create
 
diff --git a/dev/registry.sh b/dev/registry.sh
index 3c2bf79495..54cab03102 100755
--- a/dev/registry.sh
+++ b/dev/registry.sh
@@ -24,7 +24,7 @@ source $ROOT/build/images.sh
 source $ROOT/dev/util.sh
 
 images_with_builders="operator proxy async-gateway enqueuer dequeuer controller-manager kube-rbac-proxy kubexit"
-images_that_can_run_locally="operator"
+images_that_can_run_locally="operator manager"
 
 if [ -f "$ROOT/dev/config/env.sh" ]; then
   source $ROOT/dev/config/env.sh
@@ -147,7 +147,7 @@ function build_and_push() {
   local include_arm64_arch=$2
   local dir="${ROOT}/images/${image}"
 
-  set -euo pipefail  # necessary since this is called in a new shell by parallel
+  set -euo pipefail
 
   if ! in_array $image "multi_arch_images"; then
     include_arm64_arch="false"
@@ -179,11 +179,11 @@ function build_and_push() {
     green_echo "Built and pushed $image:$tag with amd64 arch support only..."
   fi
 
-  blue_echo "Exporting $image:$tag to local docker..."
   if [[ " $images_that_can_run_locally " =~ " $image " ]] && [[ "$include_arm64_arch" == "false" ]]; then
+   blue_echo "Exporting $image:$tag to local docker..."
     docker buildx build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag $platforms --load
+    green_echo "Exported $image:$tag to local docker..."
   fi
-  green_echo "Exported $image:$tag to local docker..."
 }
 
 function cleanup_local() {
@@ -270,8 +270,9 @@ elif [ "$cmd" = "update" ]; then
     build_and_push $image $include_arm64_arch
   done
 
+# usage: registry.sh clean-cache
+elif [ "$cmd" = "clean-cache" ]; then
   cleanup_local
-
 else
   echo "unknown command: $cmd"
   exit 1
diff --git a/manager/install.sh b/manager/install.sh
index d76de8a69d..69f42800d9 100755
--- a/manager/install.sh
+++ b/manager/install.sh
@@ -333,7 +333,7 @@ function add_nodegroups() {
 
   echo "￮ adding new nodegroup(s) to the cluster ..."
   python generate_eks.py $CORTEX_CLUSTER_CONFIG_FILE manifests/ami.json --add-cortex-node-groups="$nodegroup_names" > /workspace/nodegroups.yaml
-  eksctl create nodegroup --timeout=$EKSCTL_NODEGROUP_TIMEOUT --install-neuron-plugin=false --install-nvidia-plugin=false -f /workspace/nodegroups.yaml
+  eksctl create nodegroup --timeout=$EKSCTL_NODEGROUP_TIMEOUT --install-neuron-plugin=false --install-nvidia-plugin=false --skip-outdated-addons-check -f /workspace/nodegroups.yaml
   rm /workspace/nodegroups.yaml
   echo
 }

From f8747bff2662ee462f8cdd5d86fafeebab4a6ced Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 01:21:27 +0300
Subject: [PATCH 16/28] Remove cache builder from registry.sh

---
 dev/registry.sh | 41 -----------------------------------------
 1 file changed, 41 deletions(-)

diff --git a/dev/registry.sh b/dev/registry.sh
index 54cab03102..c967d0e8ec 100755
--- a/dev/registry.sh
+++ b/dev/registry.sh
@@ -23,7 +23,6 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 source $ROOT/build/images.sh
 source $ROOT/dev/util.sh
 
-images_with_builders="operator proxy async-gateway enqueuer dequeuer controller-manager kube-rbac-proxy kubexit"
 images_that_can_run_locally="operator manager"
 
 if [ -f "$ROOT/dev/config/env.sh" ]; then
@@ -111,37 +110,6 @@ function create_ecr_repository() {
 
 ### HELPERS ###
 
-function cache_builder() {
-  local image=$1
-  local include_arm64_arch=$2
-  local dir="${ROOT}/images/${image}"
-
-  if ! in_array $image "multi_arch_images"; then
-    include_arm64_arch="false"
-  fi
-
-  tag=$CORTEX_VERSION
-
-  if [ "$include_arm64_arch" = "true" ]; then
-    blue_echo "Building $image-builder with amd64/arm64 arch support..."
-  else
-    blue_echo "Building $image-builder with amd64 arch support only..."
-  fi
-
-  platforms="--platform linux/amd64"
-  if [ "$include_arm64_arch" = "true" ]; then
-    platforms+=",linux/arm64"
-  fi
-
-  docker buildx build $ROOT -f $dir/Dockerfile -t cortexlabs/$image-builder:$tag $platforms --target builder
-
-  if [ "$include_arm64_arch" = "true" ]; then
-    green_echo "Built $image-builder with amd64/arm64 arch support..."
-  else
-    green_echo "Built $image-builder with amd64 arch support only..."
-  fi
-}
-
 function build_and_push() {
   local image=$1
   local include_arm64_arch=$2
@@ -243,9 +211,6 @@ elif [ "$cmd" = "create" ]; then
 # usage: registry.sh update-single IMAGE
 elif [ "$cmd" = "update-single" ]; then
   image=$sub_cmd
-  if [[ " $images_with_builders " =~ " $image " ]]; then
-    cache_builder $image $include_arm64_arch
-  fi
   build_and_push $image $include_arm64_arch
 
 # usage: registry.sh update all|dev|api
@@ -260,12 +225,6 @@ elif [ "$cmd" = "update" ]; then
     images_to_build+=( "${dev_images[@]}" )
   fi
 
-  for image in $images_with_builders; do
-    if [[ " ${images_to_build[@]} " =~ " $image " ]]; then
-      cache_builder $image $include_arm64_arch
-    fi
-  done
-
   for image in "${images_to_build[@]}"; do
     build_and_push $image $include_arm64_arch
   done

From 05ef1ba5a63532116342c26a1a506507284c501c Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 01:58:33 +0300
Subject: [PATCH 17/28] Nits

---
 build/build-image.sh | 6 +++---
 build/push-image.sh  | 6 +++---
 dev/registry.sh      | 6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/build/build-image.sh b/build/build-image.sh
index dba2334d26..d3de44b9f4 100755
--- a/build/build-image.sh
+++ b/build/build-image.sh
@@ -24,12 +24,12 @@ CORTEX_VERSION=master
 image=$1
 include_arm64_arch=$2
 
-platforms="--platform linux/amd64"
-if [ "$include_arm64_arch" == "false" ]; then
+platforms="linux/amd64"
+if [ "$include_arm64_arch" == "true" ]; then
   platforms+=",linux/arm64"
 fi
 
 if [ "$image" == "inferentia" ]; then
   aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 790709498068.dkr.ecr.us-west-2.amazonaws.com
 fi
-docker buildx build $ROOT -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION} $platforms
+docker buildx build $ROOT -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION} --platform $platforms
diff --git a/build/push-image.sh b/build/push-image.sh
index 4a52f3b270..ddd503984c 100755
--- a/build/push-image.sh
+++ b/build/push-image.sh
@@ -25,10 +25,10 @@ host=$1
 image=$2
 include_arm64_arch=$2
 
-platforms="--platform linux/amd64"
-if [ "$include_arm64_arch" == "false" ]; then
+platforms="linux/amd64"
+if [ "$include_arm64_arch" == "true" ]; then
   platforms+=",linux/arm64"
 fi
 
 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-docker buildx build $ROOT -f $ROOT/images/$image/Dockerfile $host/cortexlabs/${image}:${CORTEX_VERSION} $platforms --push
+docker buildx build $ROOT -f $ROOT/images/$image/Dockerfile $host/cortexlabs/${image}:${CORTEX_VERSION} --platform $platforms --push
diff --git a/dev/registry.sh b/dev/registry.sh
index c967d0e8ec..f04ba7dc9d 100755
--- a/dev/registry.sh
+++ b/dev/registry.sh
@@ -134,12 +134,12 @@ function build_and_push() {
     blue_echo "Building and pushing $image:$tag with amd64 arch support only..."
   fi
 
-  platforms="--platform linux/amd64"
+  platforms="linux/amd64"
   if [ "$include_arm64_arch" = "true" ]; then
     platforms+=",linux/arm64"
   fi
 
-  docker buildx build $ROOT -f $dir/Dockerfile -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag $platforms --push
+  docker buildx build $ROOT -f $dir/Dockerfile -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag --platform $platforms --push
 
   if [ "$include_arm64_arch" = "true" ]; then
     green_echo "Built and pushed $image:$tag with amd64/arm64 arch support..."
@@ -149,7 +149,7 @@ function build_and_push() {
 
   if [[ " $images_that_can_run_locally " =~ " $image " ]] && [[ "$include_arm64_arch" == "false" ]]; then
    blue_echo "Exporting $image:$tag to local docker..."
-    docker buildx build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag $platforms --load
+    docker buildx build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag --platform $platforms --load
     green_echo "Exported $image:$tag to local docker..."
   fi
 }

From 18b759ade87f0e1e51ecb7d6a23b1c1582ea0411 Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 02:03:53 +0300
Subject: [PATCH 18/28] Fix

---
 build/push-image.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/push-image.sh b/build/push-image.sh
index ddd503984c..e14d353d7c 100755
--- a/build/push-image.sh
+++ b/build/push-image.sh
@@ -23,7 +23,7 @@ CORTEX_VERSION=master
 
 host=$1
 image=$2
-include_arm64_arch=$2
+include_arm64_arch=$3
 
 platforms="linux/amd64"
 if [ "$include_arm64_arch" == "true" ]; then

From 69df0207a642730f821cd368e79760b18c1b182c Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 02:06:08 +0300
Subject: [PATCH 19/28] Nits

---
 build/build-image.sh  | 7 +------
 build/build-images.sh | 6 +++---
 build/push-image.sh   | 7 +------
 build/push-images.sh  | 6 +++---
 4 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/build/build-image.sh b/build/build-image.sh
index d3de44b9f4..352582e251 100755
--- a/build/build-image.sh
+++ b/build/build-image.sh
@@ -22,12 +22,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 CORTEX_VERSION=master
 
 image=$1
-include_arm64_arch=$2
-
-platforms="linux/amd64"
-if [ "$include_arm64_arch" == "true" ]; then
-  platforms+=",linux/arm64"
-fi
+platforms=$2
 
 if [ "$image" == "inferentia" ]; then
   aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 790709498068.dkr.ecr.us-west-2.amazonaws.com
diff --git a/build/build-images.sh b/build/build-images.sh
index 7e35cfba1b..f3dd265982 100755
--- a/build/build-images.sh
+++ b/build/build-images.sh
@@ -23,9 +23,9 @@ source $ROOT/build/images.sh
 source $ROOT/dev/util.sh
 
 for image in "${all_images[@]}"; do
-  include_arm64_arch="false"
+  platforms="linux/amd64"
   if in_array $image "multi_arch_images"; then
-    include_arm64_arch="true"
+    platforms+=",linux/arm64"
   fi
-  $ROOT/build/build-image.sh $image $include_arm64_arch
+  $ROOT/build/build-image.sh $image $platforms
 done
diff --git a/build/push-image.sh b/build/push-image.sh
index e14d353d7c..35c6bbc422 100755
--- a/build/push-image.sh
+++ b/build/push-image.sh
@@ -23,12 +23,7 @@ CORTEX_VERSION=master
 
 host=$1
 image=$2
-include_arm64_arch=$3
-
-platforms="linux/amd64"
-if [ "$include_arm64_arch" == "true" ]; then
-  platforms+=",linux/arm64"
-fi
+platforms=$3
 
 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
 docker buildx build $ROOT -f $ROOT/images/$image/Dockerfile $host/cortexlabs/${image}:${CORTEX_VERSION} --platform $platforms --push
diff --git a/build/push-images.sh b/build/push-images.sh
index b298c2120d..75c5b73677 100755
--- a/build/push-images.sh
+++ b/build/push-images.sh
@@ -25,9 +25,9 @@ source $ROOT/dev/util.sh
 host=$1
 
 for image in "${all_images[@]}"; do
-  include_arm64_arch="false"
+  platforms="linux/amd64"
   if in_array $image "multi_arch_images"; then
-    include_arm64_arch="true"
+    platforms+=",linux/arm64"
   fi
-  $ROOT/build/push-image.sh $host $image $include_arm64_arch
+  $ROOT/build/push-image.sh $host $image $platforms
 done

From c4e5571b901d14f2a87369a02ee79dc79e70b53a Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 02:12:27 +0300
Subject: [PATCH 20/28] Revert empty line

---
 dev/registry.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dev/registry.sh b/dev/registry.sh
index f04ba7dc9d..e7a19b0028 100755
--- a/dev/registry.sh
+++ b/dev/registry.sh
@@ -232,6 +232,7 @@ elif [ "$cmd" = "update" ]; then
 # usage: registry.sh clean-cache
 elif [ "$cmd" = "clean-cache" ]; then
   cleanup_local
+  
 else
   echo "unknown command: $cmd"
   exit 1

From 0d5076ddadd5f25c1a03a4da33e3ecee22498300 Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 02:17:21 +0300
Subject: [PATCH 21/28] Fixes

---
 dev/registry.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dev/registry.sh b/dev/registry.sh
index e7a19b0028..d3bbc3ec83 100755
--- a/dev/registry.sh
+++ b/dev/registry.sh
@@ -121,11 +121,11 @@ function build_and_push() {
     include_arm64_arch="false"
   fi
 
-  registry_login
   if [ ! -n "$AWS_ACCOUNT_ID" ] || [ ! -n "$AWS_REGION" ]; then
     echo "AWS_ACCOUNT_ID or AWS_REGION env vars not found"
     exit 1
   fi
+  registry_login
 
   tag=$CORTEX_VERSION
   if [ "$include_arm64_arch" = "true" ]; then
@@ -148,7 +148,7 @@ function build_and_push() {
   fi
 
   if [[ " $images_that_can_run_locally " =~ " $image " ]] && [[ "$include_arm64_arch" == "false" ]]; then
-   blue_echo "Exporting $image:$tag to local docker..."
+    blue_echo "Exporting $image:$tag to local docker..."
     docker buildx build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag --platform $platforms --load
     green_echo "Exported $image:$tag to local docker..."
   fi
@@ -232,7 +232,7 @@ elif [ "$cmd" = "update" ]; then
 # usage: registry.sh clean-cache
 elif [ "$cmd" = "clean-cache" ]; then
   cleanup_local
-  
+
 else
   echo "unknown command: $cmd"
   exit 1

From fdf2da92b5257ebe54c7fdfd2d586620dcf43d18 Mon Sep 17 00:00:00 2001
From: David Eliahu <deliahu@users.noreply.github.com>
Date: Mon, 21 Jun 2021 16:17:30 -0700
Subject: [PATCH 22/28] Update messages

---
 dev/registry.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dev/registry.sh b/dev/registry.sh
index d3bbc3ec83..ef027233bb 100755
--- a/dev/registry.sh
+++ b/dev/registry.sh
@@ -129,9 +129,9 @@ function build_and_push() {
 
   tag=$CORTEX_VERSION
   if [ "$include_arm64_arch" = "true" ]; then
-    blue_echo "Building and pushing $image:$tag with amd64/arm64 arch support..."
+    blue_echo "Building and pushing $image:$tag (amd64 and arm64)..."
   else
-    blue_echo "Building and pushing $image:$tag with amd64 arch support only..."
+    blue_echo "Building and pushing $image:$tag (amd64)..."
   fi
 
   platforms="linux/amd64"
@@ -142,9 +142,9 @@ function build_and_push() {
   docker buildx build $ROOT -f $dir/Dockerfile -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag --platform $platforms --push
 
   if [ "$include_arm64_arch" = "true" ]; then
-    green_echo "Built and pushed $image:$tag with amd64/arm64 arch support..."
+    green_echo "Built and pushed $image:$tag (amd64 and arm64)..."
   else
-    green_echo "Built and pushed $image:$tag with amd64 arch support only..."
+    green_echo "Built and pushed $image:$tag (amd64)..."
   fi
 
   if [[ " $images_that_can_run_locally " =~ " $image " ]] && [[ "$include_arm64_arch" == "false" ]]; then

From 2f41154764a227ad81a3fa636a5af09fd651f4ed Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 02:35:18 +0300
Subject: [PATCH 23/28] Go get vpc cni 1.8.0

---
 dev/versions.md |  4 ++--
 go.mod          |  2 +-
 go.sum          | 49 ++++++++++++++++++++++++-------------------------
 3 files changed, 27 insertions(+), 28 deletions(-)

diff --git a/dev/versions.md b/dev/versions.md
index 01173ef192..182a2232ed 100644
--- a/dev/versions.md
+++ b/dev/versions.md
@@ -25,8 +25,8 @@
    1. If there are new instance types, check if any changes need to be made to `servicequotas.go` or `validateInstanceType()`.
 
 ```bash
-PREV_RELEASE=1.7.5
-NEW_RELEASE=1.7.10
+PREV_RELEASE=1.7.10
+NEW_RELEASE=1.8.0
 wget -q -O cni_supported_instances_prev.txt https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${PREV_RELEASE}/pkg/awsutils/vpc_ip_resource_limit.go; wget -q -O cni_supported_instances_new.txt https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${NEW_RELEASE}/pkg/awsutils/vpc_ip_resource_limit.go; git diff --no-index cni_supported_instances_prev.txt cni_supported_instances_new.txt; rm -rf cni_supported_instances_prev.txt; rm -rf cni_supported_instances_new.txt
 ```
 
diff --git a/go.mod b/go.mod
index 596c2b9f32..7786a3a510 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ require (
 	cloud.google.com/go v0.73.0 // indirect
 	github.com/DataDog/datadog-go v4.7.0+incompatible
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
-	github.com/aws/amazon-vpc-cni-k8s v1.7.10
+	github.com/aws/amazon-vpc-cni-k8s v1.8.0
 	github.com/aws/aws-sdk-go v1.37.23
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
 	github.com/containerd/containerd v1.4.3 // indirect
diff --git a/go.sum b/go.sum
index db7dcd0239..bf1e4681d0 100644
--- a/go.sum
+++ b/go.sum
@@ -94,8 +94,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/amazon-vpc-cni-k8s v1.7.10 h1:KW0S2QXUGmaHOwqgsd2QdLJUDrknDq4mRRSEu1zMUTs=
-github.com/aws/amazon-vpc-cni-k8s v1.7.10/go.mod h1:DvKcToLCSKRXnnoIE5u1Bphq97bdCw0KP33JA//JP0I=
+github.com/aws/amazon-vpc-cni-k8s v1.8.0 h1:YVteZl3WkKoe554gkmFVUvR9/CliamTXRSsJaE1neno=
+github.com/aws/amazon-vpc-cni-k8s v1.8.0/go.mod h1:p7GsqsIcTy/IN/zOK2k2GKg1nd1Z/1t0ydTTVY80zV8=
 github.com/aws/aws-sdk-go v1.37.23 h1:bO80NcSmRv52w+GFpBegoLdlP/Z0OwUqQ9bbeCLCy/0=
 github.com/aws/aws-sdk-go v1.37.23/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
@@ -128,7 +128,6 @@ github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5Q
 github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 h1:NmTXa/uVnDyp0TY5MKi197+3HWcnYWfnHGyaFthlnGw=
 github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/plugins v0.9.0/go.mod h1:dbWv4dI0QrBGuVgj+TuVQ6wJRZVOhrCQj91YyC92sxg=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -142,6 +141,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cortexlabs/go-input v0.0.0-20200503032952-8b67a7a7b28d h1:2SmODObcy2ekPA0eLFlR/+Vu5Yo2hoVbNMJ+vWzinpo=
 github.com/cortexlabs/go-input v0.0.0-20200503032952-8b67a7a7b28d/go.mod h1:rxijm42+fHjyPbFGqUTzUUhioEVg5LUi+w53dlY0WUM=
@@ -158,7 +158,6 @@ github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/
 github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
 github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
 github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
-github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -232,6 +231,7 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs=
 github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
 github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4=
 github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU=
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
@@ -339,12 +339,10 @@ github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-jsonnet v0.16.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
-github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
+github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -364,10 +362,9 @@ github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
 github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM=
 github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
@@ -416,7 +413,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc=
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
@@ -433,8 +430,6 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -517,7 +512,6 @@ github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXy
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -557,7 +551,6 @@ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVo
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/runc v1.0.0-rc9 h1:/k06BMULKF5hidyoZymkoDCzdJzltZpz/UU4LguQVtc=
 github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/operator-framework/operator-sdk v0.0.7/go.mod h1:iVyukRkam5JZa8AnjYf+/G3rk7JI1+M6GsU0sq0B9NA=
 github.com/ory/dockertest/v3 v3.6.5 h1:mhNKFeVEHuvaYW+/u+59mLzM/6XXGjpaet/yApgv+yc=
 github.com/ory/dockertest/v3 v3.6.5/go.mod h1:iYKQSRlYrt/2s5fJWYdB98kCQG6g/LjBMvzEYii63vg=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -572,12 +565,10 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
-github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -588,16 +579,15 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
@@ -650,7 +640,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
-github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -698,6 +687,7 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
@@ -941,6 +931,7 @@ golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -976,6 +967,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u
 golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210113180300-f96436850f18/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -983,6 +975,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU=
 gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k=
 gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -1137,26 +1130,31 @@ istio.io/client-go v0.0.0-20200807182027-d287a5abb594 h1:Rdokv2iBpeIgsER6tas937Q
 istio.io/client-go v0.0.0-20200807182027-d287a5abb594/go.mod h1:q726ySPDHo2Ob6LHZ9oXWr8nNf8Sm0uIzYQ7B4XjnvA=
 istio.io/gogo-genproto v0.0.0-20190930162913-45029607206a h1:w7zILua2dnYo9CxImhpNW4NE/8ZxEoc/wfBfHrhUhrE=
 istio.io/gogo-genproto v0.0.0-20190930162913-45029607206a/go.mod h1:OzpAts7jljZceG4Vqi5/zXy/pOg1b209T3jb7Nv5wIs=
-k8s.io/api v0.0.0-20180712090710-2d6f90ab1293/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
 k8s.io/api v0.18.1/go.mod h1:3My4jorQWzSs5a+l7Ge6JBbIxChLnY8HnuT58ZWolss=
+k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI=
 k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms=
 k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI=
+k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M=
 k8s.io/apiextensions-apiserver v0.19.2 h1:oG84UwiDsVDu7dlsGQs5GySmQHCzMhknfhFExJMz9tA=
 k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg=
-k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
 k8s.io/apimachinery v0.18.1/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA=
+k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
 k8s.io/apimachinery v0.19.2 h1:5Gy9vQpAGTKHPVOh5c4plE274X8D/6cuEiTO2zve7tc=
 k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
+k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg=
 k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA=
-k8s.io/client-go v0.0.0-20180806134042-1f13a808da65/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
 k8s.io/client-go v0.18.1/go.mod h1:iCikYRiXOj/yRRFE/aWqrpPtDt4P2JVWhtHkmESTcfY=
+k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q=
 k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc=
 k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA=
+k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
 k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk=
+k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14=
 k8s.io/component-base v0.19.2 h1:jW5Y9RcZTb79liEhW3XDVTW7MuvEGP0tQZnfSX6/+gs=
 k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo=
 k8s.io/cri-api v0.0.0-20191107035106-03d130a7dc28/go.mod h1:9a7E6pmKLfuq8ZL31k2PDpgvSdyZfUOH9czlEmpblFk=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
@@ -1166,22 +1164,23 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/kube-openapi v0.0.0-20190510232812-a01b7d5d6c22/go.mod h1:iU+ZGYsNlvU9XKUSso6SQfKTCCw7lFduMZy26Mgr2Fw=
 k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
+k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
 k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ=
 k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
 k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g=
 k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0=
+sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY=
 sigs.k8s.io/controller-runtime v0.7.2 h1:gD2JZp0bBLLuvSRYVNvox+bRCz1UUUxKDjPUCb56Ukk=
 sigs.k8s.io/controller-runtime v0.7.2/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU=
-sigs.k8s.io/structured-merge-diff v0.0.0-20190426204423-ea680f03cc65 h1:xJNnO2qzHtgVCSPoGkkltSpyEX7D7IJw1TmbE3G/7lY=
-sigs.k8s.io/structured-merge-diff v0.0.0-20190426204423-ea680f03cc65/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA=

From e9eb6437d9ae5ed924f2a769da4e5333c3d3ca6c Mon Sep 17 00:00:00 2001
From: David Eliahu <deliahu@users.noreply.github.com>
Date: Mon, 21 Jun 2021 16:41:44 -0700
Subject: [PATCH 24/28] Block mac instances

---
 dev/versions.md                           |  2 +-
 pkg/lib/aws/ec2.go                        | 13 +++++++++++++
 pkg/types/clusterconfig/cluster_config.go |  8 ++++++++
 pkg/types/clusterconfig/errors.go         |  8 ++++++++
 4 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/dev/versions.md b/dev/versions.md
index 182a2232ed..d355c6741c 100644
--- a/dev/versions.md
+++ b/dev/versions.md
@@ -96,7 +96,7 @@ see https://github.com/moby/moby/issues/39302#issuecomment-639687466_
 1. `rm -rf go.mod go.sum && go mod init && go clean -modcache`
 1. `go get k8s.io/client-go@v0.17.6 && go get k8s.io/apimachinery@v0.17.6 && go get k8s.io/api@v0.17.6`
 1. `go get istio.io/client-go@1.7.3 && go get istio.io/api@1.7.3`
-1. `go get github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils@v1.7.10`
+1. `go get github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils@v1.8.0`
 1. `go get github.com/cortexlabs/yaml@581aea36a2e4db10f8696587e48cac5248d64f4d`
 1. `go get github.com/cortexlabs/go-input@8b67a7a7b28d1c45f5c588171b3b50148462b247`
 1. `echo -e '\nreplace github.com/docker/docker => github.com/docker/engine v19.03.12' >> go.mod`
diff --git a/pkg/lib/aws/ec2.go b/pkg/lib/aws/ec2.go
index 7270818204..9a0e0f8e40 100644
--- a/pkg/lib/aws/ec2.go
+++ b/pkg/lib/aws/ec2.go
@@ -120,6 +120,19 @@ func IsAMDGPUInstance(instanceType string) (bool, error) {
 	return false, nil
 }
 
+func IsMacInstance(instanceType string) (bool, error) {
+	parsedType, err := ParseInstanceType(instanceType)
+	if err != nil {
+		return false, err
+	}
+
+	if parsedType.Family == "mac" {
+		return true, nil
+	}
+
+	return false, nil
+}
+
 func (c *Client) SpotInstancePrice(instanceType string) (float64, error) {
 	result, err := c.EC2().DescribeSpotPriceHistory(&ec2.DescribeSpotPriceHistoryInput{
 		InstanceTypes:       []*string{aws.String(instanceType)},
diff --git a/pkg/types/clusterconfig/cluster_config.go b/pkg/types/clusterconfig/cluster_config.go
index 01cd3f5caa..fcc6178c2b 100644
--- a/pkg/types/clusterconfig/cluster_config.go
+++ b/pkg/types/clusterconfig/cluster_config.go
@@ -1451,6 +1451,14 @@ func validateInstanceType(instanceType string) (string, error) {
 		return "", ErrorAMDGPUInstancesNotSupported(instanceType)
 	}
 
+	isMac, err := aws.IsMacInstance(instanceType)
+	if err != nil {
+		return "", err
+	}
+	if isMac {
+		return "", ErrorMacInstancesNotSupported(instanceType)
+	}
+
 	if _, ok := awsutils.InstanceNetworkingLimits[instanceType]; !ok {
 		return "", ErrorInstanceTypeNotSupportedByCortex(instanceType)
 	}
diff --git a/pkg/types/clusterconfig/errors.go b/pkg/types/clusterconfig/errors.go
index f719e7eff4..b4d15f50a6 100644
--- a/pkg/types/clusterconfig/errors.go
+++ b/pkg/types/clusterconfig/errors.go
@@ -47,6 +47,7 @@ const (
 	ErrSpotPriceGreaterThanMaxPrice           = "clusterconfig.spot_price_greater_than_max_price"
 	ErrInstanceTypeNotSupportedByCortex       = "clusterconfig.instance_type_not_supported_by_cortex"
 	ErrAMDGPUInstancesNotSupported            = "clusterconfig.amd_gpu_instances_not_supported"
+	ErrMacInstancesNotSupported               = "clusterconfig.mac_instances_not_supported"
 	ErrAtLeastOneInstanceDistribution         = "clusterconfig.at_least_one_instance_distribution"
 	ErrNoCompatibleSpotInstanceFound          = "clusterconfig.no_compatible_spot_instance_found"
 	ErrConfiguredWhenSpotIsNotEnabled         = "clusterconfig.configured_when_spot_is_not_enabled"
@@ -209,6 +210,13 @@ func ErrorAMDGPUInstancesNotSupported(instanceType string) error {
 	})
 }
 
+func ErrorMacInstancesNotSupported(instanceType string) error {
+	return errors.WithStack(&errors.Error{
+		Kind:    ErrMacInstancesNotSupported,
+		Message: fmt.Sprintf("mac instances (including %s) are not supported by cortex", instanceType),
+	})
+}
+
 func ErrorConfiguredWhenSpotIsNotEnabled(configKey string) error {
 	return errors.WithStack(&errors.Error{
 		Kind:    ErrConfiguredWhenSpotIsNotEnabled,

From 036a9b7e34d7237962f498ee84be7be3f14154c1 Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 02:44:42 +0300
Subject: [PATCH 25/28] Add back the Arm validator utility function

---
 pkg/lib/aws/ec2.go | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/pkg/lib/aws/ec2.go b/pkg/lib/aws/ec2.go
index 9a0e0f8e40..338a719c91 100644
--- a/pkg/lib/aws/ec2.go
+++ b/pkg/lib/aws/ec2.go
@@ -103,6 +103,23 @@ func ParseInstanceType(instanceType string) (ParsedInstanceType, error) {
 	}, nil
 }
 
+func IsARMInstance(instanceType string) (bool, error) {
+	parsedType, err := ParseInstanceType(instanceType)
+	if err != nil {
+		return false, err
+	}
+
+	if parsedType.Family == "a" {
+		return true, nil
+	}
+
+	if parsedType.Capabilities.Has("g") {
+		return true, nil
+	}
+
+	return false, nil
+}
+
 func IsAMDGPUInstance(instanceType string) (bool, error) {
 	parsedType, err := ParseInstanceType(instanceType)
 	if err != nil {

From fc5b8d8c6325da465a92366e7ec33d0ffe65bf1a Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 03:03:04 +0300
Subject: [PATCH 26/28] Nits and fixes

---
 test/e2e/e2e/tests.py               | 13 +++++++++----
 test/e2e/e2e/utils.py               | 17 ++++++++++++++++-
 test/e2e/tests/aws/test_realtime.py |  3 ++-
 3 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py
index 0092960f6a..4e7159f3ce 100644
--- a/test/e2e/e2e/tests.py
+++ b/test/e2e/e2e/tests.py
@@ -40,7 +40,8 @@
     wait_on_event,
     wait_on_futures,
     endpoint_ready,
-    request_prediction,
+    post_request,
+    get_request,
     job_done,
     jobs_done,
     request_batch_prediction,
@@ -68,6 +69,7 @@ def test_realtime_api(
     timeout: int = None,
     api_config_name: str = "cortex_cpu.yaml",
     extra_path: str = "",
+    method: str = "POST",
 ):
     api_dir = TEST_APIS_DIR / api
     with open(str(api_dir / api_config_name)) as f:
@@ -89,7 +91,10 @@ def test_realtime_api(
 
         with open(str(api_dir / "sample.json")) as f:
             payload = json.load(f)
-        response = request_prediction(client, api_name, payload, extra_path)
+        if method == "POST":
+            response = post_request(client, api_name, payload, extra_path)
+        else:
+            response = get_request(client, api_name, payload, extra_path)
 
         assert (
             response.status_code == HTTPStatus.OK
@@ -227,7 +232,7 @@ def test_async_api(
         with open(str(api_dir / "sample.json")) as f:
             payload = json.load(f)
 
-        response = request_prediction(client, api_name, payload)
+        response = post_request(client, api_name, payload)
 
         assert (
             response.status_code == HTTPStatus.OK
@@ -854,7 +859,7 @@ def test_long_running_realtime(
         counter = 0
         start_time = time.time()
         while time.time() - start_time <= time_to_run:
-            response = request_prediction(client, api_name, payload)
+            response = post_request(client, api_name, payload)
 
             assert (
                 response.status_code == HTTPStatus.OK
diff --git a/test/e2e/e2e/utils.py b/test/e2e/e2e/utils.py
index c15e2e0a77..52020a037c 100644
--- a/test/e2e/e2e/utils.py
+++ b/test/e2e/e2e/utils.py
@@ -126,7 +126,7 @@ def _runnable(job_id):
     return True
 
 
-def request_prediction(
+def post_request(
     client: cx.Client,
     api_name: str,
     payload: Union[List, Dict],
@@ -141,6 +141,21 @@ def request_prediction(
     return response
 
 
+def get_request(
+    client: cx.Client,
+    api_name: str,
+    payload: Union[List, Dict],
+    extra_path: Optional[str] = None,
+) -> requests.Response:
+    api_info = client.get_api(api_name)
+    endpoint = api_info["endpoint"]
+    if extra_path and extra_path != "":
+        endpoint = os.path.join(endpoint, extra_path)
+    response = requests.get(endpoint, json=payload)
+
+    return response
+
+
 def retrieve_async_result(client: cx.Client, api_name: str, request_id: str) -> requests.Response:
     api_info = client.get_api(api_name)
     response = requests.get(f"{api_info['endpoint']}/{request_id}")
diff --git a/test/e2e/tests/aws/test_realtime.py b/test/e2e/tests/aws/test_realtime.py
index 62b1e4638c..738f69869e 100644
--- a/test/e2e/tests/aws/test_realtime.py
+++ b/test/e2e/tests/aws/test_realtime.py
@@ -66,7 +66,7 @@ def test_realtime_api(printer: Callable, config: Dict, client: cx.Client, api: D
 
 @pytest.mark.usefixtures("client")
 @pytest.mark.parametrize("api", TEST_APIS_ARM)
-def test_realtime_api(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]):
+def test_realtime_api_arm(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]):
 
     printer(f"testing {api['name']}")
     e2e.tests.test_realtime_api(
@@ -76,6 +76,7 @@ def test_realtime_api(printer: Callable, config: Dict, client: cx.Client, api: D
         timeout=config["global"]["realtime_deploy_timeout"],
         api_config_name="cortex_cpu_arm64.yaml",
         extra_path=api["extra_path"],
+        method="GET",
     )
 
 

From 7e775f716f690529b475c93d722acbca13188657 Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 03:03:45 +0300
Subject: [PATCH 27/28] Fix arm hello-world test api

---
 test/apis/realtime/hello-world/cortex_cpu_arm64.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/apis/realtime/hello-world/cortex_cpu_arm64.yaml b/test/apis/realtime/hello-world/cortex_cpu_arm64.yaml
index 7c953a8f13..2aba4c6c0b 100644
--- a/test/apis/realtime/hello-world/cortex_cpu_arm64.yaml
+++ b/test/apis/realtime/hello-world/cortex_cpu_arm64.yaml
@@ -8,7 +8,7 @@
       image: infrastructureascode/hello-world
       readiness_probe:
         http_get:
-          path: "/healthz"
+          path: "/health"
           port: 8080
       compute:
         cpu: 200m

From dcca924cfc67c4d0b74445a683f5e21f1b680e5b Mon Sep 17 00:00:00 2001
From: Robert Lucian Chiriac <robert.lucian.chiriac@gmail.com>
Date: Tue, 22 Jun 2021 03:04:02 +0300
Subject: [PATCH 28/28] Fix cluster configure cmd

---
 cli/cmd/cluster.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go
index 0a6b48db53..26b301341c 100644
--- a/cli/cmd/cluster.go
+++ b/cli/cmd/cluster.go
@@ -319,7 +319,7 @@ var _clusterConfigureCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		accessConfig, err := getClusterAccessConfigWithCache(false)
+		accessConfig, err := getNewClusterAccessConfig(clusterConfigFile)
 		if err != nil {
 			exit.Error(err)
 		}