Merge pull request #2135 from ConnorJC3/outpost-makefile-support

Add support for outpost nodegroups to `make cluster/create`

docs/makefile.md

@@ -55,6 +55,8 @@ Creates a cluster for running E2E tests against. There are many parameters that
 - `WINDOWS`: Whether or not to create a Windows node group for the cluster (`eksctl` clusters only) - defaults to `false`
 - `AWS_REGION`: Which region to create the cluster in - defaults to `us-west-2`
 - `AWS_AVAILABILITY_ZONES`: Which AZs to create nodes for the cluster in - defaults to `us-west-2a,us-west-2b,us-west-2c`
+- `OUTPOST_ARN`: If set, create an additional nodegroup on an [outpost](https://aws.amazon.com/outposts/) (`eksctl clusters only)
+- `OUTPOST_INSTANCE_TYPE`: The instance type to use for the outpost nodegroup (only used when `OUTPOST_ARN` is non-empty) - defaults to `INSTANCE_TYPE`
 
 #### Example: Create a default (`kops`) cluster
 
@@ -91,6 +93,15 @@ export CLUSTER_TYPE="eksctl"
 make cluster/create
 ```
 
+#### Example: Create a cluster with an outpost nodegroup
+
+```bash
+export CLUSTER_TYPE="eksctl"
+export OUTPOST_ARN="arn:aws:outposts:us-east-1:123456789012:outpost/op-0f39f7c0af9b166a3"
+export OUTPOST_INSTANCE_TYPE=c5.xlarge
+make cluster/create
+```
+
 ### `make cluster/image`
 
 Builds an image for use in the E2E tests. This will automatically build the most appropriate image (for example, skipping Windows builds unless `WINDOWS` is set to `true`).

hack/e2e/config.sh

@@ -29,6 +29,8 @@ NODE_COUNT=${NODE_COUNT:-3}
 INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large}
 WINDOWS=${WINDOWS:-"false"}
 WINDOWS_HOSTPROCESS=${WINDOWS_HOSTPROCESS:-"false"}
+OUTPOST_ARN=${OUTPOST_ARN:-}
+OUTPOST_INSTANCE_TYPE=${OUTPOST_INSTANCE_TYPE:-${INSTANCE_TYPE}}
 
 # kops: must include patch version (e.g. 1.19.1)
 # eksctl: mustn't include patch version (e.g. 1.19)

hack/e2e/create-cluster.sh

@@ -68,7 +68,9 @@ elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
     "$KUBECONFIG" \
     "$WINDOWS" \
     "${BASE_DIR}/eksctl/vpc-resource-controller-configmap.yaml" \
-    "${BASE_DIR}/eksctl/cluster.yaml"
+    "${BASE_DIR}/eksctl/cluster.yaml" \
+    "${OUTPOST_ARN}" \
+    "${OUTPOST_INSTANCE_TYPE}"
 else
   echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2
   exit 1

hack/e2e/eksctl/cluster.yaml

@@ -45,3 +45,18 @@ managedNodeGroups:
     ssh:
       allow: false
 {{- end }}
+# env.Getenv will return empty for missing variables, unlike .Env
+# https://docs.gomplate.ca/syntax/#env
+{{- if env.Getenv "OUTPOST_ARN" }}
+# Outpost nodegroups cannot be managed
+nodeGroups:
+  - name: ng-outpost
+    amiFamily: AmazonLinux2
+    desiredCapacity: 3
+    # Driver needs access to IMDS on outposts for correct metadata
+    disablePodIMDS: false
+    instanceType: {{ .Env.OUTPOST_INSTANCE_TYPE }}
+    ssh:
+      allow: false
+    outpostARN: {{ .Env.OUTPOST_ARN }}
+{{- end }}

hack/e2e/eksctl/eksctl.sh

@@ -32,6 +32,8 @@ function eksctl_create_cluster() {
   WINDOWS=${10}
   VPC_CONFIGMAP_FILE=${11}
   TEMPLATE_FILE=${12}
+  OUTPOST_ARN=${13}
+  OUTPOST_INSTANCE_TYPE=${14}
 
   CLUSTER_NAME="${CLUSTER_NAME//./-}"
 
@@ -42,6 +44,7 @@ function eksctl_create_cluster() {
     ZONES="${ZONES}" \
     INSTANCE_TYPE="${INSTANCE_TYPE}" \
     WINDOWS="${WINDOWS}" \
+    OUTPOST_ARN="" \
     ${GOMPLATE_BIN} -f "${TEMPLATE_FILE}" -o "${CLUSTER_FILE}"
 
   if eksctl_cluster_exists "${CLUSTER_NAME}" "${EKSCTL_BIN}"; then
@@ -52,10 +55,28 @@ function eksctl_create_cluster() {
     ${EKSCTL_BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}"
   fi
 
-  if [[ "$WINDOWS" == true ]]; then
+  if [[ "${WINDOWS}" == true ]]; then
     loudecho "Applying VPC ConfigMap (Windows only)"
     kubectl apply --kubeconfig "${KUBECONFIG}" -f "${VPC_CONFIGMAP_FILE}"
   fi
+
+  if [[ -n "${OUTPOST_ARN}" ]]; then
+    loudecho "Creating outpost nodegroup"
+    # Outpost nodegroup cannot be created during initial cluster create
+    # Thus, re-render the cluster with the outpost nodegroup included,
+    # and then add new nodegroup to the existing cluster
+    # https://eksctl.io/usage/outposts/#extending-existing-clusters-to-aws-outposts
+    CLUSTER_NAME="${CLUSTER_NAME}" \
+      REGION="${REGION}" \
+      K8S_VERSION="${K8S_VERSION}" \
+      ZONES="${ZONES}" \
+      INSTANCE_TYPE="${INSTANCE_TYPE}" \
+      WINDOWS="${WINDOWS}" \
+      OUTPOST_ARN="${OUTPOST_ARN}" \
+      OUTPOST_INSTANCE_TYPE="${OUTPOST_INSTANCE_TYPE}" \
+      ${GOMPLATE_BIN} -f "${TEMPLATE_FILE}" -o "${CLUSTER_FILE}"
+    ${EKSCTL_BIN} create nodegroup -f "${CLUSTER_FILE}"
+  fi
 }
 
 function eksctl_cluster_exists() {