Merge pull request #631 from pulumi/vl/init_container_support

Support for setting init container options and per-pod security groups

CHANGELOG.md

@@ -1,8 +1,13 @@
 ## Unreleased
 
+- Add support for setting the init container image
+  [#631](https://github.com/pulumi/pulumi-eks/pull/631)
+- Add support for setting `DISABLE_TCP_EARLY_DEMUX`
+  [#631](https://github.com/pulumi/pulumi-eks/pull/631)
 - Make getKubeconfig method available to multi-lang
   [#628](https://github.com/pulumi/pulumi-eks/pull/628)
 
+
 ## 0.33.0 (Released August 18, 2021)
 
 - Add `capacityType` and `taints` to `ManagedNodeGroup`

dotnet/Inputs/VpcCniOptionsArgs.cs

@@ -42,7 +42,13 @@ public sealed class VpcCniOptionsArgs : Pulumi.ResourceArgs
         public Input<bool>? CustomNetworkConfig { get; set; }
 
         /// <summary>
-        /// Specifies whether to allow IPAMD to add the `vpc.amazonaws.com/has-trunk-attached` label tothe node if the instance has capacity to attach an additional ENI. Default is `false`.
+        /// Allows the kubelet's liveness and readiness probes to connect via TCP when pod ENI is enabled. This will slightly increase local TCP connection latency.
+        /// </summary>
+        [Input("disableTcpEarlyDemux")]
+        public Input<bool>? DisableTcpEarlyDemux { get; set; }
+
+        /// <summary>
+        /// Specifies whether to allow IPAMD to add the `vpc.amazonaws.com/has-trunk-attached` label to the node if the instance has capacity to attach an additional ENI. Default is `false`. If using liveness and readiness probes, you will also need to disable TCP early demux.
         /// </summary>
         [Input("enablePodEni")]
         public Input<bool>? EnablePodEni { get; set; }
@@ -80,6 +86,14 @@ public sealed class VpcCniOptionsArgs : Pulumi.ResourceArgs
         [Input("image")]
         public Input<string>? Image { get; set; }
 
+        /// <summary>
+        /// Specifies the init container image to use in the AWS CNI cluster DaemonSet.
+        /// 
+        /// Defaults to the official AWS CNI init container image in ECR.
+        /// </summary>
+        [Input("initImage")]
+        public Input<string>? InitImage { get; set; }
+
         /// <summary>
         /// Specifies the file path used for logs.
         /// 

dotnet/VpcCni.cs

@@ -86,7 +86,13 @@ public sealed class VpcCniArgs : Pulumi.ResourceArgs
         public Input<bool>? CustomNetworkConfig { get; set; }
 
         /// <summary>
-        /// Specifies whether to allow IPAMD to add the `vpc.amazonaws.com/has-trunk-attached` label tothe node if the instance has capacity to attach an additional ENI. Default is `false`.
+        /// Allows the kubelet's liveness and readiness probes to connect via TCP when pod ENI is enabled. This will slightly increase local TCP connection latency.
+        /// </summary>
+        [Input("disableTcpEarlyDemux")]
+        public Input<bool>? DisableTcpEarlyDemux { get; set; }
+
+        /// <summary>
+        /// Specifies whether to allow IPAMD to add the `vpc.amazonaws.com/has-trunk-attached` label to the node if the instance has capacity to attach an additional ENI. Default is `false`. If using liveness and readiness probes, you will also need to disable TCP early demux.
         /// </summary>
         [Input("enablePodEni")]
         public Input<bool>? EnablePodEni { get; set; }
@@ -124,6 +130,14 @@ public sealed class VpcCniArgs : Pulumi.ResourceArgs
         [Input("image")]
         public Input<string>? Image { get; set; }
 
+        /// <summary>
+        /// Specifies the init container image to use in the AWS CNI cluster DaemonSet.
+        /// 
+        /// Defaults to the official AWS CNI init container image in ECR.
+        /// </summary>
+        [Input("initImage")]
+        public Input<string>? InitImage { get; set; }
+
         /// <summary>
         /// The kubeconfig to use when setting the VPC CNI options.
         /// </summary>

examples/cluster/index.ts

@@ -9,7 +9,7 @@ const cluster1 = new eks.Cluster(`${projectName}-1`);
 
 // Create an EKS cluster with a non-default configuration.
 const vpc = new awsx.ec2.Vpc(`${projectName}-2`, {
-    tags: { "Name": `${projectName}-2` },
+    tags: {"Name": `${projectName}-2`},
 });
 
 const cluster2 = new eks.Cluster(`${projectName}-2`, {
@@ -24,6 +24,11 @@ const cluster2 = new eks.Cluster(`${projectName}-2`, {
         "audit",
         "authenticator",
     ],
+    vpcCniOptions: {
+        image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.9.0",
+        initImage: "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.9.0",
+        disableTcpEarlyDemux: true,
+    },
 });
 
 // Export the clusters' kubeconfig.

examples/examples_nodejs_test.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	appsv1 "k8s.io/api/apps/v1"
 	"os"
 	"os/exec"
 	"path"
@@ -31,16 +32,35 @@ import (
 )
 
 func TestAccCluster(t *testing.T) {
+
 	test := getJSBaseOptions(t).
 		With(integration.ProgramTestOptions{
 			Dir:           path.Join(getCwd(t), "./cluster"),
-			RunUpdateTest: true,
+			RunUpdateTest: false,
 			ExtraRuntimeValidation: func(t *testing.T, info integration.RuntimeValidationStackInfo) {
 				utils.RunEKSSmokeTest(t,
 					info.Deployment.Resources,
 					info.Outputs["kubeconfig1"],
 					info.Outputs["kubeconfig2"],
 				)
+
+				assert.NoError(t, utils.ValidateDaemonSet(t, info.Outputs["kubeconfig2"], "kube-system", "aws-node", func(ds *appsv1.DaemonSet) {
+					for _, c := range ds.Spec.Template.Spec.Containers {
+						assert.Equal(t, "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.9.0", c.Image)
+					}
+
+					for _, ic := range ds.Spec.Template.Spec.InitContainers {
+						assert.Equal(t, "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.9.0", ic.Image)
+						var tcpEarly bool
+						for _, env := range ic.Env {
+							if env.Name == "DISABLE_TCP_EARLY_DEMUX" {
+								tcpEarly = env.Value == "true"
+								assert.Equal(t, env.Value, "true")
+							}
+						}
+						assert.True(t, tcpEarly)
+					}
+				}))
 			},
 		})
 

examples/utils/utils.go

@@ -347,6 +347,48 @@ func IsPodReady(t *testing.T, clientset *kubernetes.Clientset, pod *corev1.Pod)
 	return false
 }
 
+func ValidateDaemonSet(t *testing.T, kubeconfig interface{}, namespace, name string, validateFn func(*appsv1.DaemonSet)) error {
+	clusterMap, err := mapClusterToKubeAccess(kubeconfig)
+	if err != nil {
+		return err
+	}
+	if len(clusterMap) == 0 {
+		return fmt.Errorf("missing cluster kubeconfig")
+	}
+	var clientSet *kubernetes.Clientset
+	for _, kubeAccess := range clusterMap {
+		clientSet = kubeAccess.Clientset
+	}
+
+	var ds *appsv1.DaemonSet
+	ready := false
+	for i := 0; i < MaxRetries; i++ {
+		ds, ready = IsDaemonSetReady(t, clientSet, namespace, name)
+		if ready {
+			break
+		} else {
+			waitFor(t, fmt.Sprintf("Daemonset %s/%s", namespace, name), "ready")
+		}
+	}
+
+	if !ready {
+		fmt.Errorf("daemonset wasn't ready in time")
+	}
+
+	validateFn(ds)
+	return nil
+}
+
+func IsDaemonSetReady(t *testing.T, clientset *kubernetes.Clientset, namespace, name string) (*appsv1.DaemonSet, bool) {
+	// Attempt to retrieve Deployment.
+	o, err := clientset.AppsV1().DaemonSets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil {
+		return nil, false
+	}
+
+	return o, o.Status.DesiredNumberScheduled == o.Status.NumberReady
+}
+
 // IsDeploymentReady attempts to check if the Deployments's status conditions
 // are ready.
 func IsDeploymentReady(t *testing.T, clientset *kubernetes.Clientset, deployment *appsv1.Deployment) bool {

nodejs/eks/cmd/provider/cni.ts

@@ -32,12 +32,14 @@ interface VpcCniInputs {
     logLevel?: string;
     logFile?: string;
     image?: string;
+    initImage?: string;
     vethPrefix?: string;
     eniMtu?: number;
     eniConfigLabelDef?: string;
     pluginLogLevel?: string;
     pluginLogFile?: string;
     enablePodEni?: boolean;
+    disableTcpEarlyDemux?: boolean;
     cniConfigureRpfilter?: boolean;
     cniCustomNetworkCfg?: boolean;
     cniExternalSnat?: boolean;
@@ -50,6 +52,7 @@ function computeVpcCniYaml(cniYamlText: string, args: VpcCniInputs): string {
     // Rewrite the envvars for the CNI daemon set as per the inputs.
     const daemonSet = cniYaml.filter(o => o.kind === "DaemonSet")[0];
     const env = daemonSet.spec.template.spec.containers[0].env;
+    const initEnv = daemonSet.spec.template.spec.initContainers[0].env;
     const securityContext = daemonSet.spec.template.spec.containers[0].securityContext;
     if (args.nodePortSupport) {
         env.push({name: "AWS_VPC_CNI_NODE_PORT_SUPPORT", value: args.nodePortSupport ? "true" : "false"});
@@ -91,6 +94,9 @@ function computeVpcCniYaml(cniYamlText: string, args: VpcCniInputs): string {
     if (args.image) {
         daemonSet.spec.template.spec.containers[0].image = args.image.toString();
     }
+    if (args.initImage) {
+        daemonSet.spec.template.spec.initContainers[0].image = args.initImage.toString();
+    }
     if (args.eniConfigLabelDef) {
         env.push({name: "ENI_CONFIG_LABEL_DEF", value: args.eniConfigLabelDef.toString()});
     }
@@ -109,6 +115,11 @@ function computeVpcCniYaml(cniYamlText: string, args: VpcCniInputs): string {
     } else {
         env.push({name: "ENABLE_POD_ENI", value: "false"});
     }
+    if (args.disableTcpEarlyDemux) {
+        initEnv.push({name: "DISABLE_TCP_EARLY_DEMUX", value: args.disableTcpEarlyDemux ? "true" : "false"});
+    } else {
+        initEnv.push({name: "DISABLE_TCP_EARLY_DEMUX", value: "false"});
+    }
     if (args.cniConfigureRpfilter) {
         env.push({name: "AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER", value: args.cniConfigureRpfilter ? "true" : "false"});
     } else {

nodejs/eks/cni.ts

@@ -79,6 +79,13 @@ export interface VpcCniOptions {
      */
     image?: pulumi.Input<string>;
 
+    /**
+     * Specifies the init container image to use in the AWS CNI cluster DaemonSet.
+     *
+     * Defaults to the official AWS CNI init container image in ECR.
+     */
+    initImage?: pulumi.Input<string>;
+
     /**
      * Specifies the veth prefix used to generate the host-side veth device
      * name for the CNI.
@@ -123,10 +130,20 @@ export interface VpcCniOptions {
      * Specifies whether to allow IPAMD to add the `vpc.amazonaws.com/has-trunk-attached` label to the node if the
      * instance has capacity to attach an additional ENI.
      *
+     * If using liveness and readiness probes, you will also need to disable TCP early demux.
+     *
      * Defaults to "false".
      */
     enablePodEni?: pulumi.Input<boolean>;
 
+    /**
+     * Allows the kubelet's liveness and readiness probes to connect via TCP when pod ENI is enabled.
+     * This will slightly increase local TCP connection latency.
+     *
+     * Defaults to "false".
+     */
+    disableTcpEarlyDemux?: pulumi.Input<boolean>;
+
     /**
      * Specifies whether ipamd should configure rp filter for primary interface.
      *
@@ -194,13 +211,15 @@ export class VpcCni extends pulumi.CustomResource {
             logLevel: args?.logLevel,
             logFile: args?.logFile,
             image: args?.image,
+            initImage: args?.initImage,
             eniConfigLabelDef: args?.eniConfigLabelDef,
             pluginLogLevel: args?.pluginLogLevel,
             pluginLogFile: args?.pluginLogFile,
             enablePodEni: args?.enablePodEni,
+            disableTcpEarlyDemux: args?.disableTcpEarlyDemux,
             cniConfigureRpfilter: args?.cniConfigureRpfilter,
             cniCustomNetworkCfg: args?.cniCustomNetworkCfg,
-            cniExternalSnat: args?.cniCustomNetworkCfg,
+            cniExternalSnat: args?.cniExternalSnat,
             securityContextPrivileged: args?.securityContextPrivileged,
         }, opts);
     }

nodejs/eks/cni/aws-k8s-cni.yaml

@@ -148,9 +148,7 @@ spec:
               name: xtables-lock
       hostNetwork: true
       initContainers:
-        - env:
-            - name: DISABLE_TCP_EARLY_DEMUX
-              value: "false"
+        - env: []
           image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.7.5
           imagePullPolicy: Always
           name: aws-vpc-cni-init

provider/cmd/pulumi-gen-eks/main.go

@@ -1506,6 +1506,11 @@ func vpcCniProperties(kubeconfig bool) map[string]schema.PropertySpec {
 			Description: "Specifies the container image to use in the AWS CNI cluster DaemonSet.\n\n" +
 				"Defaults to the official AWS CNI image in ECR.",
 		},
+		"initImage": {
+			TypeSpec: schema.TypeSpec{Type: "string"},
+			Description: "Specifies the init container image to use in the AWS CNI cluster DaemonSet.\n\n" +
+				"Defaults to the official AWS CNI init container image in ECR.",
+		},
 		"vethPrefix": {
 			TypeSpec: schema.TypeSpec{Type: "string"},
 			Description: "Specifies the veth prefix used to generate the host-side veth device name " +
@@ -1526,8 +1531,14 @@ func vpcCniProperties(kubeconfig bool) map[string]schema.PropertySpec {
 		},
 		"enablePodEni": {
 			TypeSpec: schema.TypeSpec{Type: "boolean"},
-			Description: "Specifies whether to allow IPAMD to add the `vpc.amazonaws.com/has-trunk-attached` label to" +
-				"the node if the instance has capacity to attach an additional ENI. Default is `false`.",
+			Description: "Specifies whether to allow IPAMD to add the `vpc.amazonaws.com/has-trunk-attached` label to " +
+				"the node if the instance has capacity to attach an additional ENI. Default is `false`. " +
+				"If using liveness and readiness probes, you will also need to disable TCP early demux.",
+		},
+		"disableTcpEarlyDemux": {
+			TypeSpec: schema.TypeSpec{Type: "boolean"},
+			Description: "Allows the kubelet's liveness and readiness probes to connect via TCP when pod ENI is enabled." +
+				" This will slightly increase local TCP connection latency.",
 		},
 		"cniConfigureRpfilter": {
 			TypeSpec:    schema.TypeSpec{Type: "boolean"},

provider/cmd/pulumi-resource-eks/schema.json

@@ -469,9 +469,13 @@
                     "type": "boolean",
                     "description": "Specifies that your pods may use subnets and security groups (within the same VPC as your control plane resources) that are independent of your cluster's `resourcesVpcConfig`.\n\nDefaults to false."
                 },
+                "disableTcpEarlyDemux": {
+                    "type": "boolean",
+                    "description": "Allows the kubelet's liveness and readiness probes to connect via TCP when pod ENI is enabled. This will slightly increase local TCP connection latency."
+                },
                 "enablePodEni": {
                     "type": "boolean",
-                    "description": "Specifies whether to allow IPAMD to add the `vpc.amazonaws.com/has-trunk-attached` label tothe node if the instance has capacity to attach an additional ENI. Default is `false`."
+                    "description": "Specifies whether to allow IPAMD to add the `vpc.amazonaws.com/has-trunk-attached` label to the node if the instance has capacity to attach an additional ENI. Default is `false`. If using liveness and readiness probes, you will also need to disable TCP early demux."
                 },
                 "eniConfigLabelDef": {
                     "type": "string",
@@ -489,6 +493,10 @@
                     "type": "string",
                     "description": "Specifies the container image to use in the AWS CNI cluster DaemonSet.\n\nDefaults to the official AWS CNI image in ECR."
                 },
+                "initImage": {
+                    "type": "string",
+                    "description": "Specifies the init container image to use in the AWS CNI cluster DaemonSet.\n\nDefaults to the official AWS CNI init container image in ECR."
+                },
                 "logFile": {
                     "type": "string",
                     "description": "Specifies the file path used for logs.\n\nDefaults to \"stdout\" to emit Pod logs for `kubectl logs`."
@@ -1185,9 +1193,13 @@
                     "type": "boolean",
                     "description": "Specifies that your pods may use subnets and security groups (within the same VPC as your control plane resources) that are independent of your cluster's `resourcesVpcConfig`.\n\nDefaults to false."
                 },
+                "disableTcpEarlyDemux": {
+                    "type": "boolean",
+                    "description": "Allows the kubelet's liveness and readiness probes to connect via TCP when pod ENI is enabled. This will slightly increase local TCP connection latency."
+                },
                 "enablePodEni": {
                     "type": "boolean",
-                    "description": "Specifies whether to allow IPAMD to add the `vpc.amazonaws.com/has-trunk-attached` label tothe node if the instance has capacity to attach an additional ENI. Default is `false`."
+                    "description": "Specifies whether to allow IPAMD to add the `vpc.amazonaws.com/has-trunk-attached` label to the node if the instance has capacity to attach an additional ENI. Default is `false`. If using liveness and readiness probes, you will also need to disable TCP early demux."
                 },
                 "eniConfigLabelDef": {
                     "type": "string",
@@ -1205,6 +1217,10 @@
                     "type": "string",
                     "description": "Specifies the container image to use in the AWS CNI cluster DaemonSet.\n\nDefaults to the official AWS CNI image in ECR."
                 },
+                "initImage": {
+                    "type": "string",
+                    "description": "Specifies the init container image to use in the AWS CNI cluster DaemonSet.\n\nDefaults to the official AWS CNI init container image in ECR."
+                },
                 "kubeconfig": {
                     "$ref": "pulumi.json#/Any",
                     "description": "The kubeconfig to use when setting the VPC CNI options."