Reduce Startup Latency by moving the init container to a regular container

charts/aws-vpc-cni/templates/daemonset.yaml

@@ -38,29 +38,23 @@ spec:
       priorityClassName: "{{ .Values.priorityClassName }}"
       serviceAccountName: {{ template "aws-vpc-cni.serviceAccountName" . }}
       hostNetwork: true
-      initContainers:
-      - name: aws-vpc-cni-init
-        image: "{{- if .Values.init.image.override }}{{- .Values.init.image.override }}{{- else }}{{- .Values.init.image.account }}.dkr.ecr.{{- .Values.init.image.region }}.{{- .Values.init.image.domain }}/amazon-k8s-cni-init:{{- .Values.init.image.tag }}{{- end}}"
-        env:
+      containers:
+        - name: aws-vpc-cni-init
+          image: "{{- if .Values.init.image.override }}{{- .Values.init.image.override }}{{- else }}{{- .Values.init.image.account }}.dkr.ecr.{{- .Values.init.image.region }}.{{- .Values.init.image.domain }}/amazon-k8s-cni-init:{{- .Values.init.image.tag }}{{- end}}"
+          env:
 {{- range $key, $value := .Values.init.env }}
-          - name: {{ $key }}
-            value: {{ $value | quote }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
 {{- end }}
-        securityContext:
-          {{- toYaml .Values.init.securityContext | nindent 12 }}
-        volumeMounts:
+          resources:
+            {{- toYaml .Values.init.resources | nindent 12 }}
+          securityContext:
+            {{- toYaml .Values.init.securityContext | nindent 12 }}
+          volumeMounts:
           - mountPath: /host/opt/cni/bin
             name: cni-bin-dir
-      terminationGracePeriodSeconds: 10
-      tolerations:
-        - operator: Exists
-    {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-    {{- end }}
-      securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      containers:
+          - name: vpc-cni-init-dir
+            mountPath: /vpc-cni-init
         - name: aws-node
           image: "{{- if .Values.image.override }}{{- .Values.image.override }}{{- else }}{{- .Values.image.account }}.dkr.ecr.{{- .Values.image.region }}.{{- .Values.image.domain }}/amazon-k8s-cni:{{- .Values.image.tag }}{{- end}}"
           ports:
@@ -103,6 +97,8 @@ spec:
             name: run-dir
           - mountPath: /run/xtables.lock
             name: xtables-lock
+          - name: vpc-cni-init-dir
+            mountPath: /vpc-cni-init
           {{- with .Values.extraVolumeMounts  }}
           {{- toYaml .| nindent 10 }}
           {{- end }}
@@ -129,6 +125,9 @@ spec:
       - name: xtables-lock
         hostPath:
           path: /run/xtables.lock
+      - name: vpc-cni-init-dir
+        emptyDir:
+          sizeLimit: 1Ki
       {{- with .Values.extraVolumes  }}
       {{- toYaml .| nindent 6 }}
       {{- end }}
@@ -140,6 +139,15 @@ spec:
       affinity:
         {{- toYaml . | nindent 8 }}
     {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+    {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      terminationGracePeriodSeconds: 10
+      tolerations:
+        - operator: Exists
     {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}

charts/aws-vpc-cni/values.yaml

@@ -20,6 +20,9 @@ init:
     ENABLE_IPv6: "false"
   securityContext:
     privileged: true
+  resources:
+    requests:
+      cpu: 5m
 
 image:
   region: us-west-2

cmd/aws-vpc-cni-init/main.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"os"
+	"time"
 
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/procsyswrapper"
 	"github.com/aws/amazon-vpc-cni-k8s/utils/cp"
@@ -179,17 +180,17 @@ func _main() int {
 		return 1
 	}
 
-	// TODO: In order to speed up pod launch time, VPC CNI init container is not a Kubernetes init container.
+	// In order to speed up pod launch time, VPC CNI init container is not a Kubernetes init container.
 	// The VPC CNI container blocks on the existence of vpcCniInitDonePath
-	//err = cp.TouchFile(vpcCniInitDonePath)
-	//if err != nil {
-	//	log.WithError(err).Errorf("Failed to set VPC CNI init done")
-	//	return 1
-	//}
+	err = cp.TouchFile(vpcCniInitDonePath)
+	if err != nil {
+		log.WithError(err).Errorf("Failed to set VPC CNI init done")
+		return 1
+	}
 
 	log.Infof("CNI init container done")
 
-	// TODO: Since VPC CNI init container is a real container, it never exits
-	// time.Sleep(time.Duration(1<<63 - 1))
+	// Since VPC CNI init container is a real container, it never exits
+	time.Sleep(time.Duration(1<<63 - 1))
 	return 0
 }

cmd/aws-vpc-cni/main.go

@@ -390,10 +390,10 @@ func _main() int {
 	}
 
 	// Wait for init container to complete
-	//if err := waitForInit(); err != nil {
-	//	log.WithError(err).Errorf("Init container failed to complete")
-	//	return 1
-	//}
+	if err := waitForInit(); err != nil {
+		log.WithError(err).Errorf("Init container failed to complete")
+		return 1
+	}
 
 	log.Infof("Copying config file... ")
 	err = generateJSON(defaultAWSconflistFile, tmpAWSconflistFile)

config/master/aws-k8s-cni-cn.yaml

@@ -119,25 +119,24 @@ spec:
       priorityClassName: "system-node-critical"
       serviceAccountName: aws-node
       hostNetwork: true
-      initContainers:
-      - name: aws-vpc-cni-init
-        image: "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni-init:v1.12.0"
-        env:
-          - name: DISABLE_TCP_EARLY_DEMUX
-            value: "false"
-          - name: ENABLE_IPv6
-            value: "false"
-        securityContext:
+      containers:
+        - name: aws-vpc-cni-init
+          image: "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni-init:v1.12.0"
+          env:
+            - name: DISABLE_TCP_EARLY_DEMUX
+              value: "false"
+            - name: ENABLE_IPv6
+              value: "false"
+          resources:
+            requests:
+              cpu: 5m
+          securityContext:
             privileged: true
-        volumeMounts:
+          volumeMounts:
           - mountPath: /host/opt/cni/bin
             name: cni-bin-dir
-      terminationGracePeriodSeconds: 10
-      tolerations:
-        - operator: Exists
-      securityContext:
-        {}
-      containers:
+          - name: vpc-cni-init-dir
+            mountPath: /vpc-cni-init
         - name: aws-node
           image: "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni:v1.12.0"
           ports:
@@ -227,6 +226,8 @@ spec:
             name: run-dir
           - mountPath: /run/xtables.lock
             name: xtables-lock
+          - name: vpc-cni-init-dir
+            mountPath: /vpc-cni-init
       volumes:
       - name: cni-bin-dir
         hostPath:
@@ -245,6 +246,9 @@ spec:
       - name: xtables-lock
         hostPath:
           path: /run/xtables.lock
+      - name: vpc-cni-init-dir
+        emptyDir:
+          sizeLimit: 1Ki
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -263,3 +267,8 @@ spec:
                 operator: NotIn
                 values:
                 - fargate
+      securityContext:
+        {}
+      terminationGracePeriodSeconds: 10
+      tolerations:
+        - operator: Exists

config/master/aws-k8s-cni-us-gov-east-1.yaml

@@ -119,25 +119,24 @@ spec:
       priorityClassName: "system-node-critical"
       serviceAccountName: aws-node
       hostNetwork: true
-      initContainers:
-      - name: aws-vpc-cni-init
-        image: "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/amazon-k8s-cni-init:v1.12.0"
-        env:
-          - name: DISABLE_TCP_EARLY_DEMUX
-            value: "false"
-          - name: ENABLE_IPv6
-            value: "false"
-        securityContext:
+      containers:
+        - name: aws-vpc-cni-init
+          image: "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/amazon-k8s-cni-init:v1.12.0"
+          env:
+            - name: DISABLE_TCP_EARLY_DEMUX
+              value: "false"
+            - name: ENABLE_IPv6
+              value: "false"
+          resources:
+            requests:
+              cpu: 5m
+          securityContext:
             privileged: true
-        volumeMounts:
+          volumeMounts:
           - mountPath: /host/opt/cni/bin
             name: cni-bin-dir
-      terminationGracePeriodSeconds: 10
-      tolerations:
-        - operator: Exists
-      securityContext:
-        {}
-      containers:
+          - name: vpc-cni-init-dir
+            mountPath: /vpc-cni-init
         - name: aws-node
           image: "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/amazon-k8s-cni:v1.12.0"
           ports:
@@ -227,6 +226,8 @@ spec:
             name: run-dir
           - mountPath: /run/xtables.lock
             name: xtables-lock
+          - name: vpc-cni-init-dir
+            mountPath: /vpc-cni-init
       volumes:
       - name: cni-bin-dir
         hostPath:
@@ -245,6 +246,9 @@ spec:
       - name: xtables-lock
         hostPath:
           path: /run/xtables.lock
+      - name: vpc-cni-init-dir
+        emptyDir:
+          sizeLimit: 1Ki
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -263,3 +267,8 @@ spec:
                 operator: NotIn
                 values:
                 - fargate
+      securityContext:
+        {}
+      terminationGracePeriodSeconds: 10
+      tolerations:
+        - operator: Exists

config/master/aws-k8s-cni-us-gov-west-1.yaml

@@ -119,25 +119,24 @@ spec:
       priorityClassName: "system-node-critical"
       serviceAccountName: aws-node
       hostNetwork: true
-      initContainers:
-      - name: aws-vpc-cni-init
-        image: "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/amazon-k8s-cni-init:v1.12.0"
-        env:
-          - name: DISABLE_TCP_EARLY_DEMUX
-            value: "false"
-          - name: ENABLE_IPv6
-            value: "false"
-        securityContext:
+      containers:
+        - name: aws-vpc-cni-init
+          image: "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/amazon-k8s-cni-init:v1.12.0"
+          env:
+            - name: DISABLE_TCP_EARLY_DEMUX
+              value: "false"
+            - name: ENABLE_IPv6
+              value: "false"
+          resources:
+            requests:
+              cpu: 5m
+          securityContext:
             privileged: true
-        volumeMounts:
+          volumeMounts:
           - mountPath: /host/opt/cni/bin
             name: cni-bin-dir
-      terminationGracePeriodSeconds: 10
-      tolerations:
-        - operator: Exists
-      securityContext:
-        {}
-      containers:
+          - name: vpc-cni-init-dir
+            mountPath: /vpc-cni-init
         - name: aws-node
           image: "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/amazon-k8s-cni:v1.12.0"
           ports:
@@ -227,6 +226,8 @@ spec:
             name: run-dir
           - mountPath: /run/xtables.lock
             name: xtables-lock
+          - name: vpc-cni-init-dir
+            mountPath: /vpc-cni-init
       volumes:
       - name: cni-bin-dir
         hostPath:
@@ -245,6 +246,9 @@ spec:
       - name: xtables-lock
         hostPath:
           path: /run/xtables.lock
+      - name: vpc-cni-init-dir
+        emptyDir:
+          sizeLimit: 1Ki
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -263,3 +267,8 @@ spec:
                 operator: NotIn
                 values:
                 - fargate
+      securityContext:
+        {}
+      terminationGracePeriodSeconds: 10
+      tolerations:
+        - operator: Exists