From f5b056f6e650defd3e24b1a64f3cee84d4df4d5f Mon Sep 17 00:00:00 2001
From: Sri Saran Balaji Vellore Rajakumar <srajakum@amazon.com>
Date: Sun, 13 Sep 2020 12:48:45 -0700
Subject: [PATCH] Disable TCP early demux when pod-eni is enabled

---
 config/master/aws-k8s-cni-cn.yaml            | 5 ++++-
 config/master/aws-k8s-cni-us-gov-east-1.yaml | 5 ++++-
 config/master/aws-k8s-cni-us-gov-west-1.yaml | 5 ++++-
 config/master/aws-k8s-cni.yaml               | 5 ++++-
 config/master/manifests.jsonnet              | 5 +++++
 scripts/init.sh                              | 9 ++++++++-
 6 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/config/master/aws-k8s-cni-cn.yaml b/config/master/aws-k8s-cni-cn.yaml
index fbaaf86809..4da5ebbd5b 100644
--- a/config/master/aws-k8s-cni-cn.yaml
+++ b/config/master/aws-k8s-cni-cn.yaml
@@ -193,7 +193,10 @@
           "name": "xtables-lock"
       "hostNetwork": true
       "initContainers":
-      - "image": "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni-init:latest"
+      - "env":
+        - "name": "DISABLE_TCP_EARLY_DEMUX"
+          "value": "false"
+        "image": "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni-init:latest"
         "imagePullPolicy": "Always"
         "name": "aws-vpc-cni-init"
         "securityContext":
diff --git a/config/master/aws-k8s-cni-us-gov-east-1.yaml b/config/master/aws-k8s-cni-us-gov-east-1.yaml
index 81ce6620a7..20ee91a6ad 100644
--- a/config/master/aws-k8s-cni-us-gov-east-1.yaml
+++ b/config/master/aws-k8s-cni-us-gov-east-1.yaml
@@ -193,7 +193,10 @@
           "name": "xtables-lock"
       "hostNetwork": true
       "initContainers":
-      - "image": "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/amazon-k8s-cni-init:latest"
+      - "env":
+        - "name": "DISABLE_TCP_EARLY_DEMUX"
+          "value": "false"
+        "image": "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/amazon-k8s-cni-init:latest"
         "imagePullPolicy": "Always"
         "name": "aws-vpc-cni-init"
         "securityContext":
diff --git a/config/master/aws-k8s-cni-us-gov-west-1.yaml b/config/master/aws-k8s-cni-us-gov-west-1.yaml
index d7aa51f802..c01674facb 100644
--- a/config/master/aws-k8s-cni-us-gov-west-1.yaml
+++ b/config/master/aws-k8s-cni-us-gov-west-1.yaml
@@ -193,7 +193,10 @@
           "name": "xtables-lock"
       "hostNetwork": true
       "initContainers":
-      - "image": "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/amazon-k8s-cni-init:latest"
+      - "env":
+        - "name": "DISABLE_TCP_EARLY_DEMUX"
+          "value": "false"
+        "image": "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/amazon-k8s-cni-init:latest"
         "imagePullPolicy": "Always"
         "name": "aws-vpc-cni-init"
         "securityContext":
diff --git a/config/master/aws-k8s-cni.yaml b/config/master/aws-k8s-cni.yaml
index da182435e5..d430763a6e 100644
--- a/config/master/aws-k8s-cni.yaml
+++ b/config/master/aws-k8s-cni.yaml
@@ -193,7 +193,10 @@
           "name": "xtables-lock"
       "hostNetwork": true
       "initContainers":
-      - "image": "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:latest"
+      - "env":
+        - "name": "DISABLE_TCP_EARLY_DEMUX"
+          "value": "false"
+        "image": "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:latest"
         "imagePullPolicy": "Always"
         "name": "aws-vpc-cni-init"
         "securityContext":
diff --git a/config/master/manifests.jsonnet b/config/master/manifests.jsonnet
index b8cb4a2eed..3b9cef7a0e 100644
--- a/config/master/manifests.jsonnet
+++ b/config/master/manifests.jsonnet
@@ -227,6 +227,11 @@ local awsnode = {
               image: "%s/amazon-k8s-cni-init:%s" % [$.ecrRepo, $.version],
               imagePullPolicy: "Always",
               securityContext: {privileged: true},
+              env: [
+                {
+                  name: "DISABLE_TCP_EARLY_DEMUX", value: "false",
+                },
+              ],
               volumeMounts: [
                 {mountPath: "/host/opt/cni/bin", name: "cni-bin-dir"},
               ],
diff --git a/scripts/init.sh b/scripts/init.sh
index b5c038595d..e7f6a2b7bb 100755
--- a/scripts/init.sh
+++ b/scripts/init.sh
@@ -27,7 +27,14 @@ TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-meta
 HOST_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/local-ipv4)
 PRIMARY_IF=$(ip -4 -o a | grep "$HOST_IP" | awk '{print $2}')
 sysctl -w "net.ipv4.conf.$PRIMARY_IF.rp_filter=2"
-
 cat "/proc/sys/net/ipv4/conf/$PRIMARY_IF/rp_filter"
 
+# Set DISABLE_TCP_EARLY_DEMUX to true to enable kubelet to pod-eni TCP communication
+# https://lwn.net/Articles/503420/ and https://github.com/aws/amazon-vpc-cni-k8s/pull/1212 for background
+if [ "${DISABLE_TCP_EARLY_DEMUX:-false}" == "true" ]; then
+    sysctl -w "net.ipv4.tcp_early_demux=0"
+else
+    sysctl -w "net.ipv4.tcp_early_demux=1"
+fi
+
 echo "CNI init container done"