all: rktnetes hack for rkt gc to work

Calling rkt from the kubelet fly container will call rkt in such a way that its invocation will break out of the container chroot. This should be removed once the write-api stuff is finalized in rktnetes or made a rkt fly feature. See: rkt/rkt#2878
coreos · Aug 12, 2016 · 78acfe9 · 78acfe9
1 parent 31e3341
commit 78acfe9
Show file tree

Hide file tree

Showing 5 changed files with 117 additions and 26 deletions.
diff --git a/multi-node/aws/pkg/config/templates/cloud-config-controller b/multi-node/aws/pkg/config/templates/cloud-config-controller
@@ -43,13 +43,13 @@ coreos:
         Environment=KUBELET_VERSION={{.K8sVer}}
         Environment=KUBELET_ACI={{.HyperkubeImageRepo}}
         Environment="RKT_OPTS=--volume dns,kind=host,source=/etc/resolv.conf \
-          --mount volume=dns,target=/etc/resolv.conf \
-          --volume rktbin,kind=host,source=/usr/bin/rkt \
-          --mount volume=rktbin,target=/usr/bin/rkt \
-          --volume var-lib-rkt,kind=host,source=/var/lib/rkt \
-          --mount volume=var-lib-rkt,target=/var/lib/rkt \
-          --volume=stage,kind=host,source=/usr/lib/rkt \
-          --mount volume=stage,target=/usr/lib/rkt"
+        --mount volume=dns,target=/etc/resolv.conf \
+        --volume=rkt,kind=host,source=/opt/bin/host-rkt \
+        --mount volume=rkt,target=/usr/bin/rkt \
+        --volume var-lib-rkt,kind=host,source=/var/lib/rkt \
+        --mount volume=var-lib-rkt,target=/var/lib/rkt \
+        --volume=stage,kind=host,source=/tmp \
+        --mount volume=stage,target=/tmp"
         ExecStart=/usr/lib/coreos/kubelet-wrapper \
         --api-servers=http://localhost:8080 \
         --network-plugin-dir=/etc/kubernetes/cni/net.d \
@@ -200,6 +200,21 @@ write_files:
           "http://127.0.0.1:8080/api/v1/namespaces/kube-system/services"
       done
 
+  - path: /opt/bin/host-rkt
+    permissions: 0755
+    owner: root:root
+    content: |
+      #!/bin/sh
+      # This is bind mounted into the kubelet rootfs and all rkt shell-outs go
+      # through this rkt wrapper. It essentially enters the host mount namespace
+      # (which it is already in) only for the purpose of breaking out of the chroot
+      # before calling rkt. It makes things like rkt gc work and avoids bind mounting
+      # in certain rkt filesystem dependancies into the kubelet rootfs. This can
+      # eventually be obviated when the write-api stuff gets upstream and rkt gc is
+      # through the api-server. Related issue:
+      # https://github.com/coreos/rkt/issues/2878
+      exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "$@"
+
 {{ if .UseCalico }}
   - path: /opt/bin/install-calico-system
     permissions: 0700

diff --git a/multi-node/aws/pkg/config/templates/cloud-config-worker b/multi-node/aws/pkg/config/templates/cloud-config-worker
@@ -25,13 +25,13 @@ coreos:
         Environment=KUBELET_VERSION={{.K8sVer}}
         Environment=KUBELET_ACI={{.HyperkubeImageRepo}}
         Environment="RKT_OPTS=--volume dns,kind=host,source=/etc/resolv.conf \
-          --mount volume=dns,target=/etc/resolv.conf \
-          --volume rktbin,kind=host,source=/usr/bin/rkt \
-          --mount volume=rktbin,target=/usr/bin/rkt \
-          --volume var-lib-rkt,kind=host,source=/var/lib/rkt \
-          --mount volume=var-lib-rkt,target=/var/lib/rkt \
-          --volume=stage,kind=host,source=/usr/lib/rkt \
-          --mount volume=stage,target=/usr/lib/rkt"
+        --mount volume=dns,target=/etc/resolv.conf \
+        --volume=rkt,kind=host,source=/opt/bin/host-rkt \
+        --mount volume=rkt,target=/usr/bin/rkt \
+        --volume var-lib-rkt,kind=host,source=/var/lib/rkt \
+        --mount volume=var-lib-rkt,target=/var/lib/rkt \
+        --volume=stage,kind=host,source=/tmp \
+        --mount volume=stage,target=/tmp"
         ExecStart=/usr/lib/coreos/kubelet-wrapper \
         --api-servers={{.SecureAPIServers}} \
         --network-plugin-dir=/etc/kubernetes/cni/net.d \
@@ -131,6 +131,21 @@ coreos:
         RequiredBy=kubelet.service
 
 write_files:
+  - path: /opt/bin/host-rkt
+    permissions: 0755
+    owner: root:root
+    content: |
+      #!/bin/sh
+      # This is bind mounted into the kubelet rootfs and all rkt shell-outs go
+      # through this rkt wrapper. It essentially enters the host mount namespace
+      # (which it is already in) only for the purpose of breaking out of the chroot
+      # before calling rkt. It makes things like rkt gc work and avoids bind mounting
+      # in certain rkt filesystem dependancies into the kubelet rootfs. This can
+      # eventually be obviated when the write-api stuff gets upstream and rkt gc is
+      # through the api-server. Related issue:
+      # https://github.com/coreos/rkt/issues/2878
+      exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "$@"
+
   - path: /etc/kubernetes/ssl/worker.pem.enc
     encoding: gzip+base64
     content: {{.TLSConfig.WorkerCert}}

diff --git a/multi-node/generic/controller-install.sh b/multi-node/generic/controller-install.sh
@@ -94,12 +94,12 @@ Environment=KUBELET_VERSION=${K8S_VER}
 Environment=KUBELET_ACI=${HYPERKUBE_IMAGE_REPO}
 Environment="RKT_OPTS=--volume dns,kind=host,source=/etc/resolv.conf \
   --mount volume=dns,target=/etc/resolv.conf \
-  --volume rktbin,kind=host,source=/usr/bin/rkt \
-  --mount volume=rktbin,target=/usr/bin/rkt \
+  --volume=rkt,kind=host,source=/opt/bin/host-rkt \
+  --mount volume=rkt,target=/usr/bin/rkt \
   --volume var-lib-rkt,kind=host,source=/var/lib/rkt \
   --mount volume=var-lib-rkt,target=/var/lib/rkt \
-  --volume=stage,kind=host,source=/usr/lib/rkt \
-  --mount volume=stage,target=/usr/lib/rkt"
+  --volume=stage,kind=host,source=/tmp \
+  --mount volume=stage,target=/tmp"
 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
 ExecStart=/usr/lib/coreos/kubelet-wrapper \
   --api-servers=http://127.0.0.1:8080 \
@@ -122,6 +122,25 @@ WantedBy=multi-user.target
 EOF
     fi
 
+    local TEMPLATE=/opt/bin/host-rkt
+    if [ ! -f $TEMPLATE ]; then
+        echo "TEMPLATE: $TEMPLATE"
+        mkdir -p $(dirname $TEMPLATE)
+        cat << EOF > $TEMPLATE
+#!/bin/sh
+# This is bind mounted into the kubelet rootfs and all rkt shell-outs go
+# through this rkt wrapper. It essentially enters the host mount namespace
+# (which it is already in) only for the purpose of breaking out of the chroot
+# before calling rkt. It makes things like rkt gc work and avoids bind mounting
+# in certain rkt filesystem dependancies into the kubelet rootfs. This can
+# eventually be obviated when the write-api stuff gets upstream and rkt gc is
+# through the api-server. Related issue:
+# https://github.com/coreos/rkt/issues/2878
+exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "\$@"
+EOF
+    fi
+
+
     local TEMPLATE=/etc/systemd/system/load-rkt-stage1.service
     if [ ${CONTAINER_RUNTIME} = "rkt" ] && [ ! -f $TEMPLATE ]; then
         echo "TEMPLATE: $TEMPLATE"
@@ -941,6 +960,8 @@ function enable_calico_policy {
 init_config
 init_templates
 
+chmod +x /opt/bin/host-rkt
+
 init_flannel
 
 systemctl stop update-engine; systemctl mask update-engine

diff --git a/multi-node/generic/worker-install.sh b/multi-node/generic/worker-install.sh
@@ -60,12 +60,12 @@ Environment=KUBELET_VERSION=${K8S_VER}
 Environment=KUBELET_ACI=${HYPERKUBE_IMAGE_REPO}
 Environment="RKT_OPTS=--volume dns,kind=host,source=/etc/resolv.conf \
   --mount volume=dns,target=/etc/resolv.conf \
-  --volume rktbin,kind=host,source=/usr/bin/rkt \
-  --mount volume=rktbin,target=/usr/bin/rkt \
+  --volume=rkt,kind=host,source=/opt/bin/host-rkt \
+  --mount volume=rkt,target=/usr/bin/rkt \
   --volume var-lib-rkt,kind=host,source=/var/lib/rkt \
   --mount volume=var-lib-rkt,target=/var/lib/rkt \
-  --volume=stage,kind=host,source=/usr/lib/rkt \
-  --mount volume=stage,target=/usr/lib/rkt"
+  --volume=stage,kind=host,source=/tmp \
+  --mount volume=stage,target=/tmp"
 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
 ExecStart=/usr/lib/coreos/kubelet-wrapper \
   --api-servers=${CONTROLLER_ENDPOINT} \
@@ -91,6 +91,24 @@ WantedBy=multi-user.target
 EOF
     fi
 
+    local TEMPLATE=/opt/bin/host-rkt
+    if [ ! -f $TEMPLATE ]; then
+        echo "TEMPLATE: $TEMPLATE"
+        mkdir -p $(dirname $TEMPLATE)
+        cat << EOF > $TEMPLATE
+#!/bin/sh
+# This is bind mounted into the kubelet rootfs and all rkt shell-outs go
+# through this rkt wrapper. It essentially enters the host mount namespace
+# (which it is already in) only for the purpose of breaking out of the chroot
+# before calling rkt. It makes things like rkt gc work and avoids bind mounting
+# in certain rkt filesystem dependancies into the kubelet rootfs. This can
+# eventually be obviated when the write-api stuff gets upstream and rkt gc is
+# through the api-server. Related issue:
+# https://github.com/coreos/rkt/issues/2878
+exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "\$@"
+EOF
+    fi
+
     local TEMPLATE=/etc/systemd/system/load-rkt-stage1.service
     if [ ${CONTAINER_RUNTIME} = "rkt" ] && [ ! -f $TEMPLATE ]; then
         echo "TEMPLATE: $TEMPLATE"
@@ -319,6 +337,8 @@ EOF
 init_config
 init_templates
 
+chmod +x /opt/bin/host-rkt
+
 systemctl stop update-engine; systemctl mask update-engine
 
 systemctl daemon-reload

diff --git a/single-node/user-data b/single-node/user-data
@@ -88,12 +88,12 @@ Environment=KUBELET_VERSION=${K8S_VER}
 Environment=KUBELET_ACI=${HYPERKUBE_IMAGE_REPO}
 Environment="RKT_OPTS=--volume dns,kind=host,source=/etc/resolv.conf \
   --mount volume=dns,target=/etc/resolv.conf \
-  --volume rktbin,kind=host,source=/usr/bin/rkt \
-  --mount volume=rktbin,target=/usr/bin/rkt \
+  --volume=rkt,kind=host,source=/opt/bin/host-rkt \
+  --mount volume=rkt,target=/usr/bin/rkt \
   --volume var-lib-rkt,kind=host,source=/var/lib/rkt \
   --mount volume=var-lib-rkt,target=/var/lib/rkt \
-  --volume=stage,kind=host,source=/usr/lib/rkt \
-  --mount volume=stage,target=/usr/lib/rkt"
+  --volume=stage,kind=host,source=/tmp \
+  --mount volume=stage,target=/tmp"
 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
 ExecStart=/usr/lib/coreos/kubelet-wrapper \
   --api-servers=http://127.0.0.1:8080 \
@@ -117,6 +117,24 @@ WantedBy=multi-user.target
 EOF
     fi
 
+    local TEMPLATE=/opt/bin/host-rkt
+    if [ ! -f $TEMPLATE ]; then
+        echo "TEMPLATE: $TEMPLATE"
+        mkdir -p $(dirname $TEMPLATE)
+        cat << EOF > $TEMPLATE
+#!/bin/sh
+# This is bind mounted into the kubelet rootfs and all rkt shell-outs go
+# through this rkt wrapper. It essentially enters the host mount namespace
+# (which it is already in) only for the purpose of breaking out of the chroot
+# before calling rkt. It makes things like rkt gc work and avoids bind mounting
+# in certain rkt filesystem dependancies into the kubelet rootfs. This can
+# eventually be obviated when the write-api stuff gets upstream and rkt gc is
+# through the api-server. Related issue:
+# https://github.com/coreos/rkt/issues/2878
+exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "\$@"
+EOF
+    fi
+
     local TEMPLATE=/etc/systemd/system/load-rkt-stage1.service
     if [ ${CONTAINER_RUNTIME} = "rkt" ] && [ ! -f $TEMPLATE ]; then
         echo "TEMPLATE: $TEMPLATE"
@@ -934,6 +952,8 @@ init_config
 init_templates
 systemctl enable etcd2; systemctl start etcd2
 
+chmod +x /opt/bin/host-rkt
+
 init_flannel
 
 systemctl stop update-engine; systemctl mask update-engine