diff --git a/files/bootstrap.sh b/files/bootstrap.sh index b2a3c1698..c59855a0c 100644 --- a/files/bootstrap.sh +++ b/files/bootstrap.sh @@ -26,6 +26,7 @@ function print_help { echo "--dns-cluster-ip Overrides the IP address to use for DNS queries within the cluster. Defaults to 10.100.0.10 or 172.20.0.10 based on the IP address of the primary interface" echo "--pause-container-account The AWS account (number) to pull the pause container from" echo "--pause-container-version The tag of the pause container" + echo "--container-runtime Specify a container runtime (default: dockerd)" } POSITIONAL=() @@ -87,6 +88,11 @@ while [[ $# -gt 0 ]]; do shift shift ;; + --container-runtime) + CONTAINER_RUNTIME=$2 + shift + shift + ;; *) # unknown option POSITIONAL+=("$1") # save it in an array for later shift # past argument @@ -109,6 +115,7 @@ ENABLE_DOCKER_BRIDGE="${ENABLE_DOCKER_BRIDGE:-false}" API_RETRY_ATTEMPTS="${API_RETRY_ATTEMPTS:-3}" DOCKER_CONFIG_JSON="${DOCKER_CONFIG_JSON:-}" PAUSE_CONTAINER_VERSION="${PAUSE_CONTAINER_VERSION:-3.1-eksbuild.1}" +CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-dockerd}" function get_pause_container_account_for_region () { local region="$1" @@ -392,24 +399,40 @@ Environment='KUBELET_EXTRA_ARGS=$KUBELET_EXTRA_ARGS' EOF fi -# Replace with custom docker config contents. -if [[ -n "$DOCKER_CONFIG_JSON" ]]; then +if [[ "$CONTAINER_RUNTIME" = "containerd" ]]; then + sudo mkdir -p /etc/containerd + sudo mkdir -p /etc/cni/net.d + sudo mv /etc/eks/containerd/containerd-config.toml /etc/containerd/config.toml + sudo mv /etc/eks/containerd/kubelet-containerd.service /etc/systemd/system/kubelet.service + sudo chown root:root /etc/systemd/system/kubelet.service + systemctl daemon-reload + systemctl enable containerd + systemctl start containerd +elif [[ "$CONTAINER_RUNTIME" = "dockerd" ]]; then mkdir -p /etc/docker - - echo "$DOCKER_CONFIG_JSON" > /etc/docker/daemon.json - systemctl restart docker + bash -c "/sbin/iptables-save > /etc/sysconfig/iptables" + mv /etc/eks/iptables-restore.service /etc/systemd/system/iptables-restore.service + sudo chown root:root /etc/systemd/system/iptables-restore.service + systemctl daemon-reload + systemctl enable iptables-restore + + if [[ -n "$DOCKER_CONFIG_JSON" ]]; then + echo "$DOCKER_CONFIG_JSON" > /etc/docker/daemon.json + fi + if [[ "$ENABLE_DOCKER_BRIDGE" = "true" ]]; then + # Enabling the docker bridge network. We have to disable live-restore as it + # prevents docker from recreating the default bridge network on restart + echo "$(jq '.bridge="docker0" | ."live-restore"=false' /etc/docker/daemon.json)" > /etc/docker/daemon.json + fi + systemctl daemon-reload + systemctl enable docker + systemctl start docker +else + echo "Container runtime ${CONTAINER_RUNTIME} is not supported." + exit 1 fi -if [[ "$ENABLE_DOCKER_BRIDGE" = "true" ]]; then - mkdir -p /etc/docker - - # Enabling the docker bridge network. We have to disable live-restore as it - # prevents docker from recreating the default bridge network on restart - echo "$(jq '.bridge="docker0" | ."live-restore"=false' /etc/docker/daemon.json)" > /etc/docker/daemon.json - systemctl restart docker -fi -systemctl daemon-reload systemctl enable kubelet systemctl start kubelet diff --git a/files/containerd-config.toml b/files/containerd-config.toml new file mode 100644 index 000000000..79e53f1ec --- /dev/null +++ b/files/containerd-config.toml @@ -0,0 +1,16 @@ +version = 2 +root = "/var/lib/containerd" +state = "/run/containerd" + +[grpc] +address = "/run/dockershim.sock" + +[plugins."io.containerd.grpc.v1.cri".containerd] +default_runtime_name = "runc" + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] +runtime_type = "io.containerd.runc.v2" + +[plugins."io.containerd.grpc.v1.cri".cni] +bin_dir = "/opt/cni/bin" +conf_dir = "/etc/cni/net.d" diff --git a/files/kubelet-containerd.service b/files/kubelet-containerd.service new file mode 100644 index 000000000..ed08ed97a --- /dev/null +++ b/files/kubelet-containerd.service @@ -0,0 +1,22 @@ +[Unit] +Description=Kubernetes Kubelet +Documentation=https://github.com/kubernetes/kubernetes +After=containerd.service +Requires=containerd.service + +[Service] +ExecStartPre=/sbin/iptables -P FORWARD ACCEPT -w 5 +ExecStart=/usr/bin/kubelet --cloud-provider aws \ + --config /etc/kubernetes/kubelet/kubelet-config.json \ + --kubeconfig /var/lib/kubelet/kubeconfig \ + --container-runtime remote \ + --container-runtime-endpoint unix:///run/dockershim.sock \ + --network-plugin cni $KUBELET_ARGS $KUBELET_EXTRA_ARGS + +Restart=on-failure +RestartForceExitStatus=SIGPIPE +RestartSec=5 +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/scripts/install-worker.sh b/scripts/install-worker.sh index d6f6787f4..cb464a31a 100644 --- a/scripts/install-worker.sh +++ b/scripts/install-worker.sh @@ -95,14 +95,8 @@ fi ################################################################################ ### iptables ################################################################### ################################################################################ - -# Enable forwarding via iptables -sudo bash -c "/sbin/iptables-save > /etc/sysconfig/iptables" - -sudo mv $TEMPLATE_DIR/iptables-restore.service /etc/systemd/system/iptables-restore.service - -sudo systemctl daemon-reload -sudo systemctl enable iptables-restore +sudo mkdir -p /etc/eks +sudo mv $TEMPLATE_DIR/iptables-restore.service /etc/eks/iptables-restore.service ################################################################################ ### Docker ##################################################################### @@ -141,9 +135,34 @@ if [[ "$INSTALL_DOCKER" == "true" ]]; then # Enable docker daemon to start on boot. sudo systemctl daemon-reload - sudo systemctl enable docker fi +############################################################################### +### Containerd setup ########################################################## +############################################################################### + +sudo mkdir -p /etc/eks/containerd +if [ -f "/etc/eks/containerd/containerd-config.toml" ]; then + ## this means we are building a gpu ami and have already placed a containerd configuration file in /etc/eks + echo "containerd config is already present" +else + sudo mv $TEMPLATE_DIR/containerd-config.toml /etc/eks/containerd/containerd-config.toml +fi + +sudo mv $TEMPLATE_DIR/kubelet-containerd.service /etc/eks/containerd/kubelet-containerd.service + +cat <