Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Containerd runtime support #698

Merged
merged 3 commits into from
Jul 13, 2021
Merged

Containerd runtime support #698

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dc4f118
Merge pull request #1 from awslabs/master
ravisinha0506 Jul 13, 2021
303b914
adding support for containerd runtime support in eks worker ami
ravisinha0506 Jul 13, 2021
985694d
msg
ravisinha0506 Jul 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 37 additions & 14 deletions files/bootstrap.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ function print_help {
echo "--dns-cluster-ip Overrides the IP address to use for DNS queries within the cluster. Defaults to 10.100.0.10 or 172.20.0.10 based on the IP address of the primary interface"
echo "--pause-container-account The AWS account (number) to pull the pause container from"
echo "--pause-container-version The tag of the pause container"
echo "--container-runtime Specify a container runtime (default: dockerd)"
}

POSITIONAL=()
Expand Down Expand Up @@ -87,6 +88,11 @@ while [[ $# -gt 0 ]]; do
shift
shift
;;
--container-runtime)
CONTAINER_RUNTIME=$2
shift
shift
;;
*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
Expand All @@ -109,6 +115,7 @@ ENABLE_DOCKER_BRIDGE="${ENABLE_DOCKER_BRIDGE:-false}"
API_RETRY_ATTEMPTS="${API_RETRY_ATTEMPTS:-3}"
DOCKER_CONFIG_JSON="${DOCKER_CONFIG_JSON:-}"
PAUSE_CONTAINER_VERSION="${PAUSE_CONTAINER_VERSION:-3.1-eksbuild.1}"
CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-dockerd}"

function get_pause_container_account_for_region () {
local region="$1"
Expand Down Expand Up @@ -392,24 +399,40 @@ Environment='KUBELET_EXTRA_ARGS=$KUBELET_EXTRA_ARGS'
EOF
fi

# Replace with custom docker config contents.
if [[ -n "$DOCKER_CONFIG_JSON" ]]; then
if [[ "$CONTAINER_RUNTIME" = "containerd" ]]; then
sudo mkdir -p /etc/containerd
sudo mkdir -p /etc/cni/net.d
sudo mv /etc/eks/containerd/containerd-config.toml /etc/containerd/config.toml
sudo mv /etc/eks/containerd/kubelet-containerd.service /etc/systemd/system/kubelet.service
sudo chown root:root /etc/systemd/system/kubelet.service
systemctl daemon-reload
systemctl enable containerd
systemctl start containerd
elif [[ "$CONTAINER_RUNTIME" = "dockerd" ]]; then
mkdir -p /etc/docker

echo "$DOCKER_CONFIG_JSON" > /etc/docker/daemon.json
systemctl restart docker
bash -c "/sbin/iptables-save > /etc/sysconfig/iptables"
mv /etc/eks/iptables-restore.service /etc/systemd/system/iptables-restore.service
sudo chown root:root /etc/systemd/system/iptables-restore.service
systemctl daemon-reload
systemctl enable iptables-restore

if [[ -n "$DOCKER_CONFIG_JSON" ]]; then
echo "$DOCKER_CONFIG_JSON" > /etc/docker/daemon.json
fi
if [[ "$ENABLE_DOCKER_BRIDGE" = "true" ]]; then
# Enabling the docker bridge network. We have to disable live-restore as it
# prevents docker from recreating the default bridge network on restart
echo "$(jq '.bridge="docker0" | ."live-restore"=false' /etc/docker/daemon.json)" > /etc/docker/daemon.json
fi
systemctl daemon-reload
systemctl enable docker
systemctl start docker
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi
In the previous version of this file the Docker daemon was always restarted after its config file was modified. But after this commit nothing happens, the changes don't have an effect, probably because now the script runs systemctl start docker instead of systemctl restart docker

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change is impacting our automated testing too. Was wondering what the rationale between switching from restart to start was.

else
echo "Container runtime ${CONTAINER_RUNTIME} is not supported."
exit 1
fi

if [[ "$ENABLE_DOCKER_BRIDGE" = "true" ]]; then
mkdir -p /etc/docker

# Enabling the docker bridge network. We have to disable live-restore as it
# prevents docker from recreating the default bridge network on restart
echo "$(jq '.bridge="docker0" | ."live-restore"=false' /etc/docker/daemon.json)" > /etc/docker/daemon.json
systemctl restart docker
fi

systemctl daemon-reload
systemctl enable kubelet
systemctl start kubelet

Expand Down
16 changes: 16 additions & 0 deletions files/containerd-config.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
version = 2
root = "/var/lib/containerd"
state = "/run/containerd"

[grpc]
address = "/run/dockershim.sock"

[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"

[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
22 changes: 22 additions & 0 deletions files/kubelet-containerd.service
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=containerd.service
Requires=containerd.service

[Service]
ExecStartPre=/sbin/iptables -P FORWARD ACCEPT -w 5
ExecStart=/usr/bin/kubelet --cloud-provider aws \
--config /etc/kubernetes/kubelet/kubelet-config.json \
--kubeconfig /var/lib/kubelet/kubeconfig \
--container-runtime remote \
--container-runtime-endpoint unix:///run/dockershim.sock \
--network-plugin cni $KUBELET_ARGS $KUBELET_EXTRA_ARGS

Restart=on-failure
RestartForceExitStatus=SIGPIPE
RestartSec=5
KillMode=process

[Install]
WantedBy=multi-user.target
37 changes: 28 additions & 9 deletions scripts/install-worker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,14 +95,8 @@ fi
################################################################################
### iptables ###################################################################
################################################################################

# Enable forwarding via iptables
sudo bash -c "/sbin/iptables-save > /etc/sysconfig/iptables"

sudo mv $TEMPLATE_DIR/iptables-restore.service /etc/systemd/system/iptables-restore.service

sudo systemctl daemon-reload
sudo systemctl enable iptables-restore
sudo mkdir -p /etc/eks
sudo mv $TEMPLATE_DIR/iptables-restore.service /etc/eks/iptables-restore.service

################################################################################
### Docker #####################################################################
Expand Down Expand Up @@ -141,9 +135,34 @@ if [[ "$INSTALL_DOCKER" == "true" ]]; then

# Enable docker daemon to start on boot.
sudo systemctl daemon-reload
sudo systemctl enable docker
fi

###############################################################################
### Containerd setup ##########################################################
###############################################################################

sudo mkdir -p /etc/eks/containerd
if [ -f "/etc/eks/containerd/containerd-config.toml" ]; then
## this means we are building a gpu ami and have already placed a containerd configuration file in /etc/eks
echo "containerd config is already present"
else
sudo mv $TEMPLATE_DIR/containerd-config.toml /etc/eks/containerd/containerd-config.toml
fi

sudo mv $TEMPLATE_DIR/kubelet-containerd.service /etc/eks/containerd/kubelet-containerd.service

cat <<EOF | sudo tee -a /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

cat <<EOF | sudo tee -a /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF


################################################################################
### Logrotate ##################################################################
################################################################################
Expand Down