Skip to content
This repository has been archived by the owner on Jan 11, 2023. It is now read-only.

Commit

Permalink
remove docker dependency from hyperkube extract (#3296)
Browse files Browse the repository at this point in the history 
* remove docker dependency from hyperkube extract

Signed-off-by: Jess Frazelle <acidburn@microsoft.com>

* update install containerd

Signed-off-by: Jess Frazelle <acidburn@microsoft.com>
  • Loading branch information
@jessfraz @acs-bot
jessfraz authored and acs-bot committed Jun 18, 2018
1 parent c4b1324 commit 2b88659
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 90 deletions.
34 changes: 0 additions & 34 deletions parts/k8s/kubernetesagentcustomdata.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,40 +90,6 @@ write_files:
name: localclustercontext
current-context: localclustercontext
- path: "/etc/systemd/system/hyperkube-extract.service"
permissions: "0644"
owner: "root"
content: |
[Unit]
Description=kubectl and kubelet extraction
Requires=docker.service
After=docker.service
{{if .IsCoreOS}}
ConditionPathExists=!/opt/kubectl
{{else}}
ConditionPathExists=!/usr/local/bin/kubectl
{{end}}

[Service]
TimeoutStartSec=0
Restart=on-failure
RestartSec=5s
ExecStartPre=/bin/mkdir -p /tmp/hyperkubedir
ExecStartPre=/usr/bin/docker pull {{WrapAsVariable "kubernetesHyperkubeSpec"}}
ExecStartPre=/usr/bin/docker run --rm -v /tmp/hyperkubedir:/opt/hyperkubedir {{WrapAsVariable "kubernetesHyperkubeSpec"}} /bin/bash -c "cp /hyperkube /opt/hyperkubedir/"
{{if .IsCoreOS}}
ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /opt/kubelet
ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /opt/kubectl
ExecStart=/bin/chmod a+x /opt/kubelet /opt/kubectl
{{else}}
ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /usr/local/bin/kubelet
ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /usr/local/bin/kubectl
ExecStart=/bin/chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl
{{end}}

[Install]
WantedBy=multi-user.target

- path: "/etc/default/kubelet"
permissions: "0644"
owner: "root"
Expand Down
58 changes: 37 additions & 21 deletions parts/k8s/kubernetescustomscript.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ function installDeps() {
# make sure walinuxagent doesn't get updated in the middle of running this script
retrycmd_if_failure 20 5 30 apt-mark hold walinuxagent || exit $ERR_HOLD_WALINUXAGENT
# See https://github.com/kubernetes/kubernetes/blob/master/build/debian-hyperkube-base/Dockerfile#L25-L44
apt_get_install 20 30 300 apt-transport-https ca-certificates iptables iproute2 socat util-linux mount ebtables ethtool init-system-helpers nfs-common ceph-common conntrack glusterfs-client ipset jq || exit $ERR_APT_INSTALL_TIMEOUT
apt_get_install 20 30 300 apt-transport-https ca-certificates iptables iproute2 ebtables socat util-linux mount ebtables ethtool init-system-helpers nfs-common ceph-common conntrack glusterfs-client ipset jq cgroup-lite git pigz xz-utils || exit $ERR_APT_INSTALL_TIMEOUT
systemctlEnableAndStart rpcbind
systemctlEnableAndStart rpc-statd
}
Expand All @@ -171,7 +171,7 @@ function installDocker() {
echo "deb ${DOCKER_REPO} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list
printf "Package: docker-engine\nPin: version ${DOCKER_ENGINE_VERSION}\nPin-Priority: 550\n" > /etc/apt/preferences.d/docker.pref
apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT
apt_get_install 20 30 120 ebtables docker-engine || exit $ERR_DOCKER_INSTALL_TIMEOUT
apt_get_install 20 30 120 docker-engine || exit $ERR_DOCKER_INSTALL_TIMEOUT
echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf
usermod -aG docker ${ADMINUSER}
}
Expand Down Expand Up @@ -308,19 +308,6 @@ function installClearContainersRuntime() {
systemctlEnableAndStart cc-proxy
}

function installContainerd() {
CRI_CONTAINERD_VERSION="1.1.0"
CONTAINERD_DOWNLOAD_URL="https://storage.googleapis.com/cri-containerd-release/cri-containerd-${CRI_CONTAINERD_VERSION}.linux-amd64.tar.gz"

CONTAINERD_TGZ_TMP=/tmp/containerd.tar.gz
retrycmd_get_tarball 60 5 "$CONTAINERD_TGZ_TMP" "$CONTAINERD_DOWNLOAD_URL"
tar -xzf "$CONTAINERD_TGZ_TMP" -C /
rm -f "$CONTAINERD_TGZ_TMP"

echo "Successfully installed cri-containerd..."
setupContainerd
}

function setupContainerd() {
echo "Configuring cri-containerd..."

Expand All @@ -342,6 +329,21 @@ function setupContainerd() {
setKubeletOpts " --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
}

function installContainerd() {
CRI_CONTAINERD_VERSION="1.1.0"
CONTAINERD_DOWNLOAD_URL="https://storage.googleapis.com/cri-containerd-release/cri-containerd-${CRI_CONTAINERD_VERSION}.linux-amd64.tar.gz"

CONTAINERD_TGZ_TMP=/tmp/containerd.tar.gz
retrycmd_get_tarball 60 5 "$CONTAINERD_TGZ_TMP" "$CONTAINERD_DOWNLOAD_URL"
tar -xzf "$CONTAINERD_TGZ_TMP" -C /
rm -f "$CONTAINERD_TGZ_TMP"

echo "Successfully installed cri-containerd..."
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
setupContainerd
fi
}

function ensureContainerd() {
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
# Enable and start cri-containerd service
Expand All @@ -364,8 +366,22 @@ function ensureKubelet() {
}

function extractHyperkube(){
retrycmd_if_failure 100 1 60 docker pull $HYPERKUBE_URL || exit $ERR_K8S_DOWNLOAD_TIMEOUT
systemctlEnableAndStart hyperkube-extract
TMP_DIR=$(mktemp -d)
retrycmd_if_failure 100 1 30 curl -sSL -o /usr/local/bin/img "https://github.com/genuinetools/img/releases/download/v0.4.6/img-linux-amd64"
chmod +x /usr/local/bin/img
retrycmd_if_failure 100 1 60 img pull $HYPERKUBE_URL || $ERR_K8S_DOWNLOAD_TIMEOUT
path=$(find /tmp/img -name "hyperkube")

if [[ $OS == $COREOS_OS_NAME ]]; then
cp "$path" "/opt/kubelet"
cp "$path" "/opt/kubectl"
chmod a+x /opt/kubelet /opt/kubectl
else
cp "$path" "/usr/local/bin/kubelet"
cp "$path" "/usr/local/bin/kubectl"
chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl
fi
rm -rf /tmp/hyperkube.tar "/tmp/img"
}

function ensureJournal(){
Expand Down Expand Up @@ -507,6 +523,10 @@ if [[ ! -z "${MASTER_NODE}" ]]; then
echo `date`,`hostname`, configAddonsDone>>/opt/m
fi

# containerd needs to be installed before extractHyperkube
# so runc is present.
echo `date`,`hostname`, installContainerdStart>>/opt/m
installContainerd
echo `date`,`hostname`, extractHyperkubeStart>>/opt/m
extractHyperkube
echo `date`,`hostname`, extractHyperkubeDone>>/opt/m
Expand All @@ -517,10 +537,6 @@ if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
installClearContainersRuntime
fi
fi
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
echo `date`,`hostname`, installContainerdStart>>/opt/m
installContainerd
fi
echo `date`,`hostname`, ensureContainerdStart>>/opt/m
ensureContainerd

Expand Down
34 changes: 0 additions & 34 deletions parts/k8s/kubernetesmastercustomdata.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,40 +144,6 @@ MASTER_ADDONS_CONFIG_PLACEHOLDER

MASTER_CUSTOM_FILES_PLACEHOLDER

- path: "/etc/systemd/system/hyperkube-extract.service"
permissions: "0644"
owner: "root"
content: |
[Unit]
Description=kubectl and kubelet extraction
Requires=docker.service
After=docker.service
{{if .MasterProfile.IsCoreOS}}
ConditionPathExists=!/opt/kubectl
{{else}}
ConditionPathExists=!/usr/local/bin/kubectl
{{end}}

[Service]
TimeoutStartSec=0
Restart=on-failure
RestartSec=5s
ExecStartPre=/bin/mkdir -p /tmp/hyperkubedir
ExecStartPre=/usr/bin/docker pull {{WrapAsVariable "kubernetesHyperkubeSpec"}}
ExecStartPre=/usr/bin/docker run --rm -v /tmp/hyperkubedir:/opt/hyperkubedir {{WrapAsVariable "kubernetesHyperkubeSpec"}} /bin/bash -c "cp /hyperkube /opt/hyperkubedir/"
{{if .MasterProfile.IsCoreOS}}
ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /opt/kubelet
ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /opt/kubectl
ExecStart=/bin/chmod a+x /opt/kubelet /opt/kubectl
{{else}}
ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /usr/local/bin/kubelet
ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /usr/local/bin/kubectl
ExecStart=/bin/chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl
{{end}}

[Install]
WantedBy=multi-user.target

- path: "/etc/default/kubelet"
permissions: "0644"
owner: "root"
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/runner/cli_provisioner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ func (cli *CLIProvisioner) FetchProvisioningMetrics(path string, cfg *config.Con
agentFiles := []string{"/var/log/azure/cluster-provision.log", "/var/log/cloud-init.log",
"/var/log/cloud-init-output.log", "/var/log/syslog", "/var/log/azure/custom-script/handler.log",
"/opt/m", "/opt/azure/containers/kubelet.sh", "/opt/azure/containers/provision.sh",
"/opt/azure/provision-ps.log", "/var/log/azure/kubelet-status.log", "/var/log/azure/hyperkube-extract-status.log",
"/opt/azure/provision-ps.log", "/var/log/azure/kubelet-status.log",
"/var/log/azure/docker-status.log", "/var/log/azure/systemd-journald-status.log"}
masterFiles := agentFiles
masterFiles = append(masterFiles, "/opt/azure/containers/mountetcd.sh", "/opt/azure/containers/setup-etcd.sh", "/opt/azure/containers/setup-etcd.log")
Expand Down

3 comments on commit 2b88659

@andyzhangx
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have we already make acs-engine use containerd by default? And we found that https://storage.googleapis.com/cri-containerd-release/cri-containerd-${CRI_CONTAINERD_VERSION}.linux-amd64.tar.gz could not be accessed in azure china cloud, I have already mirrrored containerd to https://mirror.azure.cn/kubernetes/containerd, would script parts/k8s/kubernetescustomscript.sh accept any cloud parameters, e.g. PublicCloud, ChinaCloud, etc.
@jackfrancis @karataliu

@jackfrancis
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@andyzhangx containerd is not turned on by default. Could you file an issue or open a WIP PR that reminds us that containerd won't work in mooncake until we fix this? :)

@andyzhangx
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jackfrancis Thanks, new issue here: #3487
As I know, AKS has used containerd in underlay/HCP setup in latest version, so it would be a blocking issue for adopt latest AKS version in mooncake in the near future. @karataliu for correctess.

Please sign in to comment.