From 4200a86814d1e616799bd20dbf0d3d3080f105ba Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Mon, 5 Nov 2018 15:43:06 -0800 Subject: [PATCH] Pre-pull hyperkube in VHD (#4174) --- packer/install-dependencies.sh | 36 ++++++++++++++----------- parts/k8s/kubernetescustomscript.sh | 2 +- parts/k8s/kubernetesinstalls.sh | 42 ++++++++++++++++++++--------- 3 files changed, 51 insertions(+), 29 deletions(-) diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index b7d42d9035..f1e9c282dd 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -41,7 +41,7 @@ installContainerd installImg -DASHBOARD_VERSIONS="1.8.3 1.6.3" +DASHBOARD_VERSIONS="1.10.0 1.6.3" for DASHBOARD_VERSION in ${DASHBOARD_VERSIONS}; do pullContainerImage "docker" "k8s.gcr.io/kubernetes-dashboard-amd64:v${DASHBOARD_VERSION}" done @@ -56,7 +56,7 @@ for ADDON_RESIZER_VERSION in ${ADDON_RESIZER_VERSIONS}; do pullContainerImage "docker" "k8s.gcr.io/addon-resizer:${ADDON_RESIZER_VERSION}" done -HEAPSTER_VERSIONS="1.5.3 1.5.1" +HEAPSTER_VERSIONS="1.5.3 1.5.1 1.3.0" for HEAPSTER_VERSION in ${HEAPSTER_VERSIONS}; do pullContainerImage "docker" "k8s.gcr.io/heapster-amd64:v${HEAPSTER_VERSION}" done @@ -66,12 +66,12 @@ for METRICS_SERVER_VERSION in ${METRICS_SERVER_VERSIONS}; do pullContainerImage "docker" "k8s.gcr.io/metrics-server-amd64:v${METRICS_SERVER_VERSION}" done -KUBE_DNS_VERSIONS="1.14.10 1.14.8 1.14.5" +KUBE_DNS_VERSIONS="1.14.13 1.14.5" for KUBE_DNS_VERSION in ${KUBE_DNS_VERSIONS}; do pullContainerImage "docker" "k8s.gcr.io/k8s-dns-kube-dns-amd64:${KUBE_DNS_VERSION}" done -KUBE_ADDON_MANAGER_VERSIONS="8.7 8.6" +KUBE_ADDON_MANAGER_VERSIONS="8.8 8.7 8.6" for KUBE_ADDON_MANAGER_VERSION in ${KUBE_ADDON_MANAGER_VERSIONS}; do pullContainerImage "docker" "k8s.gcr.io/kube-addon-manager-amd64:v${KUBE_ADDON_MANAGER_VERSION}" done @@ -91,6 +91,21 @@ for TILLER_VERSION in ${TILLER_VERSIONS}; do pullContainerImage "docker" "gcr.io/kubernetes-helm/tiller:v${TILLER_VERSION}" done +CLUSTER_AUTOSCALER_VERSIONS="1.3.3 1.3.1 1.3.0 1.2.2 1.1.2" +for CLUSTER_AUTOSCALER_VERSION in ${CLUSTER_AUTOSCALER_VERSIONS}; do + pullContainerImage "docker" "k8s.gcr.io/cluster-autoscaler:v${CLUSTER_AUTOSCALER_VERSION}" +done + +K8S_DNS_SIDECAR_VERSIONS="1.14.10 1.14.8 1.14.7" +for K8S_DNS_SIDECAR_VERSION in ${K8S_DNS_SIDECAR_VERSIONS}; do + pullContainerImage "docker" "k8s.gcr.io/k8s-dns-sidecar-amd64:${K8S_DNS_SIDECAR_VERSION}" +done + +CORE_DNS_VERSIONS="1.2.2" +for CORE_DNS_VERSION in ${CORE_DNS_VERSIONS}; do + pullContainerImage "docker" "k8s.gcr.io/coredns:${CORE_DNS_VERSION}" +done + RESCHEDULER_VERSIONS="0.4.0 0.3.1" for RESCHEDULER_VERSION in ${RESCHEDULER_VERSIONS}; do pullContainerImage "docker" "k8s.gcr.io/rescheduler:v${RESCHEDULER_VERSION}" @@ -106,16 +121,6 @@ for AZURE_CNI_NETWORKMONITOR_VERSION in ${AZURE_CNI_NETWORKMONITOR_VERSIONS}; do pullContainerImage "docker" "containernetworking/networkmonitor:v${AZURE_CNI_NETWORKMONITOR_VERSION}" done -CLUSTER_AUTOSCALER_VERSIONS="1.3.3 1.3.1 1.3.0 1.2.2 1.1.2" -for CLUSTER_AUTOSCALER_VERSION in ${CLUSTER_AUTOSCALER_VERSIONS}; do - pullContainerImage "docker" "k8s.gcr.io/cluster-autoscaler:v${CLUSTER_AUTOSCALER_VERSION}" -done - -K8S_DNS_SIDECAR_VERSIONS="1.14.10 1.14.8 1.14.7" -for K8S_DNS_SIDECAR_VERSION in ${K8S_DNS_SIDECAR_VERSIONS}; do - pullContainerImage "docker" "k8s.gcr.io/k8s-dns-sidecar-amd64:${K8S_DNS_SIDECAR_VERSION}" -done - NVIDIA_DEVICE_PLUGIN_VERSIONS="1.11 1.10" for NVIDIA_DEVICE_PLUGIN_VERSION in ${NVIDIA_DEVICE_PLUGIN_VERSIONS}; do pullContainerImage "docker" "nvidia/k8s-device-plugin:${NVIDIA_DEVICE_PLUGIN_VERSION}" @@ -128,7 +133,8 @@ K8S_VERSIONS="1.7.15 1.7.16 1.8.14 1.8.15 1.9.10 1.9.11 1.10.8 1.10.9 1.11.3 1.1 for KUBERNETES_VERSION in ${K8S_VERSIONS}; do HYPERKUBE_URL="k8s.gcr.io/hyperkube-amd64:v${KUBERNETES_VERSION}" - pullHyperkube + extractHyperkube "docker" + pullContainerImage "docker" "k8s.gcr.io/cloud-controller-manager-amd64:v${KUBERNETES_VERSION}" done df -h diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index 0682e27743..06d77eeaf3 100755 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -72,7 +72,7 @@ fi installContainerRuntime installNetworkPlugin installContainerd -extractHyperkube +installKubeletAndKubectl ensureRPC createKubeManifestDir diff --git a/parts/k8s/kubernetesinstalls.sh b/parts/k8s/kubernetesinstalls.sh index 5510d5afbb..47800f6c5c 100755 --- a/parts/k8s/kubernetesinstalls.sh +++ b/parts/k8s/kubernetesinstalls.sh @@ -175,30 +175,40 @@ function installImg() { retrycmd_get_executable 120 5 $img_filepath "https://acs-mirror.azureedge.net/img/img-linux-amd64-v0.4.6" ls || exit $ERR_IMG_DOWNLOAD_TIMEOUT } -function pullHyperkube() { - retrycmd_if_failure 60 1 1200 img pull $HYPERKUBE_URL || exit $ERR_K8S_DOWNLOAD_TIMEOUT - img unpack -o "/home/rootfs-${KUBERNETES_VERSION}" $HYPERKUBE_URL - path=$(find /home/rootfs-${KUBERNETES_VERSION} -name "hyperkube") +function extractHyperkube() { + CLI_TOOL=$1 + path="/home/hyperkube-downloads/${KUBERNETES_VERSION}" + mkdir -p "$path" + pullContainerImage $CLI_TOOL ${HYPERKUBE_URL} + if [[ "$CLI_TOOL" == "docker" ]]; then + docker run --rm -v $path:$path ${HYPERKUBE_URL} /bin/bash -c "cp /hyperkube $path" + else + img unpack -o "$path" ${HYPERKUBE_URL} + fi if [[ $OS == $COREOS_OS_NAME ]]; then - cp "$path" "/opt/kubelet" - mv "$path" "/opt/kubectl" + cp "$path/hyperkube" "/opt/kubelet" + mv "$path/hyperkube" "/opt/kubectl" chmod a+x /opt/kubelet /opt/kubectl else - cp "$path" "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" - mv "$path" "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" + cp "$path/hyperkube" "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" + mv "$path/hyperkube" "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" fi } -function extractHyperkube() { - if [[ ! -f "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" ]]; then - installImg - pullHyperkube +function installKubeletAndKubectl() { + if [[ ! -f "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" ]]; then + if [[ "$CONTAINER_RUNTIME" == "docker" ]]; then + extractHyperkube "docker" + else + installImg + extractHyperkube "img" + fi fi mv "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" "/usr/local/bin/kubelet" mv "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" "/usr/local/bin/kubectl" chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl - rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-* /home/rootfs-* & + rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-* /home/hyperkube-downloads & } function pullContainerImage() { @@ -206,3 +216,9 @@ function pullContainerImage() { DOCKER_IMAGE_URL=$2 retrycmd_if_failure 60 1 1200 $CLI_TOOL pull $DOCKER_IMAGE_URL || exit $ERR_IMG_DOWNLOAD_TIMEOUT } + +function cleanUpContainerImages() { + // TODO remove all unused container images at runtime + docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'hyperkube') & + docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'cloud-controller-manager') & +} \ No newline at end of file