From 802e4fcfb805380ecd6f94421573c6e8747f4794 Mon Sep 17 00:00:00 2001 From: leigh capili Date: Mon, 1 Jun 2020 16:22:05 -0600 Subject: [PATCH 1/2] Fix kubeadm image build to support multi-arch 1. Copy build context when mutating Dockerfile for cross-builds + cleanup 2. Pass GOARCH as build-arg to support local, dependent, multi-arch images in FROM statements 3. Parameterize build-args for k8s RELEASE and BINARY_REF 4. Update install.sh to take GOARCH as a 4th commandline param --- images/Makefile | 53 ++++++++++++++++++++++++++------------- images/kubeadm/Dockerfile | 13 +++++++--- images/kubeadm/install.sh | 13 ++++++---- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/images/Makefile b/images/Makefile index ca5b3950c..f16ca5da8 100644 --- a/images/Makefile +++ b/images/Makefile @@ -15,8 +15,10 @@ IS_MANIFEST_LIST?=0 TAG:=${RELEASE}$(if $(strip $(VERSION)),-${VERSION}) OP:=build +TMPDIR?=/tmp + ifeq ($(IS_MANIFEST_LIST),1) -TEMP_DIR:=$(shell mktemp -d) +BUILD_TMPDIR:=$(shell mktemp -d $(TMPDIR)/ignite-image-build.XXXXXXXXXX) ARCH_TAG=-$(GOARCH) endif @@ -36,18 +38,34 @@ endif @ls ${WHAT} >/dev/null ifeq ($(IS_MANIFEST_LIST),0) - sed "s|DOCKERARCH|$(DOCKERARCH)|g;/QEMUARCH/d" ${WHAT}/Dockerfile | docker build --build-arg RELEASE -f - -t $(FULL_IMAGE_NAME) ${WHAT} + sed "s|DOCKERARCH|$(DOCKERARCH)|g;/QEMUARCH/d" ${WHAT}/Dockerfile \ + | docker build \ + --build-arg RELEASE \ + --build-arg BINARY_REF \ + --build-arg GOARCH \ + -f -\ + -t $(FULL_IMAGE_NAME) \ + ${WHAT} else # Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel docker run --rm --privileged multiarch/qemu-user-static:register --reset - sed "s|QEMUARCH|$(QEMUARCH)|g;s|DOCKERARCH|$(DOCKERARCH)|g" ${WHAT}/Dockerfile > $(TEMP_DIR)/Dockerfile + cp -a ${WHAT}/. $(BUILD_TMPDIR)/ + sed "s|QEMUARCH|$(QEMUARCH)|g;s|DOCKERARCH|$(DOCKERARCH)|g" ${WHAT}/Dockerfile > $(BUILD_TMPDIR)/Dockerfile + ifeq ($(GOARCH),amd64) - sed "/COPY qemu/d" -i $(TEMP_DIR)/Dockerfile + sed "/COPY qemu/d" -i $(BUILD_TMPDIR)/Dockerfile else $(MAKE) -C .. qemu - cp ../bin/$(GOARCH)/qemu-$(QEMUARCH)-static $(TEMP_DIR) + cp ../bin/$(GOARCH)/qemu-$(QEMUARCH)-static $(BUILD_TMPDIR) endif - docker build --build-arg RELEASE -t $(FULL_IMAGE_NAME)$(ARCH_TAG) $(TEMP_DIR) + + docker build \ + --build-arg RELEASE \ + --build-arg BINARY_REF \ + --build-arg GOARCH \ + -t $(FULL_IMAGE_NAME)$(ARCH_TAG) \ + $(BUILD_TMPDIR) + find $(BUILD_TMPDIR)/ -mindepth 1 -delete endif docker tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(RELEASE_IMAGE_NAME)$(ARCH_TAG) @@ -77,15 +95,16 @@ push-all: build-all build-all: $(MAKE) ${OP} WHAT=amazon-kernel - $(MAKE) ${OP} WHAT=amazonlinux RELEASE=2 IS_LATEST=true + $(MAKE) ${OP} WHAT=amazonlinux RELEASE=2 IS_LATEST=true $(MAKE) ${OP} WHAT=alpine - $(MAKE) ${OP} WHAT=opensuse RELEASE=leap IS_LATEST=true - $(MAKE) ${OP} WHAT=opensuse RELEASE=tumbleweed - $(MAKE) ${OP} WHAT=ubuntu RELEASE=16.04 IS_MANIFEST_LIST=0 - $(MAKE) ${OP} WHAT=ubuntu RELEASE=18.04 IS_MANIFEST_LIST=1 GOARCH=arm64 - $(MAKE) ${OP} WHAT=ubuntu RELEASE=18.04 IS_MANIFEST_LIST=1 GOARCH=amd64 - $(MAKE) ${OP} WHAT=ubuntu RELEASE=20.04 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=arm64 - $(MAKE) ${OP} WHAT=ubuntu RELEASE=20.04 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=amd64 - $(MAKE) ${OP} WHAT=centos RELEASE=7 - $(MAKE) ${OP} WHAT=centos RELEASE=8 IS_LATEST=true - $(MAKE) ${OP} WHAT=kubeadm + $(MAKE) ${OP} WHAT=opensuse RELEASE=leap IS_LATEST=true + $(MAKE) ${OP} WHAT=opensuse RELEASE=tumbleweed + $(MAKE) ${OP} WHAT=ubuntu RELEASE=16.04 IS_MANIFEST_LIST=0 + $(MAKE) ${OP} WHAT=ubuntu RELEASE=18.04 IS_MANIFEST_LIST=1 GOARCH=arm64 + $(MAKE) ${OP} WHAT=ubuntu RELEASE=18.04 IS_MANIFEST_LIST=1 GOARCH=amd64 + $(MAKE) ${OP} WHAT=ubuntu RELEASE=20.04 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=arm64 + $(MAKE) ${OP} WHAT=ubuntu RELEASE=20.04 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=amd64 + $(MAKE) ${OP} WHAT=centos RELEASE=7 + $(MAKE) ${OP} WHAT=centos RELEASE=8 IS_LATEST=true + $(MAKE) ${OP} WHAT=kubeadm RELEASE=v1.18.3 BINARY_REF=release/stable-1.18 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=arm64 + $(MAKE) ${OP} WHAT=kubeadm RELEASE=v1.18.3 BINARY_REF=release/stable-1.18 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=amd64 diff --git a/images/kubeadm/Dockerfile b/images/kubeadm/Dockerfile index 8b4289104..02ec49266 100644 --- a/images/kubeadm/Dockerfile +++ b/images/kubeadm/Dockerfile @@ -1,5 +1,11 @@ +ARG GOARCH="amd64" + # Ubuntu 20.04 was also tested, but didn't perform very well (sshd took a long time to start), so we're sticking with Ubuntu 18.04 still FROM weaveworks/ignite-ubuntu:18.04 +ARG GOARCH="amd64" +ARG RELEASE +ARG BINARY_REF + # Install dependencies. Use containerd for running the containers (for better performance) RUN apt-get update && apt-get install -y --no-install-recommends \ apt-transport-https \ @@ -9,10 +15,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ jq \ && apt-get clean -# Install k8s v1.18.1 locally -ENV KUBERNETES_VERSION=v1.18.3 -COPY install.sh / -RUN /install.sh install release/stable-1.18 ${KUBERNETES_VERSION} +# Install k8s locally +COPY ./install.sh / +RUN /install.sh install "${BINARY_REF}" "${RELEASE}" "${GOARCH}" # Docker sets this automatically, but not containerd. # It is required when running kubeadm. RUN echo "net.ipv4.ip_forward=1" > /etc/sysctl.conf diff --git a/images/kubeadm/install.sh b/images/kubeadm/install.sh index 2316ea8b9..79680bb07 100755 --- a/images/kubeadm/install.sh +++ b/images/kubeadm/install.sh @@ -8,10 +8,11 @@ export KUBECONFIG=/etc/kubernetes/admin.conf MODE=${1} BINARY_REF=${2} CONTROL_PLANE_VERSION=${3} +GOARCH=${4} DRY_RUN=${DRY_RUN:-0} CLEANUP=${CLEANUP:-0} -if [[ $# != 3 && ${MODE} != "cleanup" ]]; then +if [[ $# != 4 && ${MODE} != "cleanup" ]]; then cat <<-EOF Usage: ${0} MODE BINARY_REF CONTROL_PLANE_VERSION @@ -19,6 +20,7 @@ if [[ $# != 3 && ${MODE} != "cleanup" ]]; then MODE=install|init|upgrade|cleanup BINARY_REF=What kubeadm binary and debs to pull. Can be a PR number, version label like "ci/latest" or exact (merged) commit like "v1.12.0-alpha.0-1035-gf2dec305ad" CONTROL_PLANE_VERSION=For init, this is the control plane version to use. For upgrade, this is the version to upgrade to + GOARCH=amd64|arm|arm64|ppc64le|s390x EOF exit 1 fi @@ -61,17 +63,17 @@ if [[ "${BINARY_REF}" =~ ^[0-9]{5}$ ]]; then BUILD_NUMBER=$(gsutil_cat gs://kubernetes-jenkins/pr-logs/pull/${PR_NUMBER}/pull-kubernetes-bazel-build/latest-build.txt) BAZEL_PULL_REF=$(gsutil_cat gs://kubernetes-jenkins/pr-logs/pull/${PR_NUMBER}/pull-kubernetes-bazel-build/${BUILD_NUMBER}/started.json | jq -r .pull) BAZEL_BUILD_LOCATION=$(gsutil_cat gs://kubernetes-jenkins/shared-results/${BAZEL_PULL_REF}/bazel-build-location.txt) - BINARY_BUCKET="${BAZEL_BUILD_LOCATION}/bin/linux/amd64" + BINARY_BUCKET="${BAZEL_BUILD_LOCATION}/bin/linux/${GOARCH}" elif [[ "${BINARY_REF}" =~ ^(ci|ci-cross){1}/latest ]]; then COMMIT=$(curl -sSL https://dl.k8s.io/${BINARY_REF}.txt) - BINARY_BUCKET="gs://kubernetes-release-dev/ci/${COMMIT}-bazel/bin/linux/amd64" + BINARY_BUCKET="gs://kubernetes-release-dev/ci/${COMMIT}-bazel/bin/linux/${GOARCH}" elif [[ "${BINARY_REF}" =~ ^release/[a-z]+(-[0-9]+.[0-9]+)*$ ]]; then RELEASE=$(curl -sSL https://dl.k8s.io/${BINARY_REF}.txt) - BINARY_BUCKET="gs://kubernetes-release/release/${RELEASE}/bin/linux/amd64" + BINARY_BUCKET="gs://kubernetes-release/release/${RELEASE}/bin/linux/${GOARCH}" INSTALL_APT=true else # Assume an exact "git describe" version/commit reference like "v1.12.0-alpha.0-1035-gf2dec305ad" - BINARY_BUCKET="gs://kubernetes-release-dev/ci/${BINARY_REF}-bazel/bin/linux/amd64" + BINARY_BUCKET="gs://kubernetes-release-dev/ci/${BINARY_REF}-bazel/bin/linux/${GOARCH}" fi # Download the debs and kubeadm @@ -155,3 +157,4 @@ crictl --version if [[ ${CLEANUP} == 1 ]]; then cleanup fi + \ No newline at end of file From 83ae8f5e62d603187c05617c363a7bfa32bbcaa0 Mon Sep 17 00:00:00 2001 From: leigh capili Date: Tue, 2 Jun 2020 07:07:08 -0600 Subject: [PATCH 2/2] Add load into containerd for image Make --- images/Makefile | 66 ++++++++++++++++++++++++++++++++++++------ images/kernel/Makefile | 34 +++++++++++++++++++++- 2 files changed, 90 insertions(+), 10 deletions(-) diff --git a/images/Makefile b/images/Makefile index f16ca5da8..ed1dcebb9 100644 --- a/images/Makefile +++ b/images/Makefile @@ -1,3 +1,30 @@ +SHELL:=/bin/bash +# Set the command for running `docker` +# -- allows user to override for things like sudo usage or container images +DOCKER := docker +# Set the first containerd.sock that successfully stats -- fallback to the docker4mac default +CONTAINERD_SOCK := $(shell \ + $(DOCKER) run -i --rm \ + -v /run:/run:ro \ + -v /var/run:/var/run:ro \ + busybox:latest \ + ls 2>/dev/null \ + /run/containerd/containerd.sock \ + /run/docker/containerd/containerd.sock \ + /var/run/containerd/containerd.sock \ + /var/run/docker/containerd/containerd.sock \ + | head -n1 \ + || echo \ + /var/run/docker/containerd/containerd.sock \ + ) +# Set the command for running `ctr` +# Use root inside a container with the host containerd socket +# This is a form of privilege escalation that avoids interactive sudo during make +CTR := $(DOCKER) run -i --rm \ + -v $(CONTAINERD_SOCK):/run/containerd/containerd.sock \ + linuxkit/containerd:751de142273e1b5d2d247d2832d654ab92e907bc \ + ctr + # WHAT specifies the OS image to build WHAT?= IS_LATEST?= @@ -30,7 +57,10 @@ DOCKERARCH=arm64v8 QEMUARCH=aarch64 endif -all: build + +all: build-all + + build: ifeq ($(WHAT),) $(error WHAT is a required argument) @@ -39,7 +69,7 @@ endif ifeq ($(IS_MANIFEST_LIST),0) sed "s|DOCKERARCH|$(DOCKERARCH)|g;/QEMUARCH/d" ${WHAT}/Dockerfile \ - | docker build \ + | $(DOCKER) build \ --build-arg RELEASE \ --build-arg BINARY_REF \ --build-arg GOARCH \ @@ -48,7 +78,7 @@ ifeq ($(IS_MANIFEST_LIST),0) ${WHAT} else # Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel - docker run --rm --privileged multiarch/qemu-user-static:register --reset + $(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset cp -a ${WHAT}/. $(BUILD_TMPDIR)/ sed "s|QEMUARCH|$(QEMUARCH)|g;s|DOCKERARCH|$(DOCKERARCH)|g" ${WHAT}/Dockerfile > $(BUILD_TMPDIR)/Dockerfile @@ -59,7 +89,7 @@ else cp ../bin/$(GOARCH)/qemu-$(QEMUARCH)-static $(BUILD_TMPDIR) endif - docker build \ + $(DOCKER) build \ --build-arg RELEASE \ --build-arg BINARY_REF \ --build-arg GOARCH \ @@ -68,17 +98,29 @@ endif find $(BUILD_TMPDIR)/ -mindepth 1 -delete endif - docker tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(RELEASE_IMAGE_NAME)$(ARCH_TAG) + $(DOCKER) tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(RELEASE_IMAGE_NAME)$(ARCH_TAG) ifeq ($(IS_LATEST),true) - docker tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(LATEST_IMAGE_NAME)$(ARCH_TAG) + $(DOCKER) tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(LATEST_IMAGE_NAME)$(ARCH_TAG) endif + +ctr-import: + $(DOCKER) image save $(FULL_IMAGE_NAME)$(ARCH_TAG) \ + | $(CTR) -n firecracker image import - + $(DOCKER) image save $(RELEASE_IMAGE_NAME)$(ARCH_TAG) \ + | $(CTR) -n firecracker image import - +ifeq ($(IS_LATEST),true) + $(DOCKER) image save $(LATEST_IMAGE_NAME)$(ARCH_TAG) \ + | $(CTR) -n firecracker image import - +endif + + push: ifeq ($(IS_MANIFEST_LIST),0) - docker push $(FULL_IMAGE_NAME) - docker push $(RELEASE_IMAGE_NAME) + $(DOCKER) push $(FULL_IMAGE_NAME) + $(DOCKER) push $(RELEASE_IMAGE_NAME) ifeq ($(IS_LATEST),true) - docker push $(LATEST_IMAGE_NAME) + $(DOCKER) push $(LATEST_IMAGE_NAME) endif else ifeq ($(GOARCH),amd64) @@ -90,9 +132,15 @@ endif endif endif + +ctr-import-all: + $(MAKE) OP=ctr-import build-all + + push-all: build-all $(MAKE) OP=push build-all + build-all: $(MAKE) ${OP} WHAT=amazon-kernel $(MAKE) ${OP} WHAT=amazonlinux RELEASE=2 IS_LATEST=true diff --git a/images/kernel/Makefile b/images/kernel/Makefile index 1acaad853..0a0bc31da 100644 --- a/images/kernel/Makefile +++ b/images/kernel/Makefile @@ -1,3 +1,30 @@ +SHELL:=/bin/bash +# Set the command for running `docker` +# -- allows user to override for things like sudo usage or container images +DOCKER := docker +# Set the first containerd.sock that successfully stats -- fallback to the docker4mac default +CONTAINERD_SOCK := $(shell \ + $(DOCKER) run -i --rm \ + -v /run:/run:ro \ + -v /var/run:/var/run:ro \ + busybox:latest \ + ls 2>/dev/null \ + /run/containerd/containerd.sock \ + /run/docker/containerd/containerd.sock \ + /var/run/containerd/containerd.sock \ + /var/run/docker/containerd/containerd.sock \ + | head -n1 \ + || echo \ + /var/run/docker/containerd/containerd.sock \ + ) +# Set the command for running `ctr` +# Use root inside a container with the host containerd socket +# This is a form of privilege escalation that avoids interactive sudo during make +CTR := $(DOCKER) run -i --rm \ + -v $(CONTAINERD_SOCK):/run/containerd/containerd.sock \ + linuxkit/containerd:751de142273e1b5d2d247d2832d654ab92e907bc \ + ctr + # Check https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/refs/ for updates REGISTRY?=weaveworks IMAGE_NAME?=${REGISTRY}/ignite-kernel @@ -28,13 +55,18 @@ upgrade-%: build: $(addprefix build-,$(KERNEL_VERSIONS)) build-%: - docker build -t $(IMAGE_NAME):$*-${GOARCH} \ + $(DOCKER) build -t $(IMAGE_NAME):$*-${GOARCH} \ --build-arg KERNEL_VERSION=$* \ --build-arg ARCH=${KERNEL_ARCH} \ --build-arg GOARCH=${GOARCH} \ --build-arg ARCH_MAKE_PARAMS=${ARCH_MAKE_PARAMS} \ --build-arg VMLINUX_PATH=${VMLINUX_PATH} . +ctr-import: $(addprefix ctr-import-,$(KERNEL_VERSIONS)) +ctr-import-%: + $(DOCKER) image save $(IMAGE_NAME):$*-${GOARCH} \ + | $(CTR) -n firecracker image import - + push: $(addprefix push-,$(KERNEL_VERSIONS)) push-%: ../../hack/push-manifest-list.sh $(IMAGE_NAME):$* $(GOARCH_LIST)