Fix image builds + publish new kernel configs

images/Makefile

@@ -1,3 +1,30 @@
+SHELL:=/bin/bash
+# Set the command for running `docker`
+# -- allows user to override for things like sudo usage or container images 
+DOCKER := docker
+# Set the first containerd.sock that successfully stats -- fallback to the docker4mac default
+CONTAINERD_SOCK := $(shell \
+	$(DOCKER) run -i --rm \
+		-v /run:/run:ro \
+		-v /var/run:/var/run:ro \
+		busybox:latest \
+		ls 2>/dev/null \
+		/run/containerd/containerd.sock \
+		/run/docker/containerd/containerd.sock \
+		/var/run/containerd/containerd.sock \
+		/var/run/docker/containerd/containerd.sock \
+		| head -n1 \
+		|| echo \
+			/var/run/docker/containerd/containerd.sock \
+	)
+# Set the command for running `ctr`
+# Use root inside a container with the host containerd socket
+# This is a form of privilege escalation that avoids interactive sudo during make
+CTR := $(DOCKER) run -i --rm \
+		-v $(CONTAINERD_SOCK):/run/containerd/containerd.sock \
+		linuxkit/containerd:751de142273e1b5d2d247d2832d654ab92e907bc \
+		ctr
+
 # WHAT specifies the OS image to build
 WHAT?=
 IS_LATEST?=
@@ -15,8 +42,10 @@ IS_MANIFEST_LIST?=0
 TAG:=${RELEASE}$(if $(strip $(VERSION)),-${VERSION})
 OP:=build
 
+TMPDIR?=/tmp
+
 ifeq ($(IS_MANIFEST_LIST),1)
-TEMP_DIR:=$(shell mktemp -d)
+BUILD_TMPDIR:=$(shell mktemp -d $(TMPDIR)/ignite-image-build.XXXXXXXXXX)
 ARCH_TAG=-$(GOARCH)
 endif
 
@@ -28,39 +57,70 @@ DOCKERARCH=arm64v8
 QEMUARCH=aarch64
 endif
 
-all: build
+
+all: build-all
+
+
 build:
 ifeq ($(WHAT),)
 	$(error WHAT is a required argument)
 endif
 	@ls ${WHAT} >/dev/null
 
 ifeq ($(IS_MANIFEST_LIST),0)
-	sed "s|DOCKERARCH|$(DOCKERARCH)|g;/QEMUARCH/d" ${WHAT}/Dockerfile | docker build --build-arg RELEASE -f - -t $(FULL_IMAGE_NAME) ${WHAT}
+	sed "s|DOCKERARCH|$(DOCKERARCH)|g;/QEMUARCH/d" ${WHAT}/Dockerfile \
+		| $(DOCKER) build \
+			--build-arg RELEASE \
+			--build-arg BINARY_REF \
+			--build-arg GOARCH \
+			-f -\
+			-t $(FULL_IMAGE_NAME) \
+			${WHAT}
 else
 	# Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel
-	docker run --rm --privileged multiarch/qemu-user-static:register --reset
-	sed "s|QEMUARCH|$(QEMUARCH)|g;s|DOCKERARCH|$(DOCKERARCH)|g" ${WHAT}/Dockerfile > $(TEMP_DIR)/Dockerfile
+	$(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset
+	cp -a ${WHAT}/. $(BUILD_TMPDIR)/
+	sed "s|QEMUARCH|$(QEMUARCH)|g;s|DOCKERARCH|$(DOCKERARCH)|g" ${WHAT}/Dockerfile > $(BUILD_TMPDIR)/Dockerfile
+
 ifeq ($(GOARCH),amd64)
-	sed "/COPY qemu/d" -i $(TEMP_DIR)/Dockerfile
+	sed "/COPY qemu/d" -i $(BUILD_TMPDIR)/Dockerfile
 else
 	$(MAKE) -C .. qemu
-	cp ../bin/$(GOARCH)/qemu-$(QEMUARCH)-static $(TEMP_DIR)
+	cp ../bin/$(GOARCH)/qemu-$(QEMUARCH)-static $(BUILD_TMPDIR)
+endif
+
+	$(DOCKER) build \
+		--build-arg RELEASE \
+		--build-arg BINARY_REF \
+		--build-arg GOARCH \
+		-t $(FULL_IMAGE_NAME)$(ARCH_TAG) \
+		$(BUILD_TMPDIR)
+	find $(BUILD_TMPDIR)/ -mindepth 1 -delete
 endif
-	docker build --build-arg RELEASE -t $(FULL_IMAGE_NAME)$(ARCH_TAG) $(TEMP_DIR)
+
+	$(DOCKER) tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(RELEASE_IMAGE_NAME)$(ARCH_TAG)
+ifeq ($(IS_LATEST),true)
+	$(DOCKER) tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(LATEST_IMAGE_NAME)$(ARCH_TAG)
 endif
 
-	docker tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(RELEASE_IMAGE_NAME)$(ARCH_TAG)
+
+ctr-import:
+	$(DOCKER) image save $(FULL_IMAGE_NAME)$(ARCH_TAG) \
+		| $(CTR) -n firecracker image import -
+	$(DOCKER) image save $(RELEASE_IMAGE_NAME)$(ARCH_TAG) \
+		| $(CTR) -n firecracker image import -
 ifeq ($(IS_LATEST),true)
-	docker tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(LATEST_IMAGE_NAME)$(ARCH_TAG)
+	$(DOCKER) image save $(LATEST_IMAGE_NAME)$(ARCH_TAG) \
+		| $(CTR) -n firecracker image import -
 endif
 
+
 push:
 ifeq ($(IS_MANIFEST_LIST),0)
-	docker push $(FULL_IMAGE_NAME)
-	docker push $(RELEASE_IMAGE_NAME)
+	$(DOCKER) push $(FULL_IMAGE_NAME)
+	$(DOCKER) push $(RELEASE_IMAGE_NAME)
 ifeq ($(IS_LATEST),true)
-	docker push $(LATEST_IMAGE_NAME)
+	$(DOCKER) push $(LATEST_IMAGE_NAME)
 endif
 else
 ifeq ($(GOARCH),amd64)
@@ -72,20 +132,27 @@ endif
 endif
 endif
 
+
+ctr-import-all:
+	$(MAKE) OP=ctr-import build-all
+
+
 push-all: build-all
 	$(MAKE) OP=push build-all
 
+
 build-all:
 	$(MAKE) ${OP} WHAT=amazon-kernel
-	$(MAKE) ${OP} WHAT=amazonlinux   RELEASE=2          IS_LATEST=true
+	$(MAKE) ${OP} WHAT=amazonlinux    RELEASE=2                                        IS_LATEST=true
 	$(MAKE) ${OP} WHAT=alpine
-	$(MAKE) ${OP} WHAT=opensuse      RELEASE=leap       IS_LATEST=true
-	$(MAKE) ${OP} WHAT=opensuse      RELEASE=tumbleweed
-	$(MAKE) ${OP} WHAT=ubuntu        RELEASE=16.04						IS_MANIFEST_LIST=0
-	$(MAKE) ${OP} WHAT=ubuntu        RELEASE=18.04      				IS_MANIFEST_LIST=1 GOARCH=arm64
-	$(MAKE) ${OP} WHAT=ubuntu        RELEASE=18.04      				IS_MANIFEST_LIST=1 GOARCH=amd64
-	$(MAKE) ${OP} WHAT=ubuntu        RELEASE=20.04      IS_LATEST=true	IS_MANIFEST_LIST=1 GOARCH=arm64
-	$(MAKE) ${OP} WHAT=ubuntu        RELEASE=20.04      IS_LATEST=true	IS_MANIFEST_LIST=1 GOARCH=amd64
-	$(MAKE) ${OP} WHAT=centos        RELEASE=7
-	$(MAKE) ${OP} WHAT=centos        RELEASE=8          IS_LATEST=true
-	$(MAKE) ${OP} WHAT=kubeadm
+	$(MAKE) ${OP} WHAT=opensuse       RELEASE=leap                                     IS_LATEST=true
+	$(MAKE) ${OP} WHAT=opensuse       RELEASE=tumbleweed
+	$(MAKE) ${OP} WHAT=ubuntu         RELEASE=16.04                                                    IS_MANIFEST_LIST=0
+	$(MAKE) ${OP} WHAT=ubuntu         RELEASE=18.04                                                    IS_MANIFEST_LIST=1  GOARCH=arm64
+	$(MAKE) ${OP} WHAT=ubuntu         RELEASE=18.04                                                    IS_MANIFEST_LIST=1  GOARCH=amd64
+	$(MAKE) ${OP} WHAT=ubuntu         RELEASE=20.04                                    IS_LATEST=true  IS_MANIFEST_LIST=1  GOARCH=arm64
+	$(MAKE) ${OP} WHAT=ubuntu         RELEASE=20.04                                    IS_LATEST=true  IS_MANIFEST_LIST=1  GOARCH=amd64
+	$(MAKE) ${OP} WHAT=centos         RELEASE=7
+	$(MAKE) ${OP} WHAT=centos         RELEASE=8                                        IS_LATEST=true
+	$(MAKE) ${OP} WHAT=kubeadm        RELEASE=v1.18.3  BINARY_REF=release/stable-1.18  IS_LATEST=true  IS_MANIFEST_LIST=1  GOARCH=arm64
+	$(MAKE) ${OP} WHAT=kubeadm        RELEASE=v1.18.3  BINARY_REF=release/stable-1.18  IS_LATEST=true  IS_MANIFEST_LIST=1  GOARCH=amd64

images/kernel/Makefile

@@ -1,3 +1,30 @@
+SHELL:=/bin/bash
+# Set the command for running `docker`
+# -- allows user to override for things like sudo usage or container images 
+DOCKER := docker
+# Set the first containerd.sock that successfully stats -- fallback to the docker4mac default
+CONTAINERD_SOCK := $(shell \
+	$(DOCKER) run -i --rm \
+		-v /run:/run:ro \
+		-v /var/run:/var/run:ro \
+		busybox:latest \
+		ls 2>/dev/null \
+		/run/containerd/containerd.sock \
+		/run/docker/containerd/containerd.sock \
+		/var/run/containerd/containerd.sock \
+		/var/run/docker/containerd/containerd.sock \
+		| head -n1 \
+		|| echo \
+			/var/run/docker/containerd/containerd.sock \
+	)
+# Set the command for running `ctr`
+# Use root inside a container with the host containerd socket
+# This is a form of privilege escalation that avoids interactive sudo during make
+CTR := $(DOCKER) run -i --rm \
+		-v $(CONTAINERD_SOCK):/run/containerd/containerd.sock \
+		linuxkit/containerd:751de142273e1b5d2d247d2832d654ab92e907bc \
+		ctr
+
 # Check https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/refs/ for updates
 REGISTRY?=weaveworks
 IMAGE_NAME?=${REGISTRY}/ignite-kernel
@@ -28,13 +55,18 @@ upgrade-%:
 
 build: $(addprefix build-,$(KERNEL_VERSIONS))
 build-%:
-	docker build -t $(IMAGE_NAME):$*-${GOARCH} \
+	$(DOCKER) build -t $(IMAGE_NAME):$*-${GOARCH} \
 		--build-arg KERNEL_VERSION=$* \
 		--build-arg ARCH=${KERNEL_ARCH} \
 		--build-arg GOARCH=${GOARCH} \
 		--build-arg ARCH_MAKE_PARAMS=${ARCH_MAKE_PARAMS} \
 		--build-arg VMLINUX_PATH=${VMLINUX_PATH} .
 
+ctr-import: $(addprefix ctr-import-,$(KERNEL_VERSIONS))
+ctr-import-%:
+	$(DOCKER) image save $(IMAGE_NAME):$*-${GOARCH} \
+		| $(CTR) -n firecracker image import -
+
 push: $(addprefix push-,$(KERNEL_VERSIONS))
 push-%:
 	../../hack/push-manifest-list.sh $(IMAGE_NAME):$* $(GOARCH_LIST)

images/kubeadm/Dockerfile

@@ -1,5 +1,11 @@
+ARG GOARCH="amd64"
+
 # Ubuntu 20.04 was also tested, but didn't perform very well (sshd took a long time to start), so we're sticking with Ubuntu 18.04 still
 FROM weaveworks/ignite-ubuntu:18.04
+ARG GOARCH="amd64"
+ARG RELEASE
+ARG BINARY_REF
+
 # Install dependencies. Use containerd for running the containers (for better performance)
 RUN apt-get update && apt-get install -y --no-install-recommends \
         apt-transport-https \
@@ -9,10 +15,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         jq \
     && apt-get clean
 
-# Install k8s v1.18.1 locally
-ENV KUBERNETES_VERSION=v1.18.3
-COPY install.sh /
-RUN /install.sh install release/stable-1.18 ${KUBERNETES_VERSION}
+# Install k8s locally
+COPY ./install.sh /
+RUN /install.sh install "${BINARY_REF}" "${RELEASE}" "${GOARCH}"
 # Docker sets this automatically, but not containerd.
 # It is required when running kubeadm.
 RUN echo "net.ipv4.ip_forward=1" > /etc/sysctl.conf

images/kubeadm/install.sh

@@ -8,17 +8,19 @@ export KUBECONFIG=/etc/kubernetes/admin.conf
 MODE=${1}
 BINARY_REF=${2}
 CONTROL_PLANE_VERSION=${3}
+GOARCH=${4}
 DRY_RUN=${DRY_RUN:-0}
 CLEANUP=${CLEANUP:-0}
 
-if [[ $# != 3 && ${MODE} != "cleanup" ]]; then
+if [[ $# != 4 && ${MODE} != "cleanup" ]]; then
 	cat <<-EOF
 	Usage:
 		${0} MODE BINARY_REF CONTROL_PLANE_VERSION
 
 	MODE=install|init|upgrade|cleanup
 	BINARY_REF=What kubeadm binary and debs to pull. Can be a PR number, version label like "ci/latest" or exact (merged) commit like "v1.12.0-alpha.0-1035-gf2dec305ad"
 	CONTROL_PLANE_VERSION=For init, this is the control plane version to use. For upgrade, this is the version to upgrade to
+	GOARCH=amd64|arm|arm64|ppc64le|s390x
 	EOF
 	exit 1
 fi
@@ -61,17 +63,17 @@ if [[ "${BINARY_REF}" =~ ^[0-9]{5}$ ]]; then
 	BUILD_NUMBER=$(gsutil_cat gs://kubernetes-jenkins/pr-logs/pull/${PR_NUMBER}/pull-kubernetes-bazel-build/latest-build.txt)
 	BAZEL_PULL_REF=$(gsutil_cat gs://kubernetes-jenkins/pr-logs/pull/${PR_NUMBER}/pull-kubernetes-bazel-build/${BUILD_NUMBER}/started.json | jq -r .pull)
 	BAZEL_BUILD_LOCATION=$(gsutil_cat gs://kubernetes-jenkins/shared-results/${BAZEL_PULL_REF}/bazel-build-location.txt)
-	BINARY_BUCKET="${BAZEL_BUILD_LOCATION}/bin/linux/amd64"
+	BINARY_BUCKET="${BAZEL_BUILD_LOCATION}/bin/linux/${GOARCH}"
 elif [[ "${BINARY_REF}" =~ ^(ci|ci-cross){1}/latest ]]; then
 	COMMIT=$(curl -sSL https://dl.k8s.io/${BINARY_REF}.txt)
-	BINARY_BUCKET="gs://kubernetes-release-dev/ci/${COMMIT}-bazel/bin/linux/amd64"
+	BINARY_BUCKET="gs://kubernetes-release-dev/ci/${COMMIT}-bazel/bin/linux/${GOARCH}"
 elif [[ "${BINARY_REF}" =~ ^release/[a-z]+(-[0-9]+.[0-9]+)*$ ]]; then
 	RELEASE=$(curl -sSL https://dl.k8s.io/${BINARY_REF}.txt)
-	BINARY_BUCKET="gs://kubernetes-release/release/${RELEASE}/bin/linux/amd64"
+	BINARY_BUCKET="gs://kubernetes-release/release/${RELEASE}/bin/linux/${GOARCH}"
 	INSTALL_APT=true
 else
 	# Assume an exact "git describe" version/commit reference like "v1.12.0-alpha.0-1035-gf2dec305ad"
-	BINARY_BUCKET="gs://kubernetes-release-dev/ci/${BINARY_REF}-bazel/bin/linux/amd64"
+	BINARY_BUCKET="gs://kubernetes-release-dev/ci/${BINARY_REF}-bazel/bin/linux/${GOARCH}"
 fi
 
 # Download the debs and kubeadm
@@ -155,3 +157,4 @@ crictl --version
 if [[ ${CLEANUP} == 1 ]]; then
 	cleanup
 fi
+