Fix image builds + publish new kernel configs

images/Makefile

@@ -1,3 +1,30 @@
+SHELL:=/bin/bash
+# Set the command for running `docker`
+# -- allows user to override for things like sudo usage or container images 
+DOCKER := docker
+# Set the first containerd.sock that successfully stats -- fallback to the docker4mac default
+CONTAINERD_SOCK := $(shell \
+ $(DOCKER) run -i --rm \
+ -v /run:/run:ro \
+ -v /var/run:/var/run:ro \
+ busybox:latest \
+ ls 2>/dev/null \
+ /run/containerd/containerd.sock \
+ /run/docker/containerd/containerd.sock \
+ /var/run/containerd/containerd.sock \
+ /var/run/docker/containerd/containerd.sock \
+ | head -n1 \
+ || echo \
+ /var/run/docker/containerd/containerd.sock \
+ )
+# Set the command for running `ctr`
+# Use root inside a container with the host containerd socket
+# This is a form of privilege escalation that avoids interactive sudo during make
+CTR := $(DOCKER) run -i --rm \
+ -v $(CONTAINERD_SOCK):/run/containerd/containerd.sock \
+ linuxkit/containerd:751de142273e1b5d2d247d2832d654ab92e907bc \
+ ctr
+
 # WHAT specifies the OS image to build
 WHAT?=
 IS_LATEST?=
@@ -15,8 +42,10 @@ IS_MANIFEST_LIST?=0
 TAG:=${RELEASE}$(if $(strip $(VERSION)),-${VERSION})
 OP:=build
 
+TMPDIR?=/tmp
+
 ifeq ($(IS_MANIFEST_LIST),1)
-TEMP_DIR:=$(shell mktemp -d)
+BUILD_TMPDIR:=$(shell mktemp -d $(TMPDIR)/ignite-image-build.XXXXXXXXXX)
 ARCH_TAG=-$(GOARCH)
 endif
 
@@ -28,39 +57,70 @@ DOCKERARCH=arm64v8
 QEMUARCH=aarch64
 endif
 
-all: build
+
+all: build-all
+
+
 build:
 ifeq ($(WHAT),)
  $(error WHAT is a required argument)
 endif
  @ls ${WHAT} >/dev/null
 
 ifeq ($(IS_MANIFEST_LIST),0)
- sed "s|DOCKERARCH|$(DOCKERARCH)|g;/QEMUARCH/d" ${WHAT}/Dockerfile | docker build --build-arg RELEASE -f - -t $(FULL_IMAGE_NAME) ${WHAT}
+ sed "s|DOCKERARCH|$(DOCKERARCH)|g;/QEMUARCH/d" ${WHAT}/Dockerfile \
+ | $(DOCKER) build \
+ --build-arg RELEASE \
+ --build-arg BINARY_REF \
+ --build-arg GOARCH \
+ -f -\
+ -t $(FULL_IMAGE_NAME) \
+ ${WHAT}
 else
  # Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel
- docker run --rm --privileged multiarch/qemu-user-static:register --reset
- sed "s|QEMUARCH|$(QEMUARCH)|g;s|DOCKERARCH|$(DOCKERARCH)|g" ${WHAT}/Dockerfile > $(TEMP_DIR)/Dockerfile
+ $(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset
+ cp -a ${WHAT}/. $(BUILD_TMPDIR)/
+ sed "s|QEMUARCH|$(QEMUARCH)|g;s|DOCKERARCH|$(DOCKERARCH)|g" ${WHAT}/Dockerfile > $(BUILD_TMPDIR)/Dockerfile
+
 ifeq ($(GOARCH),amd64)
- sed "/COPY qemu/d" -i $(TEMP_DIR)/Dockerfile
+ sed "/COPY qemu/d" -i $(BUILD_TMPDIR)/Dockerfile
 else
  $(MAKE) -C .. qemu
- cp ../bin/$(GOARCH)/qemu-$(QEMUARCH)-static $(TEMP_DIR)
+ cp ../bin/$(GOARCH)/qemu-$(QEMUARCH)-static $(BUILD_TMPDIR)
+endif
+
+ $(DOCKER) build \
+ --build-arg RELEASE \
+ --build-arg BINARY_REF \
+ --build-arg GOARCH \
+ -t $(FULL_IMAGE_NAME)$(ARCH_TAG) \
+ $(BUILD_TMPDIR)
+ find $(BUILD_TMPDIR)/ -mindepth 1 -delete
 endif
- docker build --build-arg RELEASE -t $(FULL_IMAGE_NAME)$(ARCH_TAG) $(TEMP_DIR)
+
+ $(DOCKER) tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(RELEASE_IMAGE_NAME)$(ARCH_TAG)
+ifeq ($(IS_LATEST),true)
+ $(DOCKER) tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(LATEST_IMAGE_NAME)$(ARCH_TAG)
 endif
 
- docker tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(RELEASE_IMAGE_NAME)$(ARCH_TAG)
+
+ctr-import:
+ $(DOCKER) image save $(FULL_IMAGE_NAME)$(ARCH_TAG) \
+ | $(CTR) -n firecracker image import -
+ $(DOCKER) image save $(RELEASE_IMAGE_NAME)$(ARCH_TAG) \
+ | $(CTR) -n firecracker image import -
 ifeq ($(IS_LATEST),true)
- docker tag $(FULL_IMAGE_NAME)$(ARCH_TAG) $(LATEST_IMAGE_NAME)$(ARCH_TAG)
+ $(DOCKER) image save $(LATEST_IMAGE_NAME)$(ARCH_TAG) \
+ | $(CTR) -n firecracker image import -
 endif
 
+
 push:
 ifeq ($(IS_MANIFEST_LIST),0)
- docker push $(FULL_IMAGE_NAME)
- docker push $(RELEASE_IMAGE_NAME)
+ $(DOCKER) push $(FULL_IMAGE_NAME)
+ $(DOCKER) push $(RELEASE_IMAGE_NAME)
 ifeq ($(IS_LATEST),true)
- docker push $(LATEST_IMAGE_NAME)
+ $(DOCKER) push $(LATEST_IMAGE_NAME)
 endif
 else
 ifeq ($(GOARCH),amd64)
@@ -72,20 +132,27 @@ endif
 endif
 endif
 
+
+ctr-import-all:
+ $(MAKE) OP=ctr-import build-all
+
+
 push-all: build-all
  $(MAKE) OP=push build-all
 
+
 build-all:
  $(MAKE) ${OP} WHAT=amazon-kernel
- $(MAKE) ${OP} WHAT=amazonlinux RELEASE=2 IS_LATEST=true
+ $(MAKE) ${OP} WHAT=amazonlinux  RELEASE=2  IS_LATEST=true
  $(MAKE) ${OP} WHAT=alpine
- $(MAKE) ${OP} WHAT=opensuse RELEASE=leap IS_LATEST=true
- $(MAKE) ${OP} WHAT=opensuse RELEASE=tumbleweed
- $(MAKE) ${OP} WHAT=ubuntu RELEASE=16.04 IS_MANIFEST_LIST=0
- $(MAKE) ${OP} WHAT=ubuntu RELEASE=18.04 IS_MANIFEST_LIST=1 GOARCH=arm64
- $(MAKE) ${OP} WHAT=ubuntu RELEASE=18.04 IS_MANIFEST_LIST=1 GOARCH=amd64
- $(MAKE) ${OP} WHAT=ubuntu RELEASE=20.04 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=arm64
- $(MAKE) ${OP} WHAT=ubuntu RELEASE=20.04 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=amd64
- $(MAKE) ${OP} WHAT=centos RELEASE=7
- $(MAKE) ${OP} WHAT=centos RELEASE=8 IS_LATEST=true
- $(MAKE) ${OP} WHAT=kubeadm
+ $(MAKE) ${OP} WHAT=opensuse RELEASE=leap IS_LATEST=true
+ $(MAKE) ${OP} WHAT=opensuse RELEASE=tumbleweed
+ $(MAKE) ${OP} WHAT=ubuntu RELEASE=16.04 IS_MANIFEST_LIST=0
+ $(MAKE) ${OP} WHAT=ubuntu RELEASE=18.04 IS_MANIFEST_LIST=1 GOARCH=arm64
+ $(MAKE) ${OP} WHAT=ubuntu RELEASE=18.04 IS_MANIFEST_LIST=1 GOARCH=amd64
+ $(MAKE) ${OP} WHAT=ubuntu RELEASE=20.04 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=arm64
+ $(MAKE) ${OP} WHAT=ubuntu RELEASE=20.04 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=amd64
+ $(MAKE) ${OP} WHAT=centos RELEASE=7
+ $(MAKE) ${OP} WHAT=centos RELEASE=8 IS_LATEST=true
+ $(MAKE) ${OP} WHAT=kubeadm RELEASE=v1.18.3 BINARY_REF=release/stable-1.18 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=arm64
+ $(MAKE) ${OP} WHAT=kubeadm RELEASE=v1.18.3 BINARY_REF=release/stable-1.18 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=amd64

images/kernel/Makefile

@@ -1,3 +1,30 @@
+SHELL:=/bin/bash
+# Set the command for running `docker`
+# -- allows user to override for things like sudo usage or container images 
+DOCKER := docker
+# Set the first containerd.sock that successfully stats -- fallback to the docker4mac default
+CONTAINERD_SOCK := $(shell \
+ $(DOCKER) run -i --rm \
+ -v /run:/run:ro \
+ -v /var/run:/var/run:ro \
+ busybox:latest \
+ ls 2>/dev/null \
+ /run/containerd/containerd.sock \
+ /run/docker/containerd/containerd.sock \
+ /var/run/containerd/containerd.sock \
+ /var/run/docker/containerd/containerd.sock \
+ | head -n1 \
+ || echo \
+ /var/run/docker/containerd/containerd.sock \
+ )
+# Set the command for running `ctr`
+# Use root inside a container with the host containerd socket
+# This is a form of privilege escalation that avoids interactive sudo during make
+CTR := $(DOCKER) run -i --rm \
+ -v $(CONTAINERD_SOCK):/run/containerd/containerd.sock \
+ linuxkit/containerd:751de142273e1b5d2d247d2832d654ab92e907bc \
+ ctr
+
 # Check https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/refs/ for updates
 REGISTRY?=weaveworks
 IMAGE_NAME?=${REGISTRY}/ignite-kernel
@@ -28,13 +55,18 @@ upgrade-%:
 
 build: $(addprefix build-,$(KERNEL_VERSIONS))
 build-%:
- docker build -t $(IMAGE_NAME):$*-${GOARCH} \
+ $(DOCKER) build -t $(IMAGE_NAME):$*-${GOARCH} \
  --build-arg KERNEL_VERSION=$* \
  --build-arg ARCH=${KERNEL_ARCH} \
  --build-arg GOARCH=${GOARCH} \
  --build-arg ARCH_MAKE_PARAMS=${ARCH_MAKE_PARAMS} \
  --build-arg VMLINUX_PATH=${VMLINUX_PATH} .
 
+ctr-import: $(addprefix ctr-import-,$(KERNEL_VERSIONS))
+ctr-import-%:
+ $(DOCKER) image save $(IMAGE_NAME):$*-${GOARCH} \
+ | $(CTR) -n firecracker image import -
+
 push: $(addprefix push-,$(KERNEL_VERSIONS))
 push-%:
  ../../hack/push-manifest-list.sh $(IMAGE_NAME):$* $(GOARCH_LIST)

images/kubeadm/Dockerfile

@@ -1,5 +1,11 @@
+ARG GOARCH="amd64"
+
 # Ubuntu 20.04 was also tested, but didn't perform very well (sshd took a long time to start), so we're sticking with Ubuntu 18.04 still
 FROM weaveworks/ignite-ubuntu:18.04
+ARG GOARCH="amd64"
+ARG RELEASE
+ARG BINARY_REF
+
 # Install dependencies. Use containerd for running the containers (for better performance)
 RUN apt-get update && apt-get install -y --no-install-recommends \
  apt-transport-https \
@@ -9,10 +15,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  jq \
  && apt-get clean
 
-# Install k8s v1.18.1 locally
-ENV KUBERNETES_VERSION=v1.18.3
-COPY install.sh /
-RUN /install.sh install release/stable-1.18 ${KUBERNETES_VERSION}
+# Install k8s locally
+COPY ./install.sh /
+RUN /install.sh install "${BINARY_REF}" "${RELEASE}" "${GOARCH}"
 # Docker sets this automatically, but not containerd.
 # It is required when running kubeadm.
 RUN echo "net.ipv4.ip_forward=1" > /etc/sysctl.conf

images/kubeadm/install.sh

@@ -8,17 +8,19 @@ export KUBECONFIG=/etc/kubernetes/admin.conf
 MODE=${1}
 BINARY_REF=${2}
 CONTROL_PLANE_VERSION=${3}
+GOARCH=${4}
 DRY_RUN=${DRY_RUN:-0}
 CLEANUP=${CLEANUP:-0}
 
-if [[ $# != 3 && ${MODE} != "cleanup" ]]; then
+if [[ $# != 4 && ${MODE} != "cleanup" ]]; then
  cat <<-EOF
  Usage:
  ${0} MODE BINARY_REF CONTROL_PLANE_VERSION
 
  MODE=install|init|upgrade|cleanup
  BINARY_REF=What kubeadm binary and debs to pull. Can be a PR number, version label like "ci/latest" or exact (merged) commit like "v1.12.0-alpha.0-1035-gf2dec305ad"
  CONTROL_PLANE_VERSION=For init, this is the control plane version to use. For upgrade, this is the version to upgrade to
+ GOARCH=amd64|arm|arm64|ppc64le|s390x
  EOF
  exit 1
 fi
@@ -61,17 +63,17 @@ if [[ "${BINARY_REF}" =~ ^[0-9]{5}$ ]]; then
  BUILD_NUMBER=$(gsutil_cat gs://kubernetes-jenkins/pr-logs/pull/${PR_NUMBER}/pull-kubernetes-bazel-build/latest-build.txt)
  BAZEL_PULL_REF=$(gsutil_cat gs://kubernetes-jenkins/pr-logs/pull/${PR_NUMBER}/pull-kubernetes-bazel-build/${BUILD_NUMBER}/started.json | jq -r .pull)
  BAZEL_BUILD_LOCATION=$(gsutil_cat gs://kubernetes-jenkins/shared-results/${BAZEL_PULL_REF}/bazel-build-location.txt)
- BINARY_BUCKET="${BAZEL_BUILD_LOCATION}/bin/linux/amd64"
+ BINARY_BUCKET="${BAZEL_BUILD_LOCATION}/bin/linux/${GOARCH}"
 elif [[ "${BINARY_REF}" =~ ^(ci|ci-cross){1}/latest ]]; then
  COMMIT=$(curl -sSL https://dl.k8s.io/${BINARY_REF}.txt)
- BINARY_BUCKET="gs://kubernetes-release-dev/ci/${COMMIT}-bazel/bin/linux/amd64"
+ BINARY_BUCKET="gs://kubernetes-release-dev/ci/${COMMIT}-bazel/bin/linux/${GOARCH}"
 elif [[ "${BINARY_REF}" =~ ^release/[a-z]+(-[0-9]+.[0-9]+)*$ ]]; then
  RELEASE=$(curl -sSL https://dl.k8s.io/${BINARY_REF}.txt)
- BINARY_BUCKET="gs://kubernetes-release/release/${RELEASE}/bin/linux/amd64"
+ BINARY_BUCKET="gs://kubernetes-release/release/${RELEASE}/bin/linux/${GOARCH}"
  INSTALL_APT=true
 else
  # Assume an exact "git describe" version/commit reference like "v1.12.0-alpha.0-1035-gf2dec305ad"
- BINARY_BUCKET="gs://kubernetes-release-dev/ci/${BINARY_REF}-bazel/bin/linux/amd64"
+ BINARY_BUCKET="gs://kubernetes-release-dev/ci/${BINARY_REF}-bazel/bin/linux/${GOARCH}"
 fi
 
 # Download the debs and kubeadm
@@ -155,3 +157,4 @@ crictl --version
 if [[ ${CLEANUP} == 1 ]]; then
  cleanup
 fi
+