diff --git a/images/kernel/Dockerfile b/images/kernel/Dockerfile
index 3f03b0eb6..b7c192269 100644
--- a/images/kernel/Dockerfile
+++ b/images/kernel/Dockerfile
@@ -1,14 +1,11 @@
-FROM luxas/kernel-builder:gcc-7 AS builder
+FROM weaveworks/ignite-kernel-builder:dev AS builder
 
-# Install crosscompilers for non-amd64 arches
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        binutils-multiarch \
-        gcc-7-aarch64-linux-gnu && \
-	ln -s /usr/bin/aarch64-linux-gnu-gcc-7 /usr/bin/aarch64-linux-gnu-gcc
-
-# Fetch the latest tags from the linux kernel repo 
-RUN git fetch --tags
+# Set up environment variables
+ENV CCACHE_DIR=/ccache       \
+    SRC_DIR=/usr/src         \
+    DIST_DIR=/dist           \
+    LINUX_DIR=/usr/src/linux \
+    LINUX_REPO_URL=git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
 
 ARG KERNEL_VERSION
 ARG KERNEL_EXTRA
@@ -17,9 +14,13 @@ ARG ARCH
 ARG GOARCH
 ARG ARCH_MAKE_PARAMS
 
-# Checkout the desired kernel version and make sure the environment is clean
-RUN git checkout v${KERNEL_VERSION} && \
-    make clean && make mrproper
+# Clone the desired kernel version and make sure the environment is clean
+RUN mkdir -p ${SRC_DIR} ${CCACHE_DIR} ${DIST_DIR} && \
+    git clone --depth 1 --branch v${KERNEL_VERSION} ${LINUX_REPO_URL} ${LINUX_DIR} && \
+    cd ${LINUX_DIR} && make clean && make mrproper
+
+# Change workdir to run all the build commands from LINUX_DIR.
+WORKDIR ${LINUX_DIR}
 
 COPY generated/config-${GOARCH}-${KERNEL_VERSION}${KERNEL_EXTRA} .config
 
diff --git a/images/kernel/Makefile b/images/kernel/Makefile
index bb518a87a..60aecfc7e 100644
--- a/images/kernel/Makefile
+++ b/images/kernel/Makefile
@@ -27,6 +27,7 @@ CTR := $(DOCKER) run -i --rm \
 
 REGISTRY?=weaveworks
 IMAGE_NAME?=${REGISTRY}/ignite-kernel
+KERNEL_BUILDER_NAME=${REGISTRY}/ignite-kernel-builder
 # Check https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/refs/ for updates
 KERNEL_VERSIONS ?= 4.14.182 4.19.125 5.4.43  # If you update this, please keep the .github/workflows/release-kernel-images.yml matrix up to date
 GOARCH?=amd64
@@ -44,17 +45,21 @@ endif
 
 all: build
 
+kernel-builder:
+	$(DOCKER) build -t ${KERNEL_BUILDER_NAME}:dev \
+		-f builder/Dockerfile .
+
 upgrade: $(addprefix upgrade-,$(KERNEL_VERSIONS))
 	./patch-config.sh
 	for file in generated/*; do \
 		./upgrade-config.sh $$file $$file; done
 
-upgrade-%:
+upgrade-%: kernel-builder
 	for file in upstream/*; do \
 		./upgrade-config.sh $$file versioned/$$(basename $$file)-$*; done
 
 build: $(addprefix build-,$(KERNEL_VERSIONS))
-build-%:
+build-%: kernel-builder
 	$(DOCKER) build -t $(IMAGE_NAME):$*-${GOARCH} \
 		--build-arg KERNEL_VERSION=$* \
 		--build-arg ARCH=${KERNEL_ARCH} \
diff --git a/images/kernel/builder/Dockerfile b/images/kernel/builder/Dockerfile
new file mode 100644
index 000000000..2522e07a6
--- /dev/null
+++ b/images/kernel/builder/Dockerfile
@@ -0,0 +1,27 @@
+FROM ubuntu:18.04 AS builder
+
+ARG GCC_VERSION="gcc-7"
+
+# Install dependencies
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+	bc                    \
+	bison                 \
+	build-essential       \
+	ccache                \
+	flex                  \
+	${GCC_VERSION}        \
+	git                   \
+	kmod                  \
+	libelf-dev            \
+	libncurses-dev        \
+	libssl-dev            \
+	wget                  \
+	ca-certificates    && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/${GCC_VERSION} 10
+
+# Install crosscompilers for non-amd64 arches
+RUN apt-get install -y --no-install-recommends \
+	binutils-multiarch \
+	${GCC_VERSION}-aarch64-linux-gnu && \
+	ln -s /usr/bin/aarch64-linux-gnu-${GCC_VERSION} /usr/bin/aarch64-linux-gnu-gcc
diff --git a/images/kernel/upgrade-config.sh b/images/kernel/upgrade-config.sh
index 95bee9562..d0003715b 100755
--- a/images/kernel/upgrade-config.sh
+++ b/images/kernel/upgrade-config.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
-KERNEL_BUILDER_IMAGE=luxas/kernel-builder:gcc-7
+KERNEL_BUILDER_IMAGE=weaveworks/ignite-kernel-builder:dev
+LINUX_REPO_URL=git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
 
 DOCKER_TTY="${DOCKER_TTY:+"-t"}"
 
@@ -23,12 +24,12 @@ if [[ ${FROM} != ${TO} ]]; then
     cp ${FROM} ${TO}
 fi
 
-docker pull ${KERNEL_BUILDER_IMAGE}
-docker run -i ${DOCKER_TTY} \
+docker run --rm -i ${DOCKER_TTY} \
     ${ARCH_PARAMETER} \
 	-v $(pwd)/${TO}:/tmp/.config \
     ${KERNEL_BUILDER_IMAGE} /bin/bash -c "\
-        git checkout v${VERSION} && \
+        git clone --depth 1 --branch v${VERSION} ${LINUX_REPO_URL} linux && \
+        cd linux &&
         make clean && make mrproper && cp /tmp/.config . && \
         make EXTRAVERSION="" LOCALVERSION= olddefconfig && \
         cp .config /tmp/.config"