diff --git a/.buildkite/steps/build-docker-image.sh b/.buildkite/steps/build-docker-image.sh
index a3379ad325..8065f3e119 100755
--- a/.buildkite/steps/build-docker-image.sh
+++ b/.buildkite/steps/build-docker-image.sh
@@ -3,7 +3,7 @@
 set -Eeufo pipefail
 
 ## This script can be run locally like this:
-## If you are pushing, then images for all archtectures need to be built using buildx.
+## If you are pushing, then images for all architectures need to be built using buildx.
 ## This typically requires something like `qemu-user-static` to be avaliable
 ##
 ## .buildkite/steps/build-docker-image.sh (alpine|alpine-k8s|ubuntu-18.04|ubuntu-20.04|sidecar) (image tag) (codename) (version)
@@ -105,6 +105,11 @@ trap "docker buildx rm $builder_name" EXIT
 echo "--- Building :docker: $image_tag"
 cp -a packaging/linux/root/usr/share/buildkite-agent/hooks/ "${packaging_dir}/hooks/"
 cp pkg/buildkite-agent-linux-{amd64,arm64} "$packaging_dir"
+
+# Build images for all architectures
+docker buildx build --progress plain --builder "$builder_name" --platform linux/amd64,linux/arm64 "$packaging_dir"
+# Tag images for just the architecture we are on. There is a limitation in docker that prevents this
+# from being done in one command. Luckliy the second build will be quick because of docker layer caching
 docker buildx build --progress plain --builder "$builder_name" --tag "$image_tag" --load "$packaging_dir"
 
 case $variant in
@@ -118,10 +123,15 @@ esac
 
 if [[ $push == "true" ]] ; then
   echo "--- Pushing to ECR :ecr:"
+  # Do another build with all architectures. The layers should be cached from the previous build with
+  # all architectures.
+  # Pushing in this way greatly simplifies creating the manifest list on the docker registry so that
+  # either architecture can be pulled with the same tag
   dry_run docker buildx build \
     --progress plain \
     --builder "$builder_name" \
     --tag "$image_tag" \
     --platform linux/amd64,linux/arm64 \
-    --push "$packaging_dir"
+    --push \
+    "$packaging_dir"
 fi