multiarch: use normal Dockerfiles and build command

This removes code duplication:
- the same Go environment is used for multiarch and for testing
- BUILD_PLATFORMS determines what is getting build instead
  of hard-coding that in the make target
- no separate Dockerfile.multiarch

During the actual cloud build, BUILD_PLATFORMS is taken from the
shared prow.sh. There's currently no support for overriding that per
repo.

We still need separate Dockerfiles for Linux and Windows because
Windows has special requirements.

In some lines, indention is changed to use tabs.

Dockerfile

@@ -1,6 +1,7 @@
 FROM gcr.io/distroless/static:latest
 LABEL maintainers="Kubernetes Authors"
 LABEL description="CSI Node driver registrar"
+ARG binary=./bin/csi-node-driver-registrar
 
-COPY ./bin/csi-node-driver-registrar csi-node-driver-registrar
+COPY ${binary} csi-node-driver-registrar
 ENTRYPOINT ["/csi-node-driver-registrar"]

Makefile

@@ -19,38 +19,62 @@ all: build
 
 include release-tools/build.make
 
-# This target builds multiarch images using Moby BuildKit builder toolkit.
-# Docker Buildx is included in Docker 19.03 and needs DOCKER_CLI_EXPERIMENTAL enabled to run corresponding commands.
-# Currently amd, s390x and Windows manifest is pushed for canary, release branch and released tags.
-# Images generated from Prow build are pushed to staging area on gcr
-push-multiarch-%:
-	make BUILD_PLATFORMS="windows amd64 .exe"
+# Additional parameters are needed when pushing to a local registry,
+# see https://github.com/docker/buildx/issues/94.
+# However, that then runs into https://github.com/docker/cli/issues/2396.
+#
+# What works for local testing is:
+# make push-multiarch PULL_BASE_REF=master REGISTRY_NAME=<your account on dockerhub.io> BUILD_PLATFORMS="linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x"
+DOCKER_BUILDX_CREATE_ARGS ?=
+
+# This target builds a multiarch image for one command using Moby BuildKit builder toolkit.
+# Docker Buildx is included in Docker 19.03.
+#
+# ./cmd/<command>/Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows].
+# BUILD_PLATFORMS determines which individual images are included in the multiarch image.
+# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines
+# the tag for the resulting multiarch image.
+push-multiarch-%: build-%
+	if ! [ "$(PULL_BASE_REF)" ]; then \
+		echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \
+		exit 1; \
+	fi
 	set -ex; \
 	DOCKER_CLI_EXPERIMENTAL=enabled; \
 	export DOCKER_CLI_EXPERIMENTAL; \
-	docker buildx create --use --name multiarchimage-buildertest; \
+	docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
+	trap "docker buildx rm multiarchimage-buildertest" EXIT; \
+	dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
+	dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
+	build_platforms='$(BUILD_PLATFORMS)'; \
+	if ! [ "$$build_platforms" ]; then build_platforms="linux amd64"; fi; \
 	pushMultiArch () { \
-                tag=$$1; \
-                docker buildx build --push -t $(IMAGE_NAME):amd64-linux-$$tag --platform=linux/amd64 -f $(shell if [ -e ./cmd/$*/Dockerfile.multiarch ]; then echo ./cmd/$*/Dockerfile.multiarch; else echo Dockerfile.multiarch; fi) --label revision=$(REV) .; \
-                docker buildx build --push -t $(IMAGE_NAME):s390x-linux-$$tag --platform=linux/s390x -f $(shell if [ -e ./cmd/$*/Dockerfile.multiarch ]; then echo ./cmd/$*/Dockerfile.multiarch; else echo Dockerfile.multiarch; fi) --label revision=$(REV) .; \
-                docker buildx build --push -t $(IMAGE_NAME):amd64-windows-$$tag --platform=windows -f $(shell if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi) --label revision=$(REV) .; \
-                docker manifest create --amend $(IMAGE_NAME):$$tag $(IMAGE_NAME):amd64-linux-$$tag \
-                        $(IMAGE_NAME):s390x-linux-$$tag \
-                        $(IMAGE_NAME):amd64-windows-$$tag; \
-                docker manifest push -p $(IMAGE_NAME):$$tag; \
+		tag=$$1; \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
+			docker buildx build --push \
+				--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
+				--platform=$$os/$$arch \
+				--file $$(eval echo \$${dockerfile_$$os}) \
+				--build-arg binary=./bin/$*$$suffix \
+				--label revision=$(REV) \
+				.; \
+		done; \
+		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
+		docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
+		docker manifest push -p $(IMAGE_NAME):$$tag; \
 	}; \
 	if [ $(PULL_BASE_REF) = "master" ]; then \
-                       : "creating or overwriting canary image"; \
-                       pushMultiArch canary; \
+		       : "creating or overwriting canary image"; \
+		       pushMultiArch canary; \
 	elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \
-                       : "creating or overwriting canary image for release branch"; \
-                        release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \
-                        pushMultiArch $$release_canary_tag; \
+		       : "creating or overwriting canary image for release branch"; \
+			release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \
+			pushMultiArch $$release_canary_tag; \
 	elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \
-                       : "creating release image"; \
-                       pushMultiArch $(PULL_BASE_REF); \
+		       : "creating release image"; \
+		       pushMultiArch $(PULL_BASE_REF); \
 	else \
-                       : "release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists, skipping push"; \
+		       : "release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists, skipping push"; \
 	fi; \
 
 push-multiarch: $(CMDS:%=push-multiarch-%)

cloudbuild.yaml

@@ -8,17 +8,18 @@ options:
   substitution_option: ALLOW_LOOSE
 steps:
   - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
-    entrypoint: sh
+    entrypoint: bash
     env:
     - PULL_BASE_REF=$_PULL_BASE_REF
     - HOME=/root
     args:
-    - -e
-    - -c
+    - '-ex'
+    - '-c'
     - |
       gcloud auth configure-docker
-      # Extract tag-n-hash value from _GIT_TAG(form vYYYYMMDD-tag-n-hash) for REV value
-      make push-multiarch REV=v$(echo $_GIT_TAG | cut -f3- -d 'v') REGISTRY_NAME=gcr.io/k8s-staging-csi
+      # Extract tag-n-hash value from _GIT_TAG(form vYYYYMMDD-tag-n-hash) for REV value.
+      # Uses the default build platforms as defined in prow.sh.
+      make push-multiarch REV=v$(echo $_GIT_TAG | cut -f3- -d 'v') REGISTRY_NAME=gcr.io/k8s-staging-csi BUILD_PLATFORMS="$(set +x; source release-tools/prow.sh >/dev/null; echo "$CSI_PROW_BUILD_PLATFORMS")"
 substitutions:
   # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
   # can be used as a substitution