From ea1f94aad7b211ecd18f05794582c60869bf9251 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Fri, 17 Apr 2020 09:02:07 -0700 Subject: [PATCH 01/10] update release tools instructions --- SIDECAR_RELEASE_PROCESS.md | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 91a1e1b8..c8aaa4e8 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -54,14 +54,21 @@ naming convention `-on-`. generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) 1. Generate release notes for the release. Replace arguments with the relevant information. - ``` - GITHUB_TOKEN= ./release-notes --start-sha=0ed6978fd199e3ca10326b82b4b8b8e916211c9b --end-sha=3cb3d2f18ed8cb40371c6d8886edcabd1f27e7b9 \ - --github-org=kubernetes-csi --github-repo=external-attacher -branch=master -output out.md - ``` - * `--start-sha` should point to the last release from the same branch. For - example: - * `1.X-1.0` tag when releasing `1.X.0` - * `1.X.Y-1` tag when releasing `1.X.Y` + * For new minor releases on master: + ``` + GITHUB_TOKEN= release-notes --discover=mergebase-to-latest + --github-org=kubernetes-csi --github-repo=external-provisioner + --required-author="" --output out.md + ``` + * For new patch releases on a release branch: + ``` + GITHUB_TOKEN= release-notes --branch=release-1.1 + --start-rev=v1.1.1 --end-sha=f0a9219b29cc9053047c39d149ce9b22bc7b918b + --github-org=kubernetes-csi --github-repo=external-provisioner + --required-author="" --output out.md + ``` + * `--start-rev` should point to the last patch release from the release branch. + * `--end-sha` should point to the latest commit from the release branch. 1. Compare the generated output to the new commits for the release to check if any notable change missed a release note. 1. Reword release notes as needed. Make sure to check notes for breaking From d8c76fee3e64e38a3427c9cf9df0f069801e3d74 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Wed, 15 Apr 2020 19:11:33 -0700 Subject: [PATCH 02/10] Support local snapshot RBAC for pull jobs Signed-off-by: Grant Griffiths --- prow.sh | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/prow.sh b/prow.sh index 86b2bf05..0b962f04 100755 --- a/prow.sh +++ b/prow.sh @@ -1064,18 +1064,24 @@ main () { # always pulling the image # (https://github.com/kubernetes-sigs/kind/issues/328). docker tag "$i:latest" "$i:csiprow" || die "tagging the locally built container image for $i failed" - done - if [ -e deploy/kubernetes/rbac.yaml ]; then - # This is one of those components which has its own RBAC rules (like external-provisioner). - # We are testing a locally built image and also want to test with the the current, - # potentially modified RBAC rules. - if [ "$(echo "$cmds" | wc -w)" != 1 ]; then - die "ambiguous deploy/kubernetes/rbac.yaml: need exactly one command, got: $cmds" + # For components with multiple cmds, the RBAC file should be in the following format: + # rbac-$cmd.yaml + # If this file cannot be found, we can default to the standard location: + # deploy/kubernetes/rbac.yaml + rbac_file_path=$(find . -type f -name "rbac-$i.yaml") + if [ "$rbac_file_path" == "" ]; then + rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml" fi - e=$(echo "$cmds" | tr '[:lower:]' '[:upper:]' | tr - _) - images="$images ${e}_RBAC=$(pwd)/deploy/kubernetes/rbac.yaml" - fi + + if [ -e "$rbac_file_path" ]; then + # This is one of those components which has its own RBAC rules (like external-provisioner). + # We are testing a locally built image and also want to test with the the current, + # potentially modified RBAC rules. + e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _) + images="$images ${e}_RBAC=$rbac_file_path" + fi + done fi if tests_need_non_alpha_cluster; then From 6f2322e80813459de15eb2aa0d17ddd53cb3efa8 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 23 Apr 2020 10:21:49 -0700 Subject: [PATCH 03/10] Update patch release notes generation command --- SIDECAR_RELEASE_PROCESS.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index c8aaa4e8..6960607e 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -50,7 +50,7 @@ naming convention `-on-`. ## Release Process 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. -1. Download [K8s release notes +1. Download v2.8+ [K8s release notes generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) 1. Generate release notes for the release. Replace arguments with the relevant information. @@ -62,13 +62,10 @@ naming convention `-on-`. ``` * For new patch releases on a release branch: ``` - GITHUB_TOKEN= release-notes --branch=release-1.1 - --start-rev=v1.1.1 --end-sha=f0a9219b29cc9053047c39d149ce9b22bc7b918b + GITHUB_TOKEN= release-notes --discover=patch-to-latest --branch=release-1.1 --github-org=kubernetes-csi --github-repo=external-provisioner --required-author="" --output out.md ``` - * `--start-rev` should point to the last patch release from the release branch. - * `--end-sha` should point to the latest commit from the release branch. 1. Compare the generated output to the new commits for the release to check if any notable change missed a release note. 1. Reword release notes as needed. Make sure to check notes for breaking From bd416901d4b7cad7bc70a39f508b68e2a9fff1f6 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 28 May 2020 10:22:20 +0200 Subject: [PATCH 04/10] cloud build: initial set of shared files The approach taken here extends the existing support for cross-compiling binaries on the build host and specifying the Go compiler: Go is installed if needed (as in Prow testing), binaries are build on the host, then one image is created for each platform, and finally those are combined into a single multi-architecture image. --- build.make | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ cloudbuild.sh | 6 +++++ cloudbuild.yaml | 44 ++++++++++++++++++++++++++++++++++ prow.sh | 14 +++++++++++ 4 files changed, 127 insertions(+) create mode 100755 cloudbuild.sh create mode 100644 cloudbuild.yaml diff --git a/build.make b/build.make index 5c9793c2..e3a44f96 100644 --- a/build.make +++ b/build.make @@ -105,6 +105,69 @@ build: $(CMDS:%=build-%) container: $(CMDS:%=container-%) push: $(CMDS:%=push-%) +# Additional parameters are needed when pushing to a local registry, +# see https://github.com/docker/buildx/issues/94. +# However, that then runs into https://github.com/docker/cli/issues/2396. +# +# What works for local testing is: +# make push-multiarch PULL_BASE_REF=master REGISTRY_NAME= BUILD_PLATFORMS="linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x" +DOCKER_BUILDX_CREATE_ARGS ?= + +# This target builds a multiarch image for one command using Moby BuildKit builder toolkit. +# Docker Buildx is included in Docker 19.03. +# +# ./cmd//Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows]. +# BUILD_PLATFORMS determines which individual images are included in the multiarch image. +# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines +# the tag for the resulting multiarch image. +push-multiarch-%: check-pull-base-ref build-% + set -ex; \ + DOCKER_CLI_EXPERIMENTAL=enabled; \ + export DOCKER_CLI_EXPERIMENTAL; \ + docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ + trap "docker buildx rm multiarchimage-buildertest" EXIT; \ + dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ + dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ + if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ + pushMultiArch () { \ + tag=$$1; \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ + docker buildx build --push \ + --tag $(IMAGE_NAME):$$arch-$$os-$$tag \ + --platform=$$os/$$arch \ + --file $$(eval echo \$${dockerfile_$$os}) \ + --build-arg binary=./bin/$*$$suffix \ + --label revision=$(REV) \ + .; \ + done; \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \ + docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ + docker manifest push -p $(IMAGE_NAME):$$tag; \ + }; \ + if [ $(PULL_BASE_REF) = "master" ]; then \ + : "creating or overwriting canary image"; \ + pushMultiArch canary; \ + elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \ + : "creating or overwriting canary image for release branch"; \ + release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \ + pushMultiArch $$release_canary_tag; \ + elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \ + : "creating release image"; \ + pushMultiArch $(PULL_BASE_REF); \ + else \ + : "ERROR: release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists: a new tag is required!"; \ + exit 1; \ + fi + +.PHONY: check-pull-base-ref +check-pull-base-ref: + if ! [ "$(PULL_BASE_REF)" ]; then \ + echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \ + exit 1; \ + fi + +push-multiarch: $(CMDS:%=push-multiarch-%) + clean: -rm -rf bin diff --git a/cloudbuild.sh b/cloudbuild.sh new file mode 100755 index 00000000..3ba11eca --- /dev/null +++ b/cloudbuild.sh @@ -0,0 +1,6 @@ +#! /bin/bash + +# shellcheck disable=SC1091 +. release-tools/prow.sh + +gcr_cloud_build diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 00000000..1def1499 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,44 @@ +# A configuration file for multi-arch image building with the Google cloud build service. +# +# Repos using this file must: +# - import csi-release-tools +# - add a symlink cloudbuild.yaml -> release-tools/cloudbuild.yaml +# - add a .cloudbuild.sh which can be a custom file or a symlink +# to release-tools/cloudbuild.sh +# - accept "binary" as build argument in their Dockerfile(s) (see +# https://github.com/pohly/node-driver-registrar/blob/3018101987b0bb6da2a2657de607174d6e3728f7/Dockerfile#L4-L6) +# because binaries will get built for different architectures and then +# get copied from the built host into the container image +# +# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md +# for more details on image pushing process in Kubernetes. + +# This must be specified in seconds. If omitted, defaults to 600s (10 mins). +timeout: 1200s +# This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, +# or any new substitutions added in the future. +options: + substitution_option: ALLOW_LOOSE +steps: + # The image must contain bash and curl. Ideally it should also contain + # the desired version of Go (currently defined in release-tools/travis.yml), + # but that just speeds up the build and is not required. + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' + entrypoint: ./.cloudbuild.sh + env: + - GIT_TAG=${_GIT_TAG} + - PULL_BASE_REF=${_PULL_BASE_REF} + - REGISTRY_NAME=gcr.io/${_STAGING_PROJECT} + - HOME=/root +substitutions: + # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and + # can be used as a substitution. + _GIT_TAG: '12345' + # _PULL_BASE_REF will contain the ref that was pushed to trigger this build - + # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'. + _PULL_BASE_REF: 'master' + # The default gcr.io staging project for Kubernetes-CSI + # (=> https://console.cloud.google.com/gcr/images/k8s-staging-csi/GLOBAL). + # Might be overridden in the Prow build job for a repo which wants + # images elsewhere. + _STAGING_PROJECT: 'k8s-staging-csi' diff --git a/prow.sh b/prow.sh index 0b962f04..93964e18 100755 --- a/prow.sh +++ b/prow.sh @@ -1189,3 +1189,17 @@ main () { return "$ret" } + +# This function can be called by a repo's top-level cloudbuild.sh: +# it handles environment set up in the GCR cloud build and then +# invokes "make push-multiarch" to do the actual image building. +gcr_cloud_build () { + # Register gcloud as a Docker credential helper. + # Required for "docker buildx build --push". + gcloud auth configure-docker + + # Extract tag-n-hash value from GIT_TAG (form vYYYYMMDD-tag-n-hash) for REV value. + REV=v$(echo "$GIT_TAG" | cut -f3- -d 'v') + + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make push-multiarch REV="${REV}" REGISTRY_NAME="${REGISTRY_NAME}" BUILD_PLATFORMS="${CSI_PROW_BUILD_PLATFORMS}" +} From 4569f27a8cf57336ea13d8ef3dc4423c76712913 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Jun 2020 11:03:47 +0200 Subject: [PATCH 05/10] build.make: fix push-multiarch ambiguity "make push-multiarch" matched both push-multiarch and push-%. This seems to be none-deterministic and in at least one repo (external-provisioner), make picked the wildcard rule which then failed because there is no "multiarch" command. This ambiguity gets resolved by instantiating the wildcard rules only for existing commands. The advantage also is that "make push-no-such-command" will fail with an obvious "No rule to make target 'push-no-such-command'" instead of attempting to build the command. --- build.make | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.make b/build.make index e3a44f96..0e4a9baa 100644 --- a/build.make +++ b/build.make @@ -71,7 +71,7 @@ BUILD_PLATFORMS = # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. -build-%: check-go-version-go +$(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \ @@ -80,10 +80,10 @@ build-%: check-go-version-go fi; \ done -container-%: build-% +$(CMDS:%=container-%): container-%: build-% docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . -push-%: container-% +$(CMDS:%=push-%): push-%: container-% set -ex; \ push_image () { \ docker tag $*:latest $(IMAGE_NAME):$$tag; \ @@ -120,7 +120,7 @@ DOCKER_BUILDX_CREATE_ARGS ?= # BUILD_PLATFORMS determines which individual images are included in the multiarch image. # PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines # the tag for the resulting multiarch image. -push-multiarch-%: check-pull-base-ref build-% +$(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ DOCKER_CLI_EXPERIMENTAL=enabled; \ export DOCKER_CLI_EXPERIMENTAL; \ From 5231f05d8b2f7a6b656fb27d84abcb758000842b Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Jun 2020 11:04:29 +0200 Subject: [PATCH 06/10] build.make: properly declare push-multiarch It's not a real file and thus should better be marked as phony. --- build.make | 1 + 1 file changed, 1 insertion(+) diff --git a/build.make b/build.make index 0e4a9baa..83d903e0 100644 --- a/build.make +++ b/build.make @@ -166,6 +166,7 @@ check-pull-base-ref: exit 1; \ fi +.PHONY: push-multiarch push-multiarch: $(CMDS:%=push-multiarch-%) clean: From 340e082f04321801f51cdff42576bd55dcfc3683 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Jun 2020 11:05:05 +0200 Subject: [PATCH 07/10] build.make: optional inclusion of Windows in multiarch images Most repos inherit the default BUILD_PLATFORMS, which includes Windows, but don't have the necessary Dockerfile.Windows yet. To simplify the rollout of multiarch image builds, Windows binary building continues to be tested (i.e. BUILD_PLATFORMS remains unchanged), but push-multiarch skips Windows if the Dockerfile.Windows is missing. --- build.make | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build.make b/build.make index 83d903e0..c17a1c19 100644 --- a/build.make +++ b/build.make @@ -117,6 +117,10 @@ DOCKER_BUILDX_CREATE_ARGS ?= # Docker Buildx is included in Docker 19.03. # # ./cmd//Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows]. +# It is currently optional: if no such file exists, Windows images are not included, +# even when Windows is listed in BUILD_PLATFORMS. That way, projects can test that +# Windows binaries can be built before adding a Dockerfile for it. +# # BUILD_PLATFORMS determines which individual images are included in the multiarch image. # PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines # the tag for the resulting multiarch image. @@ -129,6 +133,9 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ + if ! [ -f "$$dockerfile_windows" ]; then \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \ + fi; \ pushMultiArch () { \ tag=$$1; \ echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ From db0c2a7dc80d5b6d65ac8e4e8c64026f342fd987 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 4 Jun 2020 10:52:04 +0200 Subject: [PATCH 08/10] cloud build: initialize support for running commands in Dockerfile If the Dockerfile needs to run some command, that step fails unless QEMU is set up properly first: failed to solve: rpc error: code = Unknown desc = failed to load LLB: runtime execution on platform linux/ppc64le not supported --- prow.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/prow.sh b/prow.sh index 93964e18..32ec871a 100755 --- a/prow.sh +++ b/prow.sh @@ -1198,6 +1198,12 @@ gcr_cloud_build () { # Required for "docker buildx build --push". gcloud auth configure-docker + if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then + # Needed for "RUN" steps on non-linux/amd64 platforms. + # See https://github.com/multiarch/qemu-user-static#getting-started + (set -x; docker run --rm --privileged multiarch/qemu-user-static --reset -p yes) + fi + # Extract tag-n-hash value from GIT_TAG (form vYYYYMMDD-tag-n-hash) for REV value. REV=v$(echo "$GIT_TAG" | cut -f3- -d 'v') From 3df86b7d437d7126a1ffb79ff05b0e467748b920 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 12 Jun 2020 15:48:26 +0200 Subject: [PATCH 09/10] cloud build: k8s-staging-sig-storage As discussed in https://github.com/kubernetes/k8s.io/pull/943, we want to consolidate under k8s-staging-sig-storage. --- cloudbuild.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 1def1499..a22c0e7a 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -12,6 +12,8 @@ # # See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md # for more details on image pushing process in Kubernetes. +# +# To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage. # This must be specified in seconds. If omitted, defaults to 600s (10 mins). timeout: 1200s @@ -38,7 +40,7 @@ substitutions: # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'. _PULL_BASE_REF: 'master' # The default gcr.io staging project for Kubernetes-CSI - # (=> https://console.cloud.google.com/gcr/images/k8s-staging-csi/GLOBAL). + # (=> https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage/GLOBAL). # Might be overridden in the Prow build job for a repo which wants # images elsewhere. - _STAGING_PROJECT: 'k8s-staging-csi' + _STAGING_PROJECT: 'k8s-staging-sig-storage' From 43e50d6f6471f476acd5c90cd3788e198a86d9f8 Mon Sep 17 00:00:00 2001 From: Yibo Cai Date: Tue, 16 Jun 2020 10:45:09 +0800 Subject: [PATCH 10/10] prow.sh: enable building arm64 image --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 32ec871a..9778635f 100755 --- a/prow.sh +++ b/prow.sh @@ -85,7 +85,7 @@ get_versioned_variable () { echo "$value" } -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because