diff --git a/.cloudbuild.sh b/.cloudbuild.sh new file mode 120000 index 00000000..94226bc6 --- /dev/null +++ b/.cloudbuild.sh @@ -0,0 +1 @@ +release-tools/cloudbuild.sh \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 5e781ce1..64986b9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ FROM gcr.io/distroless/static:latest LABEL maintainers="Kubernetes Authors" LABEL description="CSI Driver liveness probe" +ARG binary=./bin/livenessprobe -COPY ./bin/livenessprobe /livenessprobe +COPY ${binary} /livenessprobe ENTRYPOINT ["/livenessprobe"] diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 120000 index 00000000..bee22c46 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1 @@ +release-tools/cloudbuild.yaml \ No newline at end of file diff --git a/release-tools/SIDECAR_RELEASE_PROCESS.md b/release-tools/SIDECAR_RELEASE_PROCESS.md index 91a1e1b8..6960607e 100644 --- a/release-tools/SIDECAR_RELEASE_PROCESS.md +++ b/release-tools/SIDECAR_RELEASE_PROCESS.md @@ -50,18 +50,22 @@ naming convention `-on-`. ## Release Process 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. -1. Download [K8s release notes +1. Download v2.8+ [K8s release notes generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) 1. Generate release notes for the release. Replace arguments with the relevant information. - ``` - GITHUB_TOKEN= ./release-notes --start-sha=0ed6978fd199e3ca10326b82b4b8b8e916211c9b --end-sha=3cb3d2f18ed8cb40371c6d8886edcabd1f27e7b9 \ - --github-org=kubernetes-csi --github-repo=external-attacher -branch=master -output out.md - ``` - * `--start-sha` should point to the last release from the same branch. For - example: - * `1.X-1.0` tag when releasing `1.X.0` - * `1.X.Y-1` tag when releasing `1.X.Y` + * For new minor releases on master: + ``` + GITHUB_TOKEN= release-notes --discover=mergebase-to-latest + --github-org=kubernetes-csi --github-repo=external-provisioner + --required-author="" --output out.md + ``` + * For new patch releases on a release branch: + ``` + GITHUB_TOKEN= release-notes --discover=patch-to-latest --branch=release-1.1 + --github-org=kubernetes-csi --github-repo=external-provisioner + --required-author="" --output out.md + ``` 1. Compare the generated output to the new commits for the release to check if any notable change missed a release note. 1. Reword release notes as needed. Make sure to check notes for breaking diff --git a/release-tools/build.make b/release-tools/build.make index 3bf3391c..e3a44f96 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -60,18 +60,25 @@ else TESTARGS = endif -ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH)) - # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. +# BUILD_PLATFORMS contains a set of triplets, +# separated by semicolon. An empty variable or empty entry (= just a +# semicolon) builds for the default platform of the current Go +# toolchain. +BUILD_PLATFORMS = + +# This builds each command (= the sub-directories of ./cmd) for the target platform(s) +# defined by BUILD_PLATFORMS. build-%: check-go-version-go mkdir -p bin - CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* - if [ "$$ARCH" = "amd64" ]; then \ - CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ - CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* ; \ - fi + echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ + if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \ + echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ + exit 1; \ + fi; \ + done container-%: build-% docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . @@ -98,6 +105,69 @@ build: $(CMDS:%=build-%) container: $(CMDS:%=container-%) push: $(CMDS:%=push-%) +# Additional parameters are needed when pushing to a local registry, +# see https://github.com/docker/buildx/issues/94. +# However, that then runs into https://github.com/docker/cli/issues/2396. +# +# What works for local testing is: +# make push-multiarch PULL_BASE_REF=master REGISTRY_NAME= BUILD_PLATFORMS="linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x" +DOCKER_BUILDX_CREATE_ARGS ?= + +# This target builds a multiarch image for one command using Moby BuildKit builder toolkit. +# Docker Buildx is included in Docker 19.03. +# +# ./cmd//Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows]. +# BUILD_PLATFORMS determines which individual images are included in the multiarch image. +# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines +# the tag for the resulting multiarch image. +push-multiarch-%: check-pull-base-ref build-% + set -ex; \ + DOCKER_CLI_EXPERIMENTAL=enabled; \ + export DOCKER_CLI_EXPERIMENTAL; \ + docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ + trap "docker buildx rm multiarchimage-buildertest" EXIT; \ + dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ + dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ + if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ + pushMultiArch () { \ + tag=$$1; \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ + docker buildx build --push \ + --tag $(IMAGE_NAME):$$arch-$$os-$$tag \ + --platform=$$os/$$arch \ + --file $$(eval echo \$${dockerfile_$$os}) \ + --build-arg binary=./bin/$*$$suffix \ + --label revision=$(REV) \ + .; \ + done; \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \ + docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ + docker manifest push -p $(IMAGE_NAME):$$tag; \ + }; \ + if [ $(PULL_BASE_REF) = "master" ]; then \ + : "creating or overwriting canary image"; \ + pushMultiArch canary; \ + elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \ + : "creating or overwriting canary image for release branch"; \ + release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \ + pushMultiArch $$release_canary_tag; \ + elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \ + : "creating release image"; \ + pushMultiArch $(PULL_BASE_REF); \ + else \ + : "ERROR: release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists: a new tag is required!"; \ + exit 1; \ + fi + +.PHONY: check-pull-base-ref +check-pull-base-ref: + if ! [ "$(PULL_BASE_REF)" ]; then \ + echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \ + exit 1; \ + fi + +push-multiarch: $(CMDS:%=push-multiarch-%) + clean: -rm -rf bin diff --git a/release-tools/cloudbuild.sh b/release-tools/cloudbuild.sh new file mode 100755 index 00000000..3ba11eca --- /dev/null +++ b/release-tools/cloudbuild.sh @@ -0,0 +1,6 @@ +#! /bin/bash + +# shellcheck disable=SC1091 +. release-tools/prow.sh + +gcr_cloud_build diff --git a/release-tools/cloudbuild.yaml b/release-tools/cloudbuild.yaml new file mode 100644 index 00000000..1def1499 --- /dev/null +++ b/release-tools/cloudbuild.yaml @@ -0,0 +1,44 @@ +# A configuration file for multi-arch image building with the Google cloud build service. +# +# Repos using this file must: +# - import csi-release-tools +# - add a symlink cloudbuild.yaml -> release-tools/cloudbuild.yaml +# - add a .cloudbuild.sh which can be a custom file or a symlink +# to release-tools/cloudbuild.sh +# - accept "binary" as build argument in their Dockerfile(s) (see +# https://github.com/pohly/node-driver-registrar/blob/3018101987b0bb6da2a2657de607174d6e3728f7/Dockerfile#L4-L6) +# because binaries will get built for different architectures and then +# get copied from the built host into the container image +# +# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md +# for more details on image pushing process in Kubernetes. + +# This must be specified in seconds. If omitted, defaults to 600s (10 mins). +timeout: 1200s +# This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, +# or any new substitutions added in the future. +options: + substitution_option: ALLOW_LOOSE +steps: + # The image must contain bash and curl. Ideally it should also contain + # the desired version of Go (currently defined in release-tools/travis.yml), + # but that just speeds up the build and is not required. + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' + entrypoint: ./.cloudbuild.sh + env: + - GIT_TAG=${_GIT_TAG} + - PULL_BASE_REF=${_PULL_BASE_REF} + - REGISTRY_NAME=gcr.io/${_STAGING_PROJECT} + - HOME=/root +substitutions: + # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and + # can be used as a substitution. + _GIT_TAG: '12345' + # _PULL_BASE_REF will contain the ref that was pushed to trigger this build - + # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'. + _PULL_BASE_REF: 'master' + # The default gcr.io staging project for Kubernetes-CSI + # (=> https://console.cloud.google.com/gcr/images/k8s-staging-csi/GLOBAL). + # Might be overridden in the Prow build job for a repo which wants + # images elsewhere. + _STAGING_PROJECT: 'k8s-staging-csi' diff --git a/release-tools/prow.sh b/release-tools/prow.sh index d54dea51..93964e18 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -85,6 +85,8 @@ get_versioned_variable () { echo "$value" } +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" + # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because # setting it globally can break other go usages (like "go get " @@ -340,7 +342,7 @@ configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kub configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment -configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0' "external-snapshotter version tag" +configvar CSI_SNAPSHOTTER_VERSION 'v2.0.1' "external-snapshotter version tag" # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply @@ -1026,7 +1028,7 @@ main () { images= if ${CSI_PROW_BUILD_JOB}; then # A successful build is required for testing. - run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make all' failed" + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" "BUILD_PLATFORMS=${CSI_PROW_BUILD_PLATFORMS}" || die "'make all' failed" # We don't want test failures to prevent E2E testing below, because the failure # might have been minor or unavoidable, for example when experimenting with # changes in "release-tools" in a PR (that fails the "is release-tools unmodified" @@ -1062,18 +1064,24 @@ main () { # always pulling the image # (https://github.com/kubernetes-sigs/kind/issues/328). docker tag "$i:latest" "$i:csiprow" || die "tagging the locally built container image for $i failed" - done - if [ -e deploy/kubernetes/rbac.yaml ]; then - # This is one of those components which has its own RBAC rules (like external-provisioner). - # We are testing a locally built image and also want to test with the the current, - # potentially modified RBAC rules. - if [ "$(echo "$cmds" | wc -w)" != 1 ]; then - die "ambiguous deploy/kubernetes/rbac.yaml: need exactly one command, got: $cmds" + # For components with multiple cmds, the RBAC file should be in the following format: + # rbac-$cmd.yaml + # If this file cannot be found, we can default to the standard location: + # deploy/kubernetes/rbac.yaml + rbac_file_path=$(find . -type f -name "rbac-$i.yaml") + if [ "$rbac_file_path" == "" ]; then + rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml" fi - e=$(echo "$cmds" | tr '[:lower:]' '[:upper:]' | tr - _) - images="$images ${e}_RBAC=$(pwd)/deploy/kubernetes/rbac.yaml" - fi + + if [ -e "$rbac_file_path" ]; then + # This is one of those components which has its own RBAC rules (like external-provisioner). + # We are testing a locally built image and also want to test with the the current, + # potentially modified RBAC rules. + e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _) + images="$images ${e}_RBAC=$rbac_file_path" + fi + done fi if tests_need_non_alpha_cluster; then @@ -1181,3 +1189,17 @@ main () { return "$ret" } + +# This function can be called by a repo's top-level cloudbuild.sh: +# it handles environment set up in the GCR cloud build and then +# invokes "make push-multiarch" to do the actual image building. +gcr_cloud_build () { + # Register gcloud as a Docker credential helper. + # Required for "docker buildx build --push". + gcloud auth configure-docker + + # Extract tag-n-hash value from GIT_TAG (form vYYYYMMDD-tag-n-hash) for REV value. + REV=v$(echo "$GIT_TAG" | cut -f3- -d 'v') + + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make push-multiarch REV="${REV}" REGISTRY_NAME="${REGISTRY_NAME}" BUILD_PLATFORMS="${CSI_PROW_BUILD_PLATFORMS}" +}