From 7dcd0a9920026b129de52eb01a5e832d91031c8c Mon Sep 17 00:00:00 2001 From: fengzixu Date: Sun, 29 Nov 2020 15:13:11 +0900 Subject: [PATCH 001/124] upgrade csi-test to v4.0.2 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 44511fb23..3daee2564 100755 --- a/prow.sh +++ b/prow.sh @@ -227,7 +227,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION 5421d9f3c37be3b95b241b44a094a3db11bee789 "csi-test version" # latest master +configvar CSI_PROW_SANITY_VERSION af2d21f66c88d7b4956c17d8123cdc563ca0ba0a "csi-test version" # latest master configvar CSI_PROW_SANITY_IMPORT_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" From 5d74e4550810ed59d4cd9f398063e35141634711 Mon Sep 17 00:00:00 2001 From: fengzixu Date: Sun, 29 Nov 2020 17:04:37 +0900 Subject: [PATCH 002/124] change the csi-test import path to v4 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 3daee2564..66c3228ab 100755 --- a/prow.sh +++ b/prow.sh @@ -228,7 +228,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" configvar CSI_PROW_SANITY_VERSION af2d21f66c88d7b4956c17d8123cdc563ca0ba0a "csi-test version" # latest master -configvar CSI_PROW_SANITY_IMPORT_PATH github.com/kubernetes-csi/csi-test "csi-test package" +configvar CSI_PROW_SANITY_IMPORT_PATH github.com/kubernetes-csi/csi-test/v4 "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI driver" From 4ab8b154c23d1bc1e062e0718d386309922051ec Mon Sep 17 00:00:00 2001 From: fengzixu Date: Tue, 1 Dec 2020 00:02:37 +0900 Subject: [PATCH 003/124] use the tag to replace commit of csi-test --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 66c3228ab..161f347f6 100755 --- a/prow.sh +++ b/prow.sh @@ -227,7 +227,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION af2d21f66c88d7b4956c17d8123cdc563ca0ba0a "csi-test version" # latest master +configvar CSI_PROW_SANITY_VERSION v4.0.2 "csi-test version" # v4.0.2 configvar CSI_PROW_SANITY_IMPORT_PATH github.com/kubernetes-csi/csi-test/v4 "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" From 9318c6ccd7b0642b481332183f1191aa55519531 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 4 Dec 2020 08:43:15 +0100 Subject: [PATCH 004/124] cloud build: double the timeout, now 1 hour Builds of external-snapshotter sometimes timed out because it builds three images and that takes almost half an hour (the previous limit) and more if the network is slow. There's no good way to set timeouts per repo and a too high timeout for shouldn't cause any issue, therefore the timeout gets increased everywhere. --- cloudbuild.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 8f678924e..7f0a4341c 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -16,7 +16,9 @@ # To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage. # This must be specified in seconds. If omitted, defaults to 600s (10 mins). -timeout: 1800s +# Building three images in external-snapshotter takes roughly half an hour, +# sometimes more. +timeout: 3600s # This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, # or any new substitutions added in the future. options: From 981be3feda57d42bdc6be31d2831f4967772d6d2 Mon Sep 17 00:00:00 2001 From: Sachin Kumar Singh Date: Wed, 6 Jan 2021 12:28:12 +0530 Subject: [PATCH 005/124] Adding spelling and boilerplate checks. The `verify-spelling.sh` and `verify-boilerplate.sh` (and its tests scripts in `boilerplate`) are added to enable checks for common typos and checking the boilerplates of the files. --- boilerplate/boilerplate.Dockerfile.txt | 13 ++ boilerplate/boilerplate.Makefile.txt | 13 ++ boilerplate/boilerplate.bzl.txt | 13 ++ boilerplate/boilerplate.go.txt | 15 ++ boilerplate/boilerplate.py | 202 +++++++++++++++++++++++++ boilerplate/boilerplate.py.txt | 13 ++ boilerplate/boilerplate.sh.txt | 13 ++ build.make | 15 ++ verify-boilerplate.sh | 51 +++++++ verify-spelling.sh | 57 +++++++ 10 files changed, 405 insertions(+) create mode 100644 boilerplate/boilerplate.Dockerfile.txt create mode 100644 boilerplate/boilerplate.Makefile.txt create mode 100644 boilerplate/boilerplate.bzl.txt create mode 100644 boilerplate/boilerplate.go.txt create mode 100755 boilerplate/boilerplate.py create mode 100644 boilerplate/boilerplate.py.txt create mode 100644 boilerplate/boilerplate.sh.txt create mode 100755 verify-boilerplate.sh create mode 100755 verify-spelling.sh diff --git a/boilerplate/boilerplate.Dockerfile.txt b/boilerplate/boilerplate.Dockerfile.txt new file mode 100644 index 000000000..34cb349c4 --- /dev/null +++ b/boilerplate/boilerplate.Dockerfile.txt @@ -0,0 +1,13 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/boilerplate/boilerplate.Makefile.txt b/boilerplate/boilerplate.Makefile.txt new file mode 100644 index 000000000..d0d526523 --- /dev/null +++ b/boilerplate/boilerplate.Makefile.txt @@ -0,0 +1,13 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/boilerplate/boilerplate.bzl.txt b/boilerplate/boilerplate.bzl.txt new file mode 100644 index 000000000..d0d526523 --- /dev/null +++ b/boilerplate/boilerplate.bzl.txt @@ -0,0 +1,13 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/boilerplate/boilerplate.go.txt b/boilerplate/boilerplate.go.txt new file mode 100644 index 000000000..3249913bd --- /dev/null +++ b/boilerplate/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright YEAR The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/boilerplate/boilerplate.py b/boilerplate/boilerplate.py new file mode 100755 index 000000000..63f74963d --- /dev/null +++ b/boilerplate/boilerplate.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import difflib +import glob +import json +import mmap +import os +import re +import sys +from datetime import date + +parser = argparse.ArgumentParser() +parser.add_argument( + "filenames", + help="list of files to check, all files if unspecified", + nargs='*') + +# Rootdir defaults to the directory **above** the repo-infra dir. +rootdir = os.path.dirname(__file__) + "./../../../" +rootdir = os.path.abspath(rootdir) +parser.add_argument( + "--rootdir", default=rootdir, help="root directory to examine") + +default_boilerplate_dir = os.path.join(rootdir, "csi-driver-nfs/hack/boilerplate") + +parser.add_argument( + "--boilerplate-dir", default=default_boilerplate_dir) + +parser.add_argument( + "-v", "--verbose", + help="give verbose output regarding why a file does not pass", + action="store_true") + +args = parser.parse_args() + +verbose_out = sys.stderr if args.verbose else open("/dev/null", "w") + +def get_refs(): + refs = {} + + for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")): + extension = os.path.basename(path).split(".")[1] + + ref_file = open(path, 'r') + ref = ref_file.read().splitlines() + ref_file.close() + refs[extension] = ref + + return refs + +def file_passes(filename, refs, regexs): + try: + f = open(filename, 'r') + except Exception as exc: + print("Unable to open %s: %s" % (filename, exc), file=verbose_out) + return False + + data = f.read() + f.close() + + basename = os.path.basename(filename) + extension = file_extension(filename) + if extension != "": + ref = refs[extension] + else: + ref = refs[basename] + + # remove build tags from the top of Go files + if extension == "go": + p = regexs["go_build_constraints"] + (data, found) = p.subn("", data, 1) + + # remove shebang from the top of shell files + if extension == "sh" or extension == "py": + p = regexs["shebang"] + (data, found) = p.subn("", data, 1) + + data = data.splitlines() + + # if our test file is smaller than the reference it surely fails! + if len(ref) > len(data): + print('File %s smaller than reference (%d < %d)' % + (filename, len(data), len(ref)), + file=verbose_out) + return False + + # trim our file to the same number of lines as the reference file + data = data[:len(ref)] + + p = regexs["year"] + for d in data: + if p.search(d): + print('File %s is missing the year' % filename, file=verbose_out) + return False + + # Replace all occurrences of the regex "CURRENT_YEAR|...|2016|2015|2014" with "YEAR" + p = regexs["date"] + for i, d in enumerate(data): + (data[i], found) = p.subn('YEAR', d) + if found != 0: + break + + # if we don't match the reference at this point, fail + if ref != data: + print("Header in %s does not match reference, diff:" % filename, file=verbose_out) + if args.verbose: + print(file=verbose_out) + for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''): + print(line, file=verbose_out) + print(file=verbose_out) + return False + + return True + +def file_extension(filename): + return os.path.splitext(filename)[1].split(".")[-1].lower() + +skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git', + 'cluster/env.sh', 'vendor', 'test/e2e/generated/bindata.go', + 'repo-infra/verify/boilerplate/test', '.glide'] + +def normalize_files(files): + newfiles = [] + for pathname in files: + if any(x in pathname for x in skipped_dirs): + continue + newfiles.append(pathname) + return newfiles + +def get_files(extensions): + files = [] + if len(args.filenames) > 0: + files = args.filenames + else: + for root, dirs, walkfiles in os.walk(args.rootdir): + # don't visit certain dirs. This is just a performance improvement + # as we would prune these later in normalize_files(). But doing it + # cuts down the amount of filesystem walking we do and cuts down + # the size of the file list + for d in skipped_dirs: + if d in dirs: + dirs.remove(d) + + for name in walkfiles: + pathname = os.path.join(root, name) + files.append(pathname) + + files = normalize_files(files) + + outfiles = [] + for pathname in files: + basename = os.path.basename(pathname) + extension = file_extension(pathname) + if extension in extensions or basename in extensions: + outfiles.append(pathname) + return outfiles + +def get_regexs(): + regexs = {} + # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing + regexs["year"] = re.compile( 'YEAR' ) + # dates can be 2014, 2015, 2016, ..., CURRENT_YEAR, company holder names can be anything + years = range(2014, date.today().year + 1) + regexs["date"] = re.compile( '(%s)' % "|".join(map(lambda l: str(l), years)) ) + # strip // +build \n\n build constraints + regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE) + # strip #!.* from shell scripts + regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) + return regexs + + + +def main(): + regexs = get_regexs() + refs = get_refs() + filenames = get_files(refs.keys()) + + for filename in filenames: + if not file_passes(filename, refs, regexs): + print(filename, file=sys.stdout) + + return 0 + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/boilerplate/boilerplate.py.txt b/boilerplate/boilerplate.py.txt new file mode 100644 index 000000000..34cb349c4 --- /dev/null +++ b/boilerplate/boilerplate.py.txt @@ -0,0 +1,13 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/boilerplate/boilerplate.sh.txt b/boilerplate/boilerplate.sh.txt new file mode 100644 index 000000000..d0d526523 --- /dev/null +++ b/boilerplate/boilerplate.sh.txt @@ -0,0 +1,13 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/build.make b/build.make index ca2be670a..a405c6700 100644 --- a/build.make +++ b/build.make @@ -275,3 +275,18 @@ test-shellcheck: .PHONY: check-go-version-% check-go-version-%: ./release-tools/verify-go-version.sh "$*" + +# Test for spelling errors. +.PHONY: test-spelling +test: test-spelling +test-spelling: + @ echo; echo "### $@:" + @ ./release-tools/verify-spelling.sh + +# Test the boilerplates of the files. +.PHONY: test-boilerplate +test: test-boilerplate +test-boilerplate: + @ echo; echo "### $@:" + @ ./release-tools/verify-boilerplate.sh + diff --git a/verify-boilerplate.sh b/verify-boilerplate.sh new file mode 100755 index 000000000..ff376e545 --- /dev/null +++ b/verify-boilerplate.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +echo "Verifying boilerplate" + +if [[ -z "$(command -v python)" ]]; then + echo "Cannot find python. Make link to python3..." + update-alternatives --install /usr/bin/python python /usr/bin/python3 1 +fi + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}") + +boilerDir="${REPO_ROOT}/boilerplate" +boiler="${boilerDir}/boilerplate.py" + +files_need_boilerplate=("$("${boiler}" --rootdir="${REPO_ROOT}" --verbose)") + +# Run boilerplate.py unit tests +unitTestOut="$(mktemp)" +trap cleanup EXIT +cleanup() { + rm "${unitTestOut}" +} + +# Run boilerplate check +if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then + for file in "${files_need_boilerplate[@]}"; do + echo "Boilerplate header is wrong for: ${file}" + done + + exit 1 +fi + +echo "Done" diff --git a/verify-spelling.sh b/verify-spelling.sh new file mode 100755 index 000000000..9a41c9319 --- /dev/null +++ b/verify-spelling.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +TOOL_VERSION="v0.3.4" + +# cd to the root path +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +cd "${ROOT}" + +# create a temporary directory +TMP_DIR=$(mktemp -d) + +# cleanup +exitHandler() ( + echo "Cleaning up..." + rm -rf "${TMP_DIR}" +) +trap exitHandler EXIT + +if [[ -z "$(command -v misspell)" ]]; then + echo "Cannot find misspell. Installing misspell..." + # perform go get in a temp dir as we are not tracking this version in a go module + # if we do the go get in the repo, it will create / update a go.mod and go.sum + cd "${TMP_DIR}" + GO111MODULE=on GOBIN="${TMP_DIR}" go get "github.com/client9/misspell/cmd/misspell@${TOOL_VERSION}" + export PATH="${TMP_DIR}:${PATH}" +fi +cd "${ROOT}" + +# check spelling +RES=0 +echo "Checking spelling..." +ERROR_LOG="${TMP_DIR}/errors.log" +git ls-files | grep -v vendor | xargs misspell > "${ERROR_LOG}" +if [[ -s "${ERROR_LOG}" ]]; then + sed 's/^/error: /' "${ERROR_LOG}" # add 'error' to each line to highlight in e2e status + echo "Found spelling errors!" + RES=1 +fi +exit "${RES}" From 48186eba9e35109d665550c7aaff6375585d0ff6 Mon Sep 17 00:00:00 2001 From: Sachin Kumar Singh Date: Thu, 7 Jan 2021 17:13:45 +0530 Subject: [PATCH 006/124] Fix spelling and boilerplate errors Some spelling and boilerplate errors are fixed which caused the failing of these tests. --- .prow.sh | 16 ++++++++++++++++ SIDECAR_RELEASE_PROCESS.md | 2 +- cloudbuild.sh | 14 ++++++++++++++ prow.sh | 8 ++++---- verify-boilerplate.sh | 10 +++++++--- verify-spelling.sh | 10 ++++++---- verify-subtree.sh | 2 +- 7 files changed, 49 insertions(+), 13 deletions(-) diff --git a/.prow.sh b/.prow.sh index b18c53581..b92678291 100755 --- a/.prow.sh +++ b/.prow.sh @@ -1,7 +1,23 @@ #! /bin/bash -e + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This is for testing csi-release-tools itself in Prow. All other # repos use prow.sh for that, but as csi-release-tools isn't a normal # repo with some Go code in it, it has a custom Prow test script. ./verify-shellcheck.sh "$(pwd)" +./verify-spelling.sh "$(pwd)" +./verify-boilerplate.sh "$(pwd)" diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 4575eb819..3aee7a81b 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -11,7 +11,7 @@ The release manager must: kubernetes/org to request membership * Be a top level approver for the repository. To become a top level approver, the candidate must demonstrate ownership and deep knowledge of the repository - through active maintainence, responding to and fixing issues, reviewing PRs, + through active maintenance, responding to and fixing issues, reviewing PRs, test triage. * Be part of the maintainers or admin group for the repository. admin is a superset of maintainers, only maintainers level is required for cutting a diff --git a/cloudbuild.sh b/cloudbuild.sh index 3ba11ecad..1edda4d3f 100755 --- a/cloudbuild.sh +++ b/cloudbuild.sh @@ -1,5 +1,19 @@ #! /bin/bash +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # shellcheck disable=SC1091 . release-tools/prow.sh diff --git a/prow.sh b/prow.sh index 161f347f6..431364055 100755 --- a/prow.sh +++ b/prow.sh @@ -1,5 +1,5 @@ #! /bin/bash -# + # Copyright 2019 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -73,7 +73,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64 # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -# Go versions can be specified seperately for different tasks +# Go versions can be specified separately for different tasks # If the pre-installed Go is missing or a different # version, the required version here will get installed # from https://golang.org/dl/. @@ -293,7 +293,7 @@ configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non- configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests" # Serial vs. parallel is always determined by these regular expressions. -# Individual regular expressions are seperated by spaces for readability +# Individual regular expressions are separated by spaces for readability # and expected to not contain spaces. Use dots instead. The complete # regex for Ginkgo will be created by joining the individual terms. configvar CSI_PROW_E2E_SERIAL '\[Serial\] \[Disruptive\]' "tags for serial E2E tests" @@ -998,7 +998,7 @@ make_test_to_junit () { echo "$line" # pass through if echo "$line" | grep -q "^### [^ ]*:$"; then if [ "$testname" ]; then - # previous test succesful + # previous test successful echo " " >>"$out" echo " " >>"$out" fi diff --git a/verify-boilerplate.sh b/verify-boilerplate.sh index ff376e545..274a16d28 100755 --- a/verify-boilerplate.sh +++ b/verify-boilerplate.sh @@ -25,12 +25,16 @@ if [[ -z "$(command -v python)" ]]; then update-alternatives --install /usr/bin/python python /usr/bin/python3 1 fi -REPO_ROOT=$(dirname "${BASH_SOURCE[0]}") +# The csi-release-tools directory. +TOOLS="$(dirname "${BASH_SOURCE[0]}")" -boilerDir="${REPO_ROOT}/boilerplate" +# Directory to check. Default is the parent of the tools themselves. +ROOT="${1:-${TOOLS}/..}" + +boilerDir="${ROOT}/boilerplate" boiler="${boilerDir}/boilerplate.py" -files_need_boilerplate=("$("${boiler}" --rootdir="${REPO_ROOT}" --verbose)") +mapfile -t files_need_boilerplate < <("${boiler}" --rootdir="${ROOT}" --verbose) # Run boilerplate.py unit tests unitTestOut="$(mktemp)" diff --git a/verify-spelling.sh b/verify-spelling.sh index 9a41c9319..38edb9985 100755 --- a/verify-spelling.sh +++ b/verify-spelling.sh @@ -20,9 +20,11 @@ set -o pipefail TOOL_VERSION="v0.3.4" -# cd to the root path -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" -cd "${ROOT}" +# The csi-release-tools directory. +TOOLS="$(dirname "${BASH_SOURCE[0]}")" + +# Directory to check. Default is the parent of the tools themselves. +ROOT="${1:-${TOOLS}/..}" # create a temporary directory TMP_DIR=$(mktemp -d) @@ -42,12 +44,12 @@ if [[ -z "$(command -v misspell)" ]]; then GO111MODULE=on GOBIN="${TMP_DIR}" go get "github.com/client9/misspell/cmd/misspell@${TOOL_VERSION}" export PATH="${TMP_DIR}:${PATH}" fi -cd "${ROOT}" # check spelling RES=0 echo "Checking spelling..." ERROR_LOG="${TMP_DIR}/errors.log" +cd "${ROOT}" git ls-files | grep -v vendor | xargs misspell > "${ERROR_LOG}" if [[ -s "${ERROR_LOG}" ]]; then sed 's/^/error: /' "${ERROR_LOG}" # add 'error' to each line to highlight in e2e status diff --git a/verify-subtree.sh b/verify-subtree.sh index f04a9fa26..aa72194a4 100755 --- a/verify-subtree.sh +++ b/verify-subtree.sh @@ -1,5 +1,5 @@ #! /bin/sh -e -# + # Copyright 2019 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); From ad29307f7c85c2a1a4de5d9b7764fc8050be901b Mon Sep 17 00:00:00 2001 From: Sachin Kumar Singh Date: Fri, 15 Jan 2021 19:56:33 +0530 Subject: [PATCH 007/124] Make the spelling and boilerplate checks optional The boilerplate and spelling tests are made optional and the default paths in the relative scripts are made absolute. --- build.make | 6 ++---- verify-boilerplate.sh | 4 ++-- verify-spelling.sh | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/build.make b/build.make index a405c6700..9b8306c06 100644 --- a/build.make +++ b/build.make @@ -278,15 +278,13 @@ check-go-version-%: # Test for spelling errors. .PHONY: test-spelling -test: test-spelling test-spelling: @ echo; echo "### $@:" - @ ./release-tools/verify-spelling.sh + @ ./release-tools/verify-spelling.sh "$(pwd)" # Test the boilerplates of the files. .PHONY: test-boilerplate -test: test-boilerplate test-boilerplate: @ echo; echo "### $@:" - @ ./release-tools/verify-boilerplate.sh + @ ./release-tools/verify-boilerplate.sh "$(pwd)" diff --git a/verify-boilerplate.sh b/verify-boilerplate.sh index 274a16d28..0781b0500 100755 --- a/verify-boilerplate.sh +++ b/verify-boilerplate.sh @@ -25,8 +25,8 @@ if [[ -z "$(command -v python)" ]]; then update-alternatives --install /usr/bin/python python /usr/bin/python3 1 fi -# The csi-release-tools directory. -TOOLS="$(dirname "${BASH_SOURCE[0]}")" +# The csi-release-tools directory (absolute path). +TOOLS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" # Directory to check. Default is the parent of the tools themselves. ROOT="${1:-${TOOLS}/..}" diff --git a/verify-spelling.sh b/verify-spelling.sh index 38edb9985..4aeb34d68 100755 --- a/verify-spelling.sh +++ b/verify-spelling.sh @@ -20,8 +20,8 @@ set -o pipefail TOOL_VERSION="v0.3.4" -# The csi-release-tools directory. -TOOLS="$(dirname "${BASH_SOURCE[0]}")" +# The csi-release-tools directory (absolute path). +TOOLS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" # Directory to check. Default is the parent of the tools themselves. ROOT="${1:-${TOOLS}/..}" From dbd896722b910fe2f041b4d9a76b2ab31b407e8d Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Sun, 17 Jan 2021 15:51:53 +0100 Subject: [PATCH 008/124] verify-boilerplate.sh: fix path to script The previous path was wrong because it was relative to the directory being checked, not the release-tools directory: csi-driver-host-path$ ./release-tools/verify-boilerplate.sh Verifying boilerplate ./release-tools/verify-boilerplate.sh: line 37: /nvme/gopath/src/github.com/kubernetes-csi/csi-driver-host-path/boilerplate/boilerplate.py: No such file or directory Done --- verify-boilerplate.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/verify-boilerplate.sh b/verify-boilerplate.sh index 0781b0500..815939574 100755 --- a/verify-boilerplate.sh +++ b/verify-boilerplate.sh @@ -31,8 +31,7 @@ TOOLS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" # Directory to check. Default is the parent of the tools themselves. ROOT="${1:-${TOOLS}/..}" -boilerDir="${ROOT}/boilerplate" -boiler="${boilerDir}/boilerplate.py" +boiler="${TOOLS}/boilerplate/boilerplate.py" mapfile -t files_need_boilerplate < <("${boiler}" --rootdir="${ROOT}" --verbose) From ece0f500ed9ee6c197ffce6d410d12edd8bf7365 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Thu, 24 Dec 2020 19:00:41 +0100 Subject: [PATCH 009/124] check namespace for snapshot-controller --- prow.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index 44511fb23..a70acbd7a 100755 --- a/prow.sh +++ b/prow.sh @@ -796,16 +796,17 @@ install_snapshot_controller() { echo "kubectl apply -f ${SNAPSHOT_CONTROLLER_YAML}(modified)" done else - echo "kubectl apply -f ${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" - kubectl apply -f "${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" + echo "kubectl apply -f $SNAPSHOT_CONTROLLER_YAML" + kubectl apply -f "$SNAPSHOT_CONTROLLER_YAML" fi cnt=0 - expected_running_pods=$(curl https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${CSI_SNAPSHOTTER_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | grep replicas | cut -d ':' -f 2-) - while [ "$(kubectl get pods -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do + expected_running_pods=$(kubectl apply --dry-run=client -o "jsonpath={.spec.replicas}" -f "$SNAPSHOT_CONTROLLER_YAML") + expected_namespace=$(kubectl apply --dry-run=client -o "jsonpath={.metadata.namespace}" -f "$SNAPSHOT_CONTROLLER_YAML") + while [ "$(kubectl get pods -n "$expected_namespace" -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do if [ $cnt -gt 30 ]; then echo "snapshot-controller pod status:" - kubectl describe pods -l app=snapshot-controller + kubectl describe pods -n "$expected_namespace" -l app=snapshot-controller echo >&2 "ERROR: snapshot controller not ready after over 5 min" exit 1 fi From 1c94220d283f8bff4dd9256bed140706f6d51cb9 Mon Sep 17 00:00:00 2001 From: fengzixu Date: Wed, 20 Jan 2021 13:20:05 +0900 Subject: [PATCH 010/124] fix: fix a bug of csi-sanity --- prow.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prow.sh b/prow.sh index 161f347f6..2a59dc0c4 100755 --- a/prow.sh +++ b/prow.sh @@ -228,7 +228,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" configvar CSI_PROW_SANITY_VERSION v4.0.2 "csi-test version" # v4.0.2 -configvar CSI_PROW_SANITY_IMPORT_PATH github.com/kubernetes-csi/csi-test/v4 "csi-test package" +configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI driver" @@ -879,8 +879,8 @@ install_sanity () ( return fi - git_checkout "${CSI_PROW_SANITY_REPO}" "${GOPATH}/src/${CSI_PROW_SANITY_IMPORT_PATH}" "${CSI_PROW_SANITY_VERSION}" --depth=1 || die "checking out csi-sanity failed" - run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed" + git_checkout "${CSI_PROW_SANITY_REPO}" "${GOPATH}/src/${CSI_PROW_SANITY_PACKAGE_PATH}" "${CSI_PROW_SANITY_VERSION}" --depth=1 || die "checking out csi-sanity failed" + ( cd "${GOPATH}/src/${CSI_PROW_SANITY_PACKAGE_PATH}/cmd/csi-sanity" && run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go build -o "${CSI_PROW_WORK}/csi-sanity" ) || die "building csi-sanity failed" ) # Captures pod output while running some other command. From f7e7ee4958a12b51dcea749368789882bc9eaea7 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 18 Dec 2020 17:34:49 +0100 Subject: [PATCH 011/124] docs: steps for adding testing against new Kubernetes release --- SIDECAR_RELEASE_PROCESS.md | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 4575eb819..59924eca3 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -104,3 +104,47 @@ naming convention `-on-`. CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) and [k/k in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) + +## Adding support for a new Kubernetes release + +1. Add the new release to `k8s_versions` in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L25 + to enable generating a job for it. Set `experimental_k8s_version` + in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L40 + to ensure that the new jobs aren't run for PRs unless explicitly + requested. Generate and submit the new jobs. +1. Create a test PR to try out the new job in some repo with `/test + pull-kubernetes-csi---on-kubernetes-` where x.y + matches the Kubernetes release. Alternatively, run .prow.sh in that + repo locally with `CSI_PROW_KUBERNETES_VERSION=x.y.z`. +1. Optional: update to a [new + release](https://github.com/kubernetes-sigs/kind/tags) of kind with + pre-built images for the new Kubernetes release. This is optional + if the current version of kind is able to build images for the new + Kubernetes release. However, jobs require less resources when they + don't need to build those images from the Kubernetes source code. + This change needs to be tried out in a PR against a component + first, then get submitted against csi-release-tools. +1. Optional: propagate the updated csi-release-tools to all components + with the script from + https://github.com/kubernetes-csi/csi-release-tools/issues/7#issuecomment-707025402 +1. Once it is likely to work in all components, unset + `experimental_k8s_version` and submit the updated jobs. +1. Once all sidecars for the new Kubernetes release are released, + either bump the version number of the images in the existing + [csi-driver-host-path + deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) + and/or create a new deployment, depending on what Kubernetes + release an updated sidecar is compatible with. If no new deployment + is needed, then add a symlink to document that there intentionally + isn't a separate deployment. This symlink is not needed for Prow + testing because that will use "kubernetes-latest" as fallback. + Update that link when creating a new deployment. +1. Create a new csi-driver-host-path release. +1. Bump `CSI_PROW_DRIVER_VERSION` in prow.sh to that new release and + (eventually) roll that change out to all repos by updating + `release-tools` in them. This is used when testing manually. The + Prow jobs override that value, so also update + `hostpath_driver_version` in + https://github.com/kubernetes/test-infra/blob/91b04e6af3a40a9bcff25aa030850a4721e2dd2b/config/jobs/kubernetes-csi/gen-jobs.sh#L46-L47 From 9f10459097335dca9659930ac0e84eebaae01892 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 26 Jan 2021 08:51:14 +0100 Subject: [PATCH 012/124] prow.sh: support building Kubernetes for a specific version Checking out the Kubernetes source code only supported branches (master, release-1.20) but not a version (like 1.20.0) because tags need a leading "v". "git clone --branch" (despite the name) is able to fetch a tag. --- prow.sh | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/prow.sh b/prow.sh index 7ccb4a4d9..d43005505 100755 --- a/prow.sh +++ b/prow.sh @@ -65,6 +65,18 @@ get_versioned_variable () { echo "$value" } +# This takes a version string like CSI_PROW_KUBERNETES_VERSION and +# maps it to the corresponding git tag, branch or commit. +version_to_git () { + version="$1" + shift + case "$version" in + latest) echo "master";; + release-*) echo "$version";; + *) echo "v$version";; + esac +} + configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only @@ -208,16 +220,7 @@ configvar CSI_PROW_DRIVER_CANARY_REGISTRY "gcr.io/k8s-staging-sig-storage" "regi # all generated files are present. # # CSI_PROW_E2E_REPO=none disables E2E testing. -tag_from_version () { - version="$1" - shift - case "$version" in - latest) echo "master";; - release-*) echo "$version";; - *) echo "v$version";; - esac -} -configvar CSI_PROW_E2E_VERSION "$(tag_from_version "${CSI_PROW_KUBERNETES_VERSION}")" "E2E version" +configvar CSI_PROW_E2E_VERSION "$(version_to_git "${CSI_PROW_KUBERNETES_VERSION}")" "E2E version" configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo" configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" @@ -465,20 +468,22 @@ git_checkout () { # This clones a repo ("https://github.com/kubernetes/kubernetes") # in a certain location ("$GOPATH/src/k8s.io/kubernetes") at -# a the head of a specific branch (i.e., release-1.13, master). -# The directory cannot exist. -git_clone_branch () { - local repo path branch parent +# a the head of a specific branch (i.e., release-1.13, master), +# tag (v1.20.0) or commit. +# +# The directory must not exist. +git_clone () { + local repo path name parent repo="$1" shift path="$1" shift - branch="$1" + name="$1" shift parent="$(dirname "$path")" mkdir -p "$parent" - (cd "$parent" && run git clone --single-branch --branch "$branch" "$repo" "$path") || die "cloning $repo" failed + (cd "$parent" && run git clone --single-branch --branch "$name" "$repo" "$path") || die "cloning $repo" failed # This is useful for local testing or when switching between different revisions in the same # repo. (cd "$path" && run git clean -fdx) || die "failed to clean $path" @@ -567,7 +572,7 @@ start_cluster () { else type="docker" fi - git_clone_branch https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version" || die "checking out Kubernetes $version failed" + git_clone https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$(version_to_git "$version")" || die "checking out Kubernetes $version failed" go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910 From 3e811d6c92b32bdd0eaf9fa370ff220f2b006271 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 1 Feb 2021 19:51:14 +0100 Subject: [PATCH 013/124] prow.sh: fix "on-master" prow jobs The recent introduction of version_to_git broke Prow jobs with CSI_PROW_KUBERNETES_VERSION=latest because that version was first converted to "master" and then once more to "vmaster". The second call must be idempotent because "master" is already a branch name. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index d43005505..ec695ec6d 100755 --- a/prow.sh +++ b/prow.sh @@ -71,7 +71,7 @@ version_to_git () { version="$1" shift case "$version" in - latest) echo "master";; + latest|master) echo "master";; release-*) echo "$version";; *) echo "v$version";; esac From 26e2ab106eeda3dc396b32d6e6eaac39526f1ccd Mon Sep 17 00:00:00 2001 From: yiyang5055 Date: Thu, 28 Jan 2021 09:18:00 +0000 Subject: [PATCH 014/124] fix: default boilerplate path --- boilerplate/boilerplate.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/boilerplate/boilerplate.py b/boilerplate/boilerplate.py index 63f74963d..5618b9ab8 100755 --- a/boilerplate/boilerplate.py +++ b/boilerplate/boilerplate.py @@ -19,8 +19,6 @@ import argparse import difflib import glob -import json -import mmap import os import re import sys @@ -33,12 +31,12 @@ nargs='*') # Rootdir defaults to the directory **above** the repo-infra dir. -rootdir = os.path.dirname(__file__) + "./../../../" +rootdir = os.path.dirname(__file__) + "./../../" rootdir = os.path.abspath(rootdir) parser.add_argument( "--rootdir", default=rootdir, help="root directory to examine") -default_boilerplate_dir = os.path.join(rootdir, "csi-driver-nfs/hack/boilerplate") +default_boilerplate_dir = os.path.abspath(os.path.dirname(__file__)) parser.add_argument( "--boilerplate-dir", default=default_boilerplate_dir) @@ -199,4 +197,4 @@ def main(): return 0 if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) From e0b02e7252aedd1a5e5ef6323e35c2c3f031327b Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 19 Jan 2021 14:12:31 +0100 Subject: [PATCH 015/124] README.md: document usage of --squash --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 60eab2a98..6315a736a 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,17 @@ is the recommended way of maintaining a copy of the rules inside the changes also locally, test them and then push them back to the shared repository at a later time. +We no longer care about importing the full commit history, so `--squash` should be used +when submitting a `release-tools` update. Also make sure that the PR for that +contains the automatically generated commit message in the PR description. +It contains the list of individual commits that were squashed. The script from +https://github.com/kubernetes-csi/csi-release-tools/issues/7 can create such +PRs automatically. + Cheat sheet: -- `git subtree add --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once) -- `git subtree pull --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes) +- `git subtree add --squash --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once) +- `git subtree pull --squash --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes) - edit, `git commit`, `git subtree push --prefix=release-tools git@github.com:/csi-release-tools.git ` - push to a new branch before submitting a PR verify-shellcheck.sh From e314a56d0750b053d8a0bf6009f68d7316a0f96f Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Sun, 21 Feb 2021 11:57:46 +0000 Subject: [PATCH 016/124] add build-arg ARCH for building multi-arch images, e.g. ARG ARCH FROM k8s.gcr.io/build-image/debian-base-${ARCH}:v2.1.3 --- build.make | 1 + 1 file changed, 1 insertion(+) diff --git a/build.make b/build.make index 9b8306c06..1faaf3b90 100644 --- a/build.make +++ b/build.make @@ -149,6 +149,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% --platform=$$os/$$arch \ --file $$(eval echo \$${dockerfile_$$os}) \ --build-arg binary=./bin/$*$$suffix \ + --build-arg ARCH=$$arch \ --label revision=$(REV) \ .; \ done; \ From ec844ea60f9b2fe61cd6dbacd610432e85e5d849 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 15 Mar 2021 11:11:11 +0100 Subject: [PATCH 017/124] remove travis.yml, Go 1.16 We no longer need TravisCI, at least not in any component branch which uses csi-release-tools master. Therefore we can remove it and make prow.sh the authoritative source for the intended Go version. While doing that, the version of Go gets bumped to 1.16 (same as Kubernetes 1.21). --- README.md | 6 +++++- cloudbuild.yaml | 2 +- prow.sh | 9 +-------- travis.yml | 21 --------------------- verify-go-version.sh | 5 +++-- 5 files changed, 10 insertions(+), 33 deletions(-) delete mode 100644 travis.yml diff --git a/README.md b/README.md index 6315a736a..b394724ce 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,11 @@ The expected repository layout is: Dockerfile in the root when only building a single command - `Makefile` - includes `release-tools/build.make` and sets configuration variables - - `.travis.yml` - a symlink to `release-tools/.travis.yml` + - `.prow.sh` script which imports `release-tools/prow.sh` + and may contain further customization + - `.cloudbuild.sh` and `cloudbuild.yaml` as symlinks to + the corresponding files in `release-tools` or (if necessary) + as custom files To create a release, tag a certain revision with a name that starts with `v`, for example `v1.0.0`, then `make push` diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 7f0a4341c..1e02ba6cb 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -25,7 +25,7 @@ options: substitution_option: ALLOW_LOOSE steps: # The image must contain bash and curl. Ideally it should also contain - # the desired version of Go (currently defined in release-tools/travis.yml), + # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' entrypoint: ./.cloudbuild.sh diff --git a/prow.sh b/prow.sh index ec695ec6d..2192b1d3a 100755 --- a/prow.sh +++ b/prow.sh @@ -85,14 +85,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64 # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -# Go versions can be specified separately for different tasks -# If the pre-installed Go is missing or a different -# version, the required version here will get installed -# from https://golang.org/dl/. -go_from_travis_yml () { - grep "^ *- go:" "${RELEASE_TOOLS_ROOT}/travis.yml" | sed -e 's/.*go: *//' -} -configvar CSI_PROW_GO_VERSION_BUILD "$(go_from_travis_yml)" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.16" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below diff --git a/travis.yml b/travis.yml deleted file mode 100644 index 1ab13aef2..000000000 --- a/travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: go -sudo: required -services: - - docker -git: - depth: false -matrix: - include: - - go: 1.15 -before_script: -- mkdir -p bin -- wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep -- chmod u+x bin/dep -- export PATH=$PWD/bin:$PATH -script: -- make -k all test GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ) -after_success: - - if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then - docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" quay.io; - make push GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ); - fi diff --git a/verify-go-version.sh b/verify-go-version.sh index f242e769d..d130126a1 100755 --- a/verify-go-version.sh +++ b/verify-go-version.sh @@ -29,8 +29,9 @@ die () { version=$("$GO" version) || die "determining version of $GO failed" # shellcheck disable=SC2001 majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/') -# shellcheck disable=SC2001 -expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/') +# SC1091: Not following: release-tools/prow.sh was not specified as input (see shellcheck -x). +# shellcheck disable=SC1091 +expected=$(. release-tools/prow.sh && echo "$$CSI_PROW_GO_VERSION_BUILD") if [ "$majorminor" != "$expected" ]; then cat >&2 < Date: Thu, 18 Mar 2021 10:17:45 +0100 Subject: [PATCH 018/124] verify-go-version.sh: fix check after removal of travis.yml Two problems: - sourcing prow.sh prints info messages about settings which should better be suppressed - the expected value was hidden by the incorrect make-style quoting of CSI_PROW_GO_VERSION_BUILD --- verify-go-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verify-go-version.sh b/verify-go-version.sh index d130126a1..c235e74fe 100755 --- a/verify-go-version.sh +++ b/verify-go-version.sh @@ -31,7 +31,7 @@ version=$("$GO" version) || die "determining version of $GO failed" majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/') # SC1091: Not following: release-tools/prow.sh was not specified as input (see shellcheck -x). # shellcheck disable=SC1091 -expected=$(. release-tools/prow.sh && echo "$$CSI_PROW_GO_VERSION_BUILD") +expected=$(. release-tools/prow.sh >/dev/null && echo "$CSI_PROW_GO_VERSION_BUILD") if [ "$majorminor" != "$expected" ]; then cat >&2 < Date: Mon, 22 Mar 2021 12:52:29 +0100 Subject: [PATCH 019/124] prow.sh: KinD 0.10.0 The new release adds a pre-built image for Kubernetes 1.20.2. --- prow.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/prow.sh b/prow.sh index 2192b1d3a..71a8707f4 100755 --- a/prow.sh +++ b/prow.sh @@ -94,17 +94,17 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # kind version to use. If the pre-installed version is different, # the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases # (if available), otherwise it is built from source. -configvar CSI_PROW_KIND_VERSION "v0.9.0" "kind" +configvar CSI_PROW_KIND_VERSION "v0.10.0" "kind" # kind images to use. Must match the kind version. # The release notes of each kind release list the supported images. -configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600 -kindest/node:v1.18.8@sha256:f4bcc97a0ad6e7abaf3f643d890add7efe6ee4ab90baeb374b4f41a4c95567eb -kindest/node:v1.17.11@sha256:5240a7a2c34bf241afb54ac05669f8a46661912eab05705d660971eeb12f6555 -kindest/node:v1.16.15@sha256:a89c771f7de234e6547d43695c7ab047809ffc71a0c3b65aa54eda051c45ed20 -kindest/node:v1.15.12@sha256:d9b939055c1e852fe3d86955ee24976cab46cba518abcb8b13ba70917e6547a6 -kindest/node:v1.14.10@sha256:ce4355398a704fca68006f8a29f37aafb49f8fc2f64ede3ccd0d9198da910146 -kindest/node:v1.13.12@sha256:1c1a48c2bfcbae4d5f4fa4310b5ed10756facad0b7a2ca93c7a4b5bae5db29f5" "kind images" +configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab +kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca +kindest/node:v1.18.15@sha256:5c1b980c4d0e0e8e7eb9f36f7df525d079a96169c8a8f20d8bd108c0d0889cc4 +kindest/node:v1.17.17@sha256:7b6369d27eee99c7a85c48ffd60e11412dc3f373658bc59b7f4d530b7056823e +kindest/node:v1.16.15@sha256:c10a63a5bda231c0a379bf91aebf8ad3c79146daca59db816fb963f731852a99 +kindest/node:v1.15.12@sha256:67181f94f0b3072fb56509107b380e38c55e23bf60e6f052fbd8052d26052fb5 +kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f5ec34578f6180" "kind images" # Use kind node-image --type=bazel by default, but allow to disable that. configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" From 1c0fb096ccff59b9ebb39a4d7059dbfaa89c3ad0 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 24 Mar 2021 09:18:00 +0100 Subject: [PATCH 020/124] prow.sh: use KinD main for latest Kubernetes KinD 0.10.0 fails to bring up a Kubernetes cluster for the latest (> 1.21.0-beta1) sources. Such breakage is expected and will be dealt with on the main branch of KinD. Therefore we should always use when building a cluster from source code. We have been lucky in the past that the released versions of KinD also happend to support the latest Kubernetes. The downside is that temporary regressions in KinD will also affect the Prow jobs that use the latest Kubernetes, but this is not new and could also have happened because of regressions in Kubernetes. While at it, building KinD gets fixed to use the configured Go version. It happened to use Go from the default PATH. The most recent KinD no longer supports the --type parameter, so we have to filter that out when using KinD main. --- prow.sh | 55 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/prow.sh b/prow.sh index 71a8707f4..b09c29200 100755 --- a/prow.sh +++ b/prow.sh @@ -91,24 +91,6 @@ configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below -# kind version to use. If the pre-installed version is different, -# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases -# (if available), otherwise it is built from source. -configvar CSI_PROW_KIND_VERSION "v0.10.0" "kind" - -# kind images to use. Must match the kind version. -# The release notes of each kind release list the supported images. -configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab -kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca -kindest/node:v1.18.15@sha256:5c1b980c4d0e0e8e7eb9f36f7df525d079a96169c8a8f20d8bd108c0d0889cc4 -kindest/node:v1.17.17@sha256:7b6369d27eee99c7a85c48ffd60e11412dc3f373658bc59b7f4d530b7056823e -kindest/node:v1.16.15@sha256:c10a63a5bda231c0a379bf91aebf8ad3c79146daca59db816fb963f731852a99 -kindest/node:v1.15.12@sha256:67181f94f0b3072fb56509107b380e38c55e23bf60e6f052fbd8052d26052fb5 -kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f5ec34578f6180" "kind images" - -# Use kind node-image --type=bazel by default, but allow to disable that. -configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" - # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. configvar CSI_PROW_GINKGO_VERSION v1.7.0 "Ginkgo" @@ -145,6 +127,38 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes" # when a Prow job just defines the Kubernetes version. csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr . _ | tr '[:lower:]' '[:upper:]' | sed -e 's/^RELEASE-//' -e 's/\([0-9]*\)_\([0-9]*\).*/\1_\2/')" +# Only the latest KinD is (eventually) guaranteed to work with the +# latest Kubernetes. For example, KinD 0.10.0 failed with Kubernetes +# 1.21.0-beta1. Therefore the default version of KinD is "main" +# for that, otherwise the latest stable release for which we then +# list the officially supported images below. +kind_version_default () { + case "${CSI_PROW_KUBERNETES_VERSION}" in + latest|master) + echo main;; + *) + echo v0.10.0;; + esac +} + +# kind version to use. If the pre-installed version is different, +# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases +# (if available), otherwise it is built from source. +configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind" + +# kind images to use. Must match the kind version. +# The release notes of each kind release list the supported images. +configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab +kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca +kindest/node:v1.18.15@sha256:5c1b980c4d0e0e8e7eb9f36f7df525d079a96169c8a8f20d8bd108c0d0889cc4 +kindest/node:v1.17.17@sha256:7b6369d27eee99c7a85c48ffd60e11412dc3f373658bc59b7f4d530b7056823e +kindest/node:v1.16.15@sha256:c10a63a5bda231c0a379bf91aebf8ad3c79146daca59db816fb963f731852a99 +kindest/node:v1.15.12@sha256:67181f94f0b3072fb56509107b380e38c55e23bf60e6f052fbd8052d26052fb5 +kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f5ec34578f6180" "kind images" + +# Use kind node-image --type=bazel by default, but allow to disable that. +configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" + # Work directory. It has to allow running executables, therefore /tmp # is avoided. Cleaning up after the script is intentionally left to # the caller. @@ -403,7 +417,7 @@ install_kind () { chmod u+x "${CSI_PROW_WORK}/bin/kind" else git_checkout https://github.com/kubernetes-sigs/kind "${GOPATH}/src/sigs.k8s.io/kind" "${CSI_PROW_KIND_VERSION}" --depth=1 && - (cd "${GOPATH}/src/sigs.k8s.io/kind" && make install INSTALL_DIR="${CSI_PROW_WORK}/bin") + (cd "${GOPATH}/src/sigs.k8s.io/kind" && run_with_go "$CSI_PROW_GO_VERSION_KIND" make install INSTALL_DIR="${CSI_PROW_WORK}/bin") fi } @@ -569,7 +583,8 @@ start_cluster () { go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910 - (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed" + # shellcheck disable=SC2046 + (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest $(if [ "$CSI_PROW_KIND_VERSION" != "main" ]; then echo --type="$type"; fi) --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" From 5b1de1adbac74ebf3c5e9155eccdfedccc042dd8 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Wed, 14 Apr 2021 10:14:52 +0200 Subject: [PATCH 021/124] go-get-kubernetes.sh: remove unused k8s libs go-get-kubernetes.sh should not add all Kubernetes libraries to "replace" section, only those that are actually used. --- go-get-kubernetes.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go-get-kubernetes.sh b/go-get-kubernetes.sh index 8c4e30244..cbbbb7c3c 100755 --- a/go-get-kubernetes.sh +++ b/go-get-kubernetes.sh @@ -55,6 +55,12 @@ mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubuserconten sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' ) || die "failed to determine Kubernetes staging modules" for mod in $mods; do + if ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then + echo "Kubernetes module $mod is not used, skipping" + # Remove the module from go.mod "replace" that was added by an older version of this script. + (set -x; env GO111MODULE=on go mod edit "-dropreplace=$mod") || die "'go mod edit' failed" + continue + fi # The presence of a potentially incomplete go.mod file affects this command, # so move elsewhere. modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") || From b46691a426e739ef9200f254af9a19e29113122b Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 23 Apr 2021 10:06:22 +0200 Subject: [PATCH 022/124] go-get-kubernetes.sh: make replace statement pruning optional Not pruning is safer because it allows adding k/k to a project at a later time. Therefore adding all replace statements is the default. A new flag can be used to clean up if desired. It's just a cosmetic change because unused replace statements are ignored by Go. --- go-get-kubernetes.sh | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/go-get-kubernetes.sh b/go-get-kubernetes.sh index cbbbb7c3c..0a960c47e 100755 --- a/go-get-kubernetes.sh +++ b/go-get-kubernetes.sh @@ -26,14 +26,43 @@ set -o pipefail cmd=$0 function help () { - echo "$cmd - update all components from kubernetes/kubernetes to that version" + cat < + +Update all components from kubernetes/kubernetes to that version. + +By default, replace statements are added for all Kubernetes packages, +whether they are used or not. This is useful when preparing a +repository for using k8s.io/kubernetes, because those replace +statements are needed to avoid "unknown revision v0.0.0" errors +(https://github.com/kubernetes/kubernetes/issues/79384). + +With the optional -p flag, all unused replace statements are +pruned. This makes go.mod smaller, but isn't required. + +The replace statements are needed for "go get -u ./..." which +otherwise ends up updating Kubernetes packages like client-go to +incompatible versions (in that case, a very old 1.x release which +happens to have a "higher" version number than the current +0.. numbers. +EOF } +prune=false + +while getopts "ph" o; do + case "$o" in + h) help; exit 0;; + p) prune=true;; + *) help; exit 1;; + esac +done +shift $((OPTIND-1)) + if [ $# -ne 1 ]; then help exit 1 fi -case "$1" in -h|--help|help) help; exit 0;; esac die () { echo >&2 "$@" @@ -55,7 +84,7 @@ mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubuserconten sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' ) || die "failed to determine Kubernetes staging modules" for mod in $mods; do - if ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then + if $prune && ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then echo "Kubernetes module $mod is not used, skipping" # Remove the module from go.mod "replace" that was added by an older version of this script. (set -x; env GO111MODULE=on go mod edit "-dropreplace=$mod") || die "'go mod edit' failed" From 6880b0c80d2c43187c8d0ce3ff61c9517a56d34d Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 26 Apr 2021 19:39:23 +0200 Subject: [PATCH 023/124] prow.sh: avoid creating paths unless really running tests verify-go-version.sh sources prow.sh to determine the desired Go version. This had the unintended side effect of creating one $GOPATH/pkg/csiprow.* directory per make invocation and triggered error messages when GOPATH was not set. Also, `go env GOPATH` is now used as fallback when not set explicitly. --- prow.sh | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/prow.sh b/prow.sh index b09c29200..aa471c0b0 100755 --- a/prow.sh +++ b/prow.sh @@ -159,11 +159,6 @@ kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f # Use kind node-image --type=bazel by default, but allow to disable that. configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" -# Work directory. It has to allow running executables, therefore /tmp -# is avoided. Cleaning up after the script is intentionally left to -# the caller. -configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" - # By default, this script tests sidecars with the CSI hostpath driver, # using the install_csi_driver function. That function depends on # a deployment script that it searches for in several places: @@ -361,10 +356,23 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external # to all the K8s versions we test against configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" -# This is the directory for additional result files. Usually set by Prow, but -# if not (for example, when invoking manually) it defaults to the work directory. -configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" -mkdir -p "${ARTIFACTS}" +# This creates directories that are required for testing. +ensure_paths () { + # Work directory. It has to allow running executables, therefore /tmp + # is avoided. Cleaning up after the script is intentionally left to + # the caller. + configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" + + # This is the directory for additional result files. Usually set by Prow, but + # if not (for example, when invoking manually) it defaults to the work directory. + configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" + mkdir -p "${ARTIFACTS}" + + # For additional tools. + CSI_PROW_BIN="${CSI_PROW_WORK}/bin" + mkdir -p "${CSI_PROW_BIN}" + PATH="${CSI_PROW_BIN}:$PATH" +} run () { echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2 @@ -384,11 +392,6 @@ die () { exit 1 } -# For additional tools. -CSI_PROW_BIN="${CSI_PROW_WORK}/bin" -mkdir -p "${CSI_PROW_BIN}" -PATH="${CSI_PROW_BIN}:$PATH" - # Ensure that PATH has the desired version of the Go tools, then run command given as argument. # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by # bumping the container image regularly. @@ -1098,6 +1101,9 @@ main () { local images ret ret=0 + # Set up work directory. + ensure_paths + images= if ${CSI_PROW_BUILD_JOB}; then # A successful build is required for testing. @@ -1259,6 +1265,9 @@ gcr_cloud_build () { # Required for "docker buildx build --push". gcloud auth configure-docker + # Might not be needed here, but call it just in case. + ensure_paths + if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then # Needed for "RUN" steps on non-linux/amd64 platforms. # See https://github.com/multiarch/qemu-user-static#getting-started From dd54c9269eb956b6d025699be3f792ca66ef06fe Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 28 Apr 2021 18:39:56 +0200 Subject: [PATCH 024/124] pull-test.sh: test importing csi-release-tools into other repo This will be used by special Prow pull jobs (https://github.com/kubernetes/test-infra/pull/21995) to ensure that an update csi-release-tools works in consuming repos. --- pull-test.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 pull-test.sh diff --git a/pull-test.sh b/pull-test.sh new file mode 100755 index 000000000..b019c1776 --- /dev/null +++ b/pull-test.sh @@ -0,0 +1,32 @@ +#! /bin/sh + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is called by pull Prow jobs for the csi-release-tools +# repo to ensure that the changes in the PR work when imported into +# some other repo. + +set -ex + +# It must be called inside the updated csi-release-tools repo. +CSI_RELEASE_TOOLS_DIR="$(pwd)" + +# Update the other repo. +cd "$PULL_TEST_REPO_DIR" +git subtree pull --squash --prefix=release-tools "$CSI_RELEASE_TOOLS_DIR" master +git log -n2 + +# Now fall through to testing. +exec ./.prow.sh From c63c61b3b23ed429103533f4bf76ad85e01e1d35 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 3 May 2021 21:26:19 +0200 Subject: [PATCH 025/124] prow.sh: add CSI_PROW_DEPLOYMENT_SUFFIX This can be used in Prow jobs which want to let prow.sh find the deployment directory based on the Kubernetes version but don't want the default "kubernetes-x.yy" deployments. The alternative would be to let Prow jobs set CSI_PROW_DEPLOYMENT, but then the fallback to kubernetes-latest wouldn't work. --- prow.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index aa471c0b0..74fe53381 100755 --- a/prow.sh +++ b/prow.sh @@ -185,8 +185,8 @@ configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation # CSI_PROW_DEPLOYMENT variable can be set in the # .prow.sh of each component when there are breaking changes # that require using a non-default deployment. The default -# is a deployment named "kubernetes-x.yy" (if available), -# otherwise "kubernetes-latest". +# is a deployment named "kubernetes-x.yy${CSI_PROW_DEPLOYMENT_SUFFIX}" (if available), +# otherwise "kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}". # "none" disables the deployment of the hostpath driver. # # When no deploy script is found (nothing in `deploy` directory, @@ -198,6 +198,7 @@ configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" +configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" # The install_csi_driver function may work also for other CSI drivers, # as long as they follow the conventions of the CSI hostpath driver. @@ -650,9 +651,9 @@ find_deployment () { # Ignore: See if you can use ${variable//search/replace} instead. # shellcheck disable=SC2001 - file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh" + file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" if ! [ -e "$file" ]; then - file="$dir/kubernetes-latest/deploy.sh" + file="$dir/kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" if ! [ -e "$file" ]; then return 1 fi From 510fb0f9c957bb8b131ea3cf5e524ecde51ce0dc Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 4 May 2021 14:17:42 +0200 Subject: [PATCH 026/124] prow.sh: support Kubernetes 1.21 Jobs that specifically use Kubernetes 1.21 have to use the unrelease KinD because the latest stable release, 0.10.0, is not compatible with it. This special case can be removed once the next KinD release is out. --- prow.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/prow.sh b/prow.sh index 74fe53381..fe23cf516 100755 --- a/prow.sh +++ b/prow.sh @@ -136,6 +136,9 @@ kind_version_default () { case "${CSI_PROW_KUBERNETES_VERSION}" in latest|master) echo main;; + 1.21*|release-1.21) + # TODO: replace this special case once the next KinD release supports 1.21. + echo main;; *) echo v0.10.0;; esac From 4a3f1103624b64d077b9739dd5436d47a2abb2ed Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 6 May 2021 17:43:12 +0200 Subject: [PATCH 027/124] prow.sh: remove obsolete test suppression The "different node" test was fixed in Kubernetes long ago enough that we no longer need to suppress it in prow.sh. --- prow.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index fe23cf516..6638ed5e2 100755 --- a/prow.sh +++ b/prow.sh @@ -354,11 +354,7 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -# -# "different node" test skips can be removed once -# https://github.com/kubernetes/kubernetes/pull/82678 has been backported -# to all the K8s versions we test against -configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" +configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" # This creates directories that are required for testing. ensure_paths () { From 74cfbc97775f1b56c7457ab1e400483bb8626f57 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 6 May 2021 17:46:25 +0200 Subject: [PATCH 028/124] prow.sh: support mock tests We already run the e2e.test binary from Kubernetes in different invocations (serial, parallel, with and without alpha features). Enabling also the in-tree mock tests is therefore easy and makes sense in particular for our canary jobs where we try out how our canary images work in existing deployments before updating those deployments. --- prow.sh | 54 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/prow.sh b/prow.sh index 6638ed5e2..27341f737 100755 --- a/prow.sh +++ b/prow.sh @@ -296,6 +296,17 @@ tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" } +# Enabling mock tests adds the "CSI mock volume" tests from https://github.com/kubernetes/kubernetes/blob/master/test/e2e/storage/csi_mock_volume.go +# to the e2e.test invocations (serial, parallel, and the corresponding alpha variants). +# When testing canary images, those get used instead of the images specified +# in the e2e.test's normal YAML files. +# +# The default is to enable this for all jobs which use canary images +# because we want to know whether our release candidates will pass all +# existing tests (the storage testsuites and mock testing in +# Kubernetes). +configvar CSI_PROW_E2E_MOCK "$(if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then echo true; else echo false; fi)" "enable CSI mock volume tests" + # Regex for non-alpha, feature-tagged tests that should be run. # configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version" @@ -873,6 +884,29 @@ start_loggers () { done } +# Patches the image versions of test/e2e/testing-manifests/storage-csi/mock in the k/k +# source code, if needed. +patch_kubernetes () { + local source="$1" target="$2" + + if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then + # We cannot replace k8s.gcr.io/sig-storage with gcr.io/k8s-staging-sig-storage because + # e2e.test does not support it (see test/utils/image/manifest.go). Instead we + # invoke the e2e.test binary with KUBE_TEST_REPO_LIST set to a file that + # overrides that registry. + find "$source/test/e2e/testing-manifests/storage-csi/mock" -name '*.yaml' -print0 | xargs -0 sed -i -e 's;k8s.gcr.io/sig-storage/\(.*\):v.*;k8s.gcr.io/sig-storage/\1:canary;' + cat >"$target/e2e-repo-list" <&2 <&1) + +EOF + fi +} + # Makes the E2E test suite binary available as "${CSI_PROW_WORK}/e2e.test". install_e2e () { if [ -e "${CSI_PROW_WORK}/e2e.test" ]; then @@ -881,6 +915,7 @@ install_e2e () { git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 && if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then + patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" && go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" && run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" @@ -932,7 +967,7 @@ run_e2e () ( trap move_junit EXIT cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" ) # Run csi-sanity against installed CSI driver. @@ -1163,6 +1198,12 @@ main () { done fi + # Run the external driver tests and optionally also mock tests. + local focus="External.Storage" + if "$CSI_PROW_E2E_MOCK"; then + focus="($focus|CSI.mock.volume)" + fi + if tests_need_non_alpha_cluster; then start_cluster || die "starting the non-alpha cluster failed" @@ -1170,6 +1211,7 @@ main () { install_snapshot_crds install_snapshot_controller + # Installing the driver might be disabled. if ${CSI_PROW_DRIVER_INSTALL} "$images"; then collect_cluster_info @@ -1184,7 +1226,7 @@ main () { # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ - -focus="External.Storage" \ + -focus="$focus" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then warn "E2E parallel failed" ret=1 @@ -1194,7 +1236,7 @@ main () { # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 if ! run_e2e parallel-features ${CSI_PROW_GINKO_PARALLEL} \ - -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ + -focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then warn "E2E parallel features failed" ret=1 @@ -1203,7 +1245,7 @@ main () { if tests_enabled "serial"; then if ! run_e2e serial \ - -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \ + -focus="$focus.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then warn "E2E serial failed" ret=1 @@ -1229,7 +1271,7 @@ main () { # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \ - -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \ + -focus="$focus.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then warn "E2E parallel alpha failed" ret=1 @@ -1238,7 +1280,7 @@ main () { if tests_enabled "serial-alpha"; then if ! run_e2e serial-alpha \ - -focus="External.Storage.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \ + -focus="$focus.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \ -skip="$(regex_join "${CSI_PROW_E2E_SKIP}")"; then warn "E2E serial alpha failed" ret=1 From 2d2e03b7fe5e14f69d5e500e521b43d44822b993 Mon Sep 17 00:00:00 2001 From: Jonathan Dobson Date: Fri, 7 May 2021 14:30:12 -0600 Subject: [PATCH 029/124] prow.sh: enable -csi.checkpathcmd option in csi-sanity This enables the -csi.checkpathcmd option, which is needed in order to test if TargetPath was successfully removed by the CSI driver as part of NodeUnpublishVolume. --- prow.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/prow.sh b/prow.sh index fe23cf516..8017171fc 100755 --- a/prow.sh +++ b/prow.sh @@ -975,6 +975,24 @@ if ! kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- exit 1 fi EOF + + cat >"${CSI_PROW_WORK}/checkdir_in_pod.sh" < Date: Tue, 11 May 2021 15:12:19 +0200 Subject: [PATCH 030/124] prow.sh: collect cluster logs We create at most two different clusters for each job, one with alpha features enabled and one without. For both the logs of system components are collected before tearing down the cluster and stored as artifacts under "cluster-logs/[non-]alpha". This is useful in particular for kubelet logs. --- prow.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/prow.sh b/prow.sh index aa471c0b0..7c6ef81f2 100755 --- a/prow.sh +++ b/prow.sh @@ -621,11 +621,16 @@ EOF # Deletes kind cluster inside a prow job delete_cluster_inside_prow_job() { + local name="$1" + # Inside a real Prow job it is better to clean up at runtime # instead of leaving that to the Prow job cleanup code # because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872). + # + # This is also a good time to collect logs. if [ "$JOB_NAME" ]; then if kind get clusters | grep -q csi-prow; then + run kind export logs --name=csi-prow "${ARTIFACTS}/cluster-logs/$name" run kind delete cluster --name=csi-prow || die "kind delete failed" fi unset KUBECONFIG @@ -1210,7 +1215,7 @@ main () { fi fi fi - delete_cluster_inside_prow_job + delete_cluster_inside_prow_job non-alpha fi if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then @@ -1245,7 +1250,7 @@ main () { fi fi fi - delete_cluster_inside_prow_job + delete_cluster_inside_prow_job alpha fi fi From 48e71f06238680e35ad4be083c5a9b869f751ebe Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Mon, 17 May 2021 20:16:59 +0530 Subject: [PATCH 031/124] Replace `which` command ( non standard) with `command -v` builtin Signed-off-by: Humble Chirammal --- verify-shellcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verify-shellcheck.sh b/verify-shellcheck.sh index fd28021ac..664b919a0 100755 --- a/verify-shellcheck.sh +++ b/verify-shellcheck.sh @@ -84,7 +84,7 @@ done < <(find . -name "*.sh" \ # detect if the host machine has the required shellcheck version installed # if so, we will use that instead. HAVE_SHELLCHECK=false -if which shellcheck &>/dev/null; then +if command -v shellcheck &>/dev/null; then detected_version="$(shellcheck --version | grep 'version: .*')" if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then HAVE_SHELLCHECK=true From bfbb6f357b9367db00aafba148aab14aa701ba38 Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Wed, 12 May 2021 03:42:43 +0000 Subject: [PATCH 032/124] add parameter base_image and addon_image to BUILD_PARAMETERS --- build.make | 49 ++++++++++++++++++++++++++++++++++++++----------- cloudbuild.yaml | 2 +- prow.sh | 17 ++++++++++------- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/build.make b/build.make index 1faaf3b90..35e43e451 100644 --- a/build.make +++ b/build.make @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# force the usage of /bin/bash instead of /bin/sh +SHELL := /bin/bash + .PHONY: build-% build container-% container push-% push clean test # A space-separated list of all commands in the repository, must be @@ -63,26 +66,35 @@ endif # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. -# BUILD_PLATFORMS contains a set of triplets, +# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image] # separated by semicolon. An empty variable or empty entry (= just a # semicolon) builds for the default platform of the current Go # toolchain. BUILD_PLATFORMS = # Add go ldflags using LDFLAGS at the time of compilation. -IMPORTPATH_LDFLAGS = -X main.version=$(REV) +IMPORTPATH_LDFLAGS = -X main.version=$(REV) EXT_LDFLAGS = -extldflags "-static" -LDFLAGS = +LDFLAGS = FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS) # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. $(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin - echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ + # os_arch_seen captures all of the $$os-$$arch seen for the current binary + # that we want to build, if we've seen an $$os-$$arch before it means that + # we don't need to build it again, this is done to avoid building + # the windows binary multiple times (see the default value of $$BUILD_PLATFORMS) + export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \ + if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \ + continue; \ + fi; \ if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \ echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ exit 1; \ fi; \ + os_arch_seen+=";$$os-$$arch"; \ done $(CMDS:%=container-%): container-%: build-% @@ -131,30 +143,46 @@ DOCKER_BUILDX_CREATE_ARGS ?= # the tag for the resulting multiarch image. $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ - DOCKER_CLI_EXPERIMENTAL=enabled; \ - export DOCKER_CLI_EXPERIMENTAL; \ + export DOCKER_CLI_EXPERIMENTAL=enabled; \ docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ trap "docker buildx rm multiarchimage-buildertest" EXIT; \ dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ if ! [ -f "$$dockerfile_windows" ]; then \ - build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$//')"; \ fi; \ pushMultiArch () { \ tag=$$1; \ - echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ docker buildx build --push \ - --tag $(IMAGE_NAME):$$arch-$$os-$$tag \ + --tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \ --platform=$$os/$$arch \ --file $$(eval echo \$${dockerfile_$$os}) \ --build-arg binary=./bin/$*$$suffix \ --build-arg ARCH=$$arch \ + --build-arg BASE_IMAGE=$$base_image \ + --build-arg ADDON_IMAGE=$$addon_image \ --label revision=$(REV) \ .; \ done; \ - images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + done); \ docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + if [ $$os = "windows" ]; then \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \ + docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \ + fi; \ + done; \ docker manifest push -p $(IMAGE_NAME):$$tag; \ }; \ if [ $(PULL_BASE_REF) = "master" ]; then \ @@ -288,4 +316,3 @@ test-spelling: test-boilerplate: @ echo; echo "### $@:" @ ./release-tools/verify-boilerplate.sh "$(pwd)" - diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 1e02ba6cb..b39a6db33 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -27,7 +27,7 @@ steps: # The image must contain bash and curl. Ideally it should also contain # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583' entrypoint: ./.cloudbuild.sh env: - GIT_TAG=${_GIT_TAG} diff --git a/prow.sh b/prow.sh index 952b0513f..7d6edfd22 100755 --- a/prow.sh +++ b/prow.sh @@ -77,7 +77,10 @@ version_to_git () { esac } -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +# the list of windows versions was matched from: +# - https://hub.docker.com/_/microsoft-windows-nanoserver +# - https://hub.docker.com/_/microsoft-windows-servercore +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because @@ -723,7 +726,7 @@ install_csi_driver () { fi } -# Installs all nessesary snapshotter CRDs +# Installs all nessesary snapshotter CRDs install_snapshot_crds() { # Wait until volumesnapshot CRDs are in place. CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd" @@ -770,7 +773,7 @@ install_snapshot_controller() { fi echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" @@ -839,7 +842,7 @@ install_snapshot_controller() { fi echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done } @@ -979,7 +982,7 @@ run_sanity () ( kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@" EOF # Using "rm -rf" as fallback for "rmdir" is a workaround for: - # Node Service + # Node Service # should work # /nvme/gopath.tmp/src/github.com/kubernetes-csi/csi-test/pkg/sanity/node.go:624 # STEP: reusing connection to CSI driver at dns:///172.17.0.2:30896 @@ -1143,7 +1146,7 @@ make_test_to_junit () { # version_gt 1.3.1 v1.2.0 (returns true) # version_gt 1.1.1 release-1.2.0 (returns false) # version_gt 1.2.0 1.2.2 (returns false) -function version_gt() { +function version_gt() { versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done) greaterVersion=${1#"release-"}; greaterVersion=${greaterVersion#"kubernetes-"}; @@ -1206,7 +1209,7 @@ main () { if [ "$rbac_file_path" == "" ]; then rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml" fi - + if [ -e "$rbac_file_path" ]; then # This is one of those components which has its own RBAC rules (like external-provisioner). # We are testing a locally built image and also want to test with the the current, From bc427931a9fc476cc6ba2e61c1cb965a827ed99c Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 19 May 2021 18:00:29 +0200 Subject: [PATCH 033/124] prow.sh: use csi-test v4.2.0 The recently added -csi.checkpathcmd and the associated test are supported only by csi-test v4.2.0. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 7d6edfd22..d01e0fbd2 100755 --- a/prow.sh +++ b/prow.sh @@ -239,7 +239,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v4.0.2 "csi-test version" # v4.0.2 +configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version" configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" From 5561780172df4a863139f8677905f112758cea82 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 19 May 2021 22:49:42 +0200 Subject: [PATCH 034/124] build.make: fix image publishng Commit bfbb6f357b9367db00aafba148aab14aa701ba38 added a sed expression with $. $ must be escaped as $$ when embedded inside a Makefile. --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index 35e43e451..0ed51e80a 100644 --- a/build.make +++ b/build.make @@ -150,7 +150,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ if ! [ -f "$$dockerfile_windows" ]; then \ - build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$//')"; \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \ fi; \ pushMultiArch () { \ tag=$$1; \ From a6453c86d7b49b6e5c42f995b015a9788471aece Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 20 May 2021 15:27:35 +0200 Subject: [PATCH 035/124] owners: introduce aliases The intention is to use these aliases in all repos by making OWNERS_ALIASES a symlink to KUBERNETES_CSI_OWNERS_ALIASES. Then adding or removing reviewers for all repos can be done by updating csi-release-tools and then updating repos using the normal semi-automatic process. While at it, several new reviewers get added to spread the load among more people and Saad gets removed from the list of active reviewers because he is too busy. --- KUBERNETES_CSI_OWNERS_ALIASES | 44 +++++++++++++++++++++++++++++++++++ OWNERS | 7 ++---- OWNERS_ALIASES | 1 + 3 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 KUBERNETES_CSI_OWNERS_ALIASES create mode 120000 OWNERS_ALIASES diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES new file mode 100644 index 000000000..6fb858aa3 --- /dev/null +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -0,0 +1,44 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + +aliases: + + # SIG-Storage chairs and leads should always have approval rights in all repos. + # Others may be added as needed here or in each repo. + kubernetes-csi-approvers: + - jsafrane + - msau42 + - saad-ali + - xing-yang + + # Reviewers are automatically assigned to new PRs. The following + # reviewers will be active in all repos. Other reviewers can be + # added in each repo. + # + # Reviewers are encouraged to set the "Busy" flag in their GitHub status + # when they are temporarily unable to review PRs. + kubernetes-csi-reviewers: + - andyzhangx + - chrishenzie + - ggriffiths + - gnufied + - j-griffith + - Jiawei0227 + - jingxu97 + - jsafrane + - pohly + - xing-yang + +# This documents who previously contributed to Kubernetes-CSI +# as approver. +emeritus_approver: +- lpabon +- sbezverk +- vladimirvivien + +# This documents who previously contributed to Kubernetes-CSI +# as reviewer. +emeritus_reviewer: +- lpabon +- saad-ali +- sbezverk +- vladimirvivien diff --git a/OWNERS b/OWNERS index 6d2f474e1..1fb74587b 100644 --- a/OWNERS +++ b/OWNERS @@ -1,11 +1,8 @@ # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md approvers: -- saad-ali -- msau42 +- kubernetes-csi-approvers - pohly reviewers: -- saad-ali -- msau42 -- pohly +- kubernetes-csi-reviewers diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES new file mode 120000 index 000000000..7ec6c034d --- /dev/null +++ b/OWNERS_ALIASES @@ -0,0 +1 @@ +KUBERNETES_CSI_OWNERS_ALIASES \ No newline at end of file From a2fcd6de4102cd68e701e7258e5086048cea4c6c Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Fri, 21 May 2021 16:47:50 +0530 Subject: [PATCH 036/124] Adding myself to csi reviewers group Signed-off-by: Humble Chirammal --- KUBERNETES_CSI_OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES index 6fb858aa3..8ea922edb 100644 --- a/KUBERNETES_CSI_OWNERS_ALIASES +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -21,6 +21,7 @@ aliases: - chrishenzie - ggriffiths - gnufied + - humblec - j-griffith - Jiawei0227 - jingxu97 From 61538bb776c5cae63d8dc40c68e6830389774c62 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 25 May 2021 15:03:39 +0200 Subject: [PATCH 037/124] prow.sh: more flexible CSI_PROW_DEPLOYMENT CSI_PROW_DEPLOYMENT was set explicitly for a variety of jobs. When testing an upcoming Kubernetes release, those jobs failed because an explicit CSI_PROW_DEPLOYMENT=kubernetes-1.21 was not matched to kubernetes-latest and thus failed because such a deployment usually only gets added much later. This must have been broken for a long time. --- prow.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/prow.sh b/prow.sh index 55f2a91cf..ca71e27a9 100755 --- a/prow.sh +++ b/prow.sh @@ -654,22 +654,30 @@ delete_cluster_inside_prow_job() { # Looks for the deployment as specified by CSI_PROW_DEPLOYMENT and CSI_PROW_KUBERNETES_VERSION # in the given directory. find_deployment () { - local dir file + local dir file k8sver dir="$1" - # Fixed deployment name? Use it if it exists, otherwise fail. + # major/minor without release- prefix. + k8sver="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/' -e 's/^release-//')" + + # Fixed deployment name? Use it if it exists. if [ "${CSI_PROW_DEPLOYMENT}" ]; then file="$dir/${CSI_PROW_DEPLOYMENT}/deploy.sh" - if ! [ -e "$file" ]; then + if [ -e "$file" ]; then + echo "$file" + return 0 + fi + + # CSI_PROW_DEPLOYMENT=kubernetes-x.yy is handled below with a fallback + # to kubernetes-latest. If it is something else, then fail here. + if ! echo "${CSI_PROW_DEPLOYMENT}" | grep -q "^kubernetes-${k8sver}\$"; then return 1 fi - echo "$file" - return 0 fi # Ignore: See if you can use ${variable//search/replace} instead. # shellcheck disable=SC2001 - file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" + file="$dir/kubernetes-${k8sver}${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" if ! [ -e "$file" ]; then file="$dir/kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" if ! [ -e "$file" ]; then From ef8bd33b4c2982370eb771627d026c7c3160072c Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 25 May 2021 20:44:01 +0200 Subject: [PATCH 038/124] prow.sh: more flexible CSI_PROW_DEPLOYMENT, part II Commit 61538bb776c5cae63d8dc40c68e6830389774c62 worked for some jobs, but not for all, for example not for CSI_PROW_KUBERNETES_VERSION=latest CSI_PROW_DEPLOYMENT=kubernetes-1.21 The approach taken now is to have one code path for all file checks regardless whether CSI_PROW_DEPLOYMENT is set or not. The first version number in the deployment name gets replaced with "latest" for the fallback. --- prow.sh | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/prow.sh b/prow.sh index ca71e27a9..07ec4dfb5 100755 --- a/prow.sh +++ b/prow.sh @@ -654,11 +654,18 @@ delete_cluster_inside_prow_job() { # Looks for the deployment as specified by CSI_PROW_DEPLOYMENT and CSI_PROW_KUBERNETES_VERSION # in the given directory. find_deployment () { - local dir file k8sver - dir="$1" + local dir="$1" + local file # major/minor without release- prefix. - k8sver="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/' -e 's/^release-//')" + local k8sver + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + k8sver="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/^release-//' -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')" + + # Desired deployment, either specified completely, including version, or derived from other variables. + local deployment + deployment=${CSI_PROW_DEPLOYMENT:-kubernetes-${k8sver}${CSI_PROW_DEPLOYMENT_SUFFIX}} # Fixed deployment name? Use it if it exists. if [ "${CSI_PROW_DEPLOYMENT}" ]; then @@ -668,18 +675,17 @@ find_deployment () { return 0 fi - # CSI_PROW_DEPLOYMENT=kubernetes-x.yy is handled below with a fallback - # to kubernetes-latest. If it is something else, then fail here. - if ! echo "${CSI_PROW_DEPLOYMENT}" | grep -q "^kubernetes-${k8sver}\$"; then - return 1 - fi + # CSI_PROW_DEPLOYMENT=kubernetes-x.yy must be mapped to kubernetes-latest + # as fallback. Same for kubernetes-distributed-x.yy. fi - # Ignore: See if you can use ${variable//search/replace} instead. - # shellcheck disable=SC2001 - file="$dir/kubernetes-${k8sver}${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" + file="$dir/${deployment}/deploy.sh" if ! [ -e "$file" ]; then - file="$dir/kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" + # Replace the first xx.yy number with "latest", for example + # kubernetes-1.21-test -> kubernetes-latest-test. + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + file="$dir/$(echo "$deployment" | sed -e 's/[0-9][0-9]*\.[0-9][0-9]*/latest/')/deploy.sh" if ! [ -e "$file" ]; then return 1 fi From 4b6aa609e0cb5e9f7af3ddf856983f93c31c74a9 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 26 May 2021 12:26:41 +0200 Subject: [PATCH 039/124] prow.sh: update to KinD v0.11.0 The new release has pre-built images for Kubernetes 1.21, so we no longer need to build those ourselves. --- prow.sh | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/prow.sh b/prow.sh index d01e0fbd2..84d4052cc 100755 --- a/prow.sh +++ b/prow.sh @@ -139,11 +139,8 @@ kind_version_default () { case "${CSI_PROW_KUBERNETES_VERSION}" in latest|master) echo main;; - 1.21*|release-1.21) - # TODO: replace this special case once the next KinD release supports 1.21. - echo main;; *) - echo v0.10.0;; + echo v0.11.0;; esac } @@ -154,13 +151,14 @@ configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind" # kind images to use. Must match the kind version. # The release notes of each kind release list the supported images. -configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab -kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca -kindest/node:v1.18.15@sha256:5c1b980c4d0e0e8e7eb9f36f7df525d079a96169c8a8f20d8bd108c0d0889cc4 -kindest/node:v1.17.17@sha256:7b6369d27eee99c7a85c48ffd60e11412dc3f373658bc59b7f4d530b7056823e -kindest/node:v1.16.15@sha256:c10a63a5bda231c0a379bf91aebf8ad3c79146daca59db816fb963f731852a99 -kindest/node:v1.15.12@sha256:67181f94f0b3072fb56509107b380e38c55e23bf60e6f052fbd8052d26052fb5 -kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f5ec34578f6180" "kind images" +configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.21.1@sha256:fae9a58f17f18f06aeac9772ca8b5ac680ebbed985e266f711d936e91d113bad +kindest/node:v1.20.7@sha256:e645428988191fc824529fd0bb5c94244c12401cf5f5ea3bd875eb0a787f0fe9 +kindest/node:v1.19.11@sha256:7664f21f9cb6ba2264437de0eb3fe99f201db7a3ac72329547ec4373ba5f5911 +kindest/node:v1.18.19@sha256:530378628c7c518503ade70b1df698b5de5585dcdba4f349328d986b8849b1ee +kindest/node:v1.17.17@sha256:c581fbf67f720f70aaabc74b44c2332cc753df262b6c0bca5d26338492470c17 +kindest/node:v1.16.15@sha256:430c03034cd856c1f1415d3e37faf35a3ea9c5aaa2812117b79e6903d1fc9651 +kindest/node:v1.15.12@sha256:8d575f056493c7778935dd855ded0e95c48cb2fab90825792e8fc9af61536bf9 +kindest/node:v1.14.10@sha256:6033e04bcfca7c5f2a9c4ce77551e1abf385bcd2709932ec2f6a9c8c0aff6d4f" "kind images" # Use kind node-image --type=bazel by default, but allow to disable that. configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" From 9e438f8e63734165937ad8bae9112e36c4873a01 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 26 May 2021 20:50:17 +0200 Subject: [PATCH 040/124] prow.sh: fix mock testing Mock testing was added in 74cfbc97 with the goal of increasing test coverage and ensuring that the sidecars and driver work once we try to use them in Kubernetes. It turned out that this is problematic in "canary-on-" jobs because those old Kubernetes version sometimes have incompatible deployments (-provisioner argument in external-provisioner!) because they did mock testing with much older sidecars. Only testing against Kubernetes master is (relatively) save and also more useful because that is where intend to use the sidecars. Another issue was that filtering of JUnit entries did not preserve the new mock volume entries. --- prow.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index 1d0bad94b..f4e0dcc4d 100755 --- a/prow.sh +++ b/prow.sh @@ -303,10 +303,12 @@ tests_need_alpha_cluster () { # in the e2e.test's normal YAML files. # # The default is to enable this for all jobs which use canary images -# because we want to know whether our release candidates will pass all -# existing tests (the storage testsuites and mock testing in -# Kubernetes). -configvar CSI_PROW_E2E_MOCK "$(if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then echo true; else echo false; fi)" "enable CSI mock volume tests" +# and the latest Kubernetes because those images will be used for mock +# testing once they are released. Using them for mock testing with +# older Kubernetes releases is too risky because the deployment files +# can be very old (for example, still using a removed -provisioner +# parameter in external-provisioner). +configvar CSI_PROW_E2E_MOCK "$(if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ] && [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ]; then echo true; else echo false; fi)" "enable CSI mock volume tests" # Regex for non-alpha, feature-tagged tests that should be run. # @@ -981,7 +983,7 @@ run_e2e () ( # the full Kubernetes E2E testsuite while only running a few tests. move_junit () { if ls "${ARTIFACTS}"/junit_[0-9]*.xml 2>/dev/null >/dev/null; then - run_filter_junit -t="External Storage" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml + run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml fi } trap move_junit EXIT From c5f59c5a12220c525bea48d783d476527fe6b9c5 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 27 May 2021 21:33:49 +0200 Subject: [PATCH 041/124] prow.sh: allow shell commands in CSI_PROW_SANITY_POD This is required for distributed provisioning where the Pod name is dynamically generated as part of the DaemonSet. --- prow.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index f4e0dcc4d..1d0732fd8 100755 --- a/prow.sh +++ b/prow.sh @@ -996,9 +996,17 @@ run_e2e () ( run_sanity () ( install_sanity || die "installing csi-sanity failed" + if [[ "${CSI_PROW_SANITY_POD}" =~ " " ]]; then + # Contains spaces, more complex than a simple pod name. + # Evaluate as a shell command. + pod=$(eval "${CSI_PROW_SANITY_POD}") || die "evaluation failed: CSI_PROW_SANITY_POD=${CSI_PROW_SANITY_POD}" + else + pod="${CSI_PROW_SANITY_POD}" + fi + cat >"${CSI_PROW_WORK}/mkdir_in_pod.sh" <"${CSI_PROW_WORK}/rmdir_in_pod.sh" < Date: Fri, 28 May 2021 08:12:19 +0200 Subject: [PATCH 042/124] prow.sh: remove Bazel build support In KinD 0.11.0, --type=bazel triggers an error: ERROR: --type is no longer supported, please remove this flag This broke Prow jobs with CSI_PROW_KUBERNETES_VERSION=release-1.[18|19|20]. --- prow.sh | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/prow.sh b/prow.sh index f4e0dcc4d..3c877ee9e 100755 --- a/prow.sh +++ b/prow.sh @@ -33,8 +33,6 @@ # The expected environment is: # - $GOPATH/src/ for the repository that is to be tested, # with PR branch merged (when testing a PR) -# - optional: bazel installed (when testing against Kubernetes master), -# must be recent enough for Kubernetes master # - running on linux-amd64 # - kind (https://github.com/kubernetes-sigs/kind) installed # - optional: Go already installed @@ -160,9 +158,6 @@ kindest/node:v1.16.15@sha256:430c03034cd856c1f1415d3e37faf35a3ea9c5aaa2812117b79 kindest/node:v1.15.12@sha256:8d575f056493c7778935dd855ded0e95c48cb2fab90825792e8fc9af61536bf9 kindest/node:v1.14.10@sha256:6033e04bcfca7c5f2a9c4ce77551e1abf385bcd2709932ec2f6a9c8c0aff6d4f" "kind images" -# Use kind node-image --type=bazel by default, but allow to disable that. -configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" - # By default, this script tests sidecars with the CSI hostpath driver, # using the install_csi_driver function. That function depends on # a deployment script that it searches for in several places: @@ -591,17 +586,12 @@ start_cluster () { if [ "$version" = "latest" ]; then version=master fi - if ${CSI_PROW_USE_BAZEL}; then - type="bazel" - else - type="docker" - fi git_clone https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$(version_to_git "$version")" || die "checking out Kubernetes $version failed" go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910 # shellcheck disable=SC2046 - (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest $(if [ "$CSI_PROW_KIND_VERSION" != "main" ]; then echo --type="$type"; fi) --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed" + (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" From 9c6a6c08798609481c6575dd594b585e905c358e Mon Sep 17 00:00:00 2001 From: Anubhav Vardhan Date: Thu, 1 Jul 2021 12:09:45 +0530 Subject: [PATCH 043/124] Master to main cleanup --- SIDECAR_RELEASE_PROCESS.md | 2 +- cloudbuild.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index f5ec71b9c..0cab8233c 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -95,7 +95,7 @@ naming convention `-on-`. 1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build). 1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From the [k8s image - repo](https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage), + repo](https://github.com/kubernetes/k8s.io/tree/main/k8s.gcr.io/images/k8s-staging-sig-storage), run `./generate.sh > images.yaml`, and send a PR with the updated images. Once merged, the image promoter will copy the images from staging to prod. 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar diff --git a/cloudbuild.yaml b/cloudbuild.yaml index b39a6db33..823fd1c5a 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -13,7 +13,7 @@ # See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md # for more details on image pushing process in Kubernetes. # -# To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage. +# To promote release images, see https://github.com/kubernetes/k8s.io/tree/main/k8s.gcr.io/images/k8s-staging-sig-storage. # This must be specified in seconds. If omitted, defaults to 600s (10 mins). # Building three images in external-snapshotter takes roughly half an hour, From fd6706935eb8bfd529f1cd6342cb0de40728e2e8 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 2 Aug 2021 12:32:32 +0200 Subject: [PATCH 044/124] docs: make github links reference HEAD instead of master --- README.md | 4 ++-- SECURITY_CONTACTS | 2 +- SIDECAR_RELEASE_PROCESS.md | 16 ++++++++-------- cloudbuild.yaml | 2 +- prow.sh | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index b394724ce..4a4e56892 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ images. Building from master creates the main `canary` image. Sharing and updating -------------------- -[`git subtree`](https://github.com/git/git/blob/master/contrib/subtree/git-subtree.txt) +[`git subtree`](https://github.com/git/git/blob/HEAD/contrib/subtree/git-subtree.txt) is the recommended way of maintaining a copy of the rules inside the `release-tools` directory of a project. This way, it is possible to make changes also locally, test them and then push them back to the shared @@ -89,7 +89,7 @@ main All Kubernetes-CSI repos are expected to switch to Prow. For details on what is enabled in Prow, see -https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-csi +https://github.com/kubernetes/test-infra/tree/HEAD/config/jobs/kubernetes-csi Test results for periodic jobs are visible in https://testgrid.k8s.io/sig-storage-csi-ci diff --git a/SECURITY_CONTACTS b/SECURITY_CONTACTS index 2af1414e0..d34984eb0 100644 --- a/SECURITY_CONTACTS +++ b/SECURITY_CONTACTS @@ -4,7 +4,7 @@ # to for triaging and handling of incoming issues. # # The below names agree to abide by the -# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) +# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/HEAD/security-release-process-documentation/security-release-process.md#embargo-policy) # and will be removed and replaced if they violate that agreement. # # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 0cab8233c..2b388b8f3 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -31,16 +31,16 @@ naming convention `-on-`. 1. "-on-master" jobs are the closest reflection to the new Kubernetes version. 1. Fixes to our prow.sh CI script can be tested in the [CSI hostpath repo](https://github.com/kubernetes-csi/csi-driver-host-path) by modifying - [prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/release-tools/prow.sh) + [prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/HEAD/release-tools/prow.sh) along with any overrides in - [.prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/.prow.sh) + [.prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/HEAD/.prow.sh) to mirror the failing environment. Once e2e tests are passing (verify-unit tests will fail), then the prow.sh changes can be submitted to [csi-release-tools](https://github.com/kubernetes-csi/csi-release-tools). 1. Changes can then be updated in all the sidecar repos and hostpath driver repo by following the [update - instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/master/README.md#sharing-and-updating). + instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/HEAD/README.md#sharing-and-updating). 1. New pull and CI jobs are configured by adding new K8s versions to the top of - [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-csi/gen-jobs.sh). + [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/HEAD/config/jobs/kubernetes-csi/gen-jobs.sh). New pull jobs that have been unverified should be initially made optional by setting the new K8s version as [experimental](https://github.com/kubernetes/test-infra/blob/a1858f46d6014480b130789df58b230a49203a64/config/jobs/kubernetes-csi/gen-jobs.sh#L40). @@ -52,7 +52,7 @@ naming convention `-on-`. 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. 1. Download v2.8+ [K8s release notes - generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) + generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes) 1. Generate release notes for the release. Replace arguments with the relevant information. * Clean up old cached information (also needed if you are generating release @@ -101,9 +101,9 @@ naming convention `-on-`. 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar and feature pages with the new released version. 1. After all the sidecars have been released, update - CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) + CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/HEAD/deploy) and [k/k - in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) + in-tree](https://github.com/kubernetes/kubernetes/tree/HEAD/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) ## Adding support for a new Kubernetes release @@ -134,7 +134,7 @@ naming convention `-on-`. 1. Once all sidecars for the new Kubernetes release are released, either bump the version number of the images in the existing [csi-driver-host-path - deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) + deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/HEAD/deploy) and/or create a new deployment, depending on what Kubernetes release an updated sidecar is compatible with. If no new deployment is needed, then add a symlink to document that there intentionally diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 823fd1c5a..30f5869cf 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -10,7 +10,7 @@ # because binaries will get built for different architectures and then # get copied from the built host into the container image # -# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md +# See https://github.com/kubernetes/test-infra/blob/HEAD/config/jobs/image-pushing/README.md # for more details on image pushing process in Kubernetes. # # To promote release images, see https://github.com/kubernetes/k8s.io/tree/main/k8s.gcr.io/images/k8s-staging-sig-storage. diff --git a/prow.sh b/prow.sh index d1c7b27fa..ed3197df2 100755 --- a/prow.sh +++ b/prow.sh @@ -292,7 +292,7 @@ tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" } -# Enabling mock tests adds the "CSI mock volume" tests from https://github.com/kubernetes/kubernetes/blob/master/test/e2e/storage/csi_mock_volume.go +# Enabling mock tests adds the "CSI mock volume" tests from https://github.com/kubernetes/kubernetes/blob/HEAD/test/e2e/storage/csi_mock_volume.go # to the e2e.test invocations (serial, parallel, and the corresponding alpha variants). # When testing canary images, those get used instead of the images specified # in the e2e.test's normal YAML files. From 999b483d624c5775835e8174ce7f9f9db0a2ffcc Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 2 Aug 2021 15:09:57 +0200 Subject: [PATCH 045/124] docs: make github links reference HEAD instead of main --- SIDECAR_RELEASE_PROCESS.md | 2 +- cloudbuild.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 2b388b8f3..071f89fba 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -95,7 +95,7 @@ naming convention `-on-`. 1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build). 1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From the [k8s image - repo](https://github.com/kubernetes/k8s.io/tree/main/k8s.gcr.io/images/k8s-staging-sig-storage), + repo](https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage), run `./generate.sh > images.yaml`, and send a PR with the updated images. Once merged, the image promoter will copy the images from staging to prod. 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 30f5869cf..d88fca6f1 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -13,7 +13,7 @@ # See https://github.com/kubernetes/test-infra/blob/HEAD/config/jobs/image-pushing/README.md # for more details on image pushing process in Kubernetes. # -# To promote release images, see https://github.com/kubernetes/k8s.io/tree/main/k8s.gcr.io/images/k8s-staging-sig-storage. +# To promote release images, see https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage. # This must be specified in seconds. If omitted, defaults to 600s (10 mins). # Building three images in external-snapshotter takes roughly half an hour, From 30a4f7bb319f5af8827ade0935e5284baf38d860 Mon Sep 17 00:00:00 2001 From: Dmitrij Fedorenko Date: Tue, 10 Aug 2021 22:46:04 +0300 Subject: [PATCH 046/124] Release armv7 image --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index ed3197df2..e9839daf9 100755 --- a/prow.sh +++ b/prow.sh @@ -78,7 +78,7 @@ version_to_git () { # the list of windows versions was matched from: # - https://hub.docker.com/_/microsoft-windows-nanoserver # - https://hub.docker.com/_/microsoft-windows-servercore -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm -arm; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because From 6a2dc64a14a39b3d023cbe8ae5beaf79b5e3df4e Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 11 Aug 2021 12:04:46 -0700 Subject: [PATCH 047/124] Remove requirement to be top-level approver. Only maintainers membership is required to do a release --- SIDECAR_RELEASE_PROCESS.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 071f89fba..e4b30e898 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -9,13 +9,8 @@ The release manager must: * Be a member of the kubernetes-csi organization. Open an [issue](https://github.com/kubernetes/org/issues/new?assignees=&labels=area%2Fgithub-membership&template=membership.md&title=REQUEST%3A+New+membership+for+%3Cyour-GH-handle%3E) in kubernetes/org to request membership -* Be a top level approver for the repository. To become a top level approver, - the candidate must demonstrate ownership and deep knowledge of the repository - through active maintenance, responding to and fixing issues, reviewing PRs, - test triage. -* Be part of the maintainers or admin group for the repository. admin is a - superset of maintainers, only maintainers level is required for cutting a - release. Membership can be requested by submitting a PR to kubernetes/org. +* Be part of the maintainers group for the repository. + Membership can be requested by submitting a PR to kubernetes/org. [Example](https://github.com/kubernetes/org/pull/1467) ## Updating CI Jobs From c0bdfb3ab90b57d1fd9b9e9d077ecab8ca38068c Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 25 Aug 2021 13:11:34 +0200 Subject: [PATCH 048/124] prow.sh: deploy canary snapshot-controller in canary jobs https://k8s-testgrid.appspot.com/sig-storage-csi-ci#canary-on-master was using the snapshot-controller from the YAML file in the master branch, which is the latest stable release. What needs to be tested instead is the canary image to detect regressions before a release. --- prow.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/prow.sh b/prow.sh index e9839daf9..d86034778 100755 --- a/prow.sh +++ b/prow.sh @@ -795,7 +795,7 @@ install_snapshot_controller() { kind load docker-image --name csi-prow ${NEW_IMG} || die "could not load the snapshot-controller:csiprow image into the kind cluster" # deploy snapshot-controller - echo "Deploying snapshot-controller" + echo "Deploying snapshot-controller from ${SNAPSHOT_CONTROLLER_YAML} with $NEW_IMG." # Replace image in SNAPSHOT_CONTROLLER_YAML with snapshot-controller:csiprow and deploy # NOTE: This logic is similar to the logic here: # https://github.com/kubernetes-csi/csi-driver-host-path/blob/v1.4.0/deploy/util/deploy-hostpath.sh#L155 @@ -832,8 +832,19 @@ install_snapshot_controller() { echo "$modified" exit 1 fi - echo "kubectl apply -f ${SNAPSHOT_CONTROLLER_YAML}(modified)" done + elif [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then + echo "Deploying snapshot-controller from ${SNAPSHOT_CONTROLLER_YAML} with canary images." + yaml="$(kubectl apply --dry-run=client -o yaml -f "$SNAPSHOT_CONTROLLER_YAML")" + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + modified="$(echo "$yaml" | sed -e "s;image: .*/\([^/:]*\):.*;image: ${CSI_PROW_DRIVER_CANARY_REGISTRY}/\1:canary;")" + diff <(echo "$yaml") <(echo "$modified") + if ! echo "$modified" | kubectl apply -f -; then + echo "modified version of $SNAPSHOT_CONTROLLER_YAML:" + echo "$modified" + exit 1 + fi else echo "kubectl apply -f $SNAPSHOT_CONTROLLER_YAML" kubectl apply -f "$SNAPSHOT_CONTROLLER_YAML" From 02cda510cc1dda9dbeb9ce018595828739c2af41 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 26 Aug 2021 11:45:40 +0200 Subject: [PATCH 049/124] build.make: support binaries outside of cmd, with optional go.mod Setting CMDS_DIR in the top-level Makefile before including build.make will ensure that binaries are picked up from that other directory. This is useful for lib-volume-populator where an example populator is provided under "example". The commands are no longer expected to share the go.mod file with the project. Entering the command directory and using `go build .` will pick up a local go.mod if it exists and otherwise fall back to the project's go.mod. --- build.make | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/build.make b/build.make index 0ed51e80a..6caf64fd5 100644 --- a/build.make +++ b/build.make @@ -21,6 +21,10 @@ SHELL := /bin/bash # set in main Makefile of a repository. # CMDS= +# Normally, commands are expected in "cmd". That can be changed for a +# repository to something else by setting CMDS_DIR before including build.make. +CMDS_DIR ?= cmd + # This is the default. It can be overridden in the main Makefile after # including build.make. REGISTRY_NAME?=quay.io/k8scsi @@ -90,7 +94,7 @@ $(CMDS:%=build-%): build-%: check-go-version-go if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \ continue; \ fi; \ - if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \ + if ! (set -x; cd ./$(CMDS_DIR)/$* && CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "$(abspath ./bin)/$*$$suffix" .); then \ echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ exit 1; \ fi; \ @@ -98,7 +102,7 @@ $(CMDS:%=build-%): build-%: check-go-version-go done $(CMDS:%=container-%): container-%: build-% - docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . + docker build -t $*:latest -f $(shell if [ -e ./$(CMDS_DIR)/$*/Dockerfile ]; then echo ./$(CMDS_DIR)/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . $(CMDS:%=push-%): push-%: container-% set -ex; \ @@ -133,7 +137,7 @@ DOCKER_BUILDX_CREATE_ARGS ?= # This target builds a multiarch image for one command using Moby BuildKit builder toolkit. # Docker Buildx is included in Docker 19.03. # -# ./cmd//Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows]. +# ./$(CMDS_DIR)//Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows]. # It is currently optional: if no such file exists, Windows images are not included, # even when Windows is listed in BUILD_PLATFORMS. That way, projects can test that # Windows binaries can be built before adding a Dockerfile for it. @@ -146,8 +150,8 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% export DOCKER_CLI_EXPERIMENTAL=enabled; \ docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ trap "docker buildx rm multiarchimage-buildertest" EXIT; \ - dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ - dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ + dockerfile_linux=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile ]; then echo ./$(CMDS_DIR)/$*/Dockerfile; else echo Dockerfile; fi); \ + dockerfile_windows=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile.Windows ]; then echo ./$(CMDS_DIR)/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ if ! [ -f "$$dockerfile_windows" ]; then \ build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \ From 44c710c5b24109c5c08476431bd44fe276ddfcc3 Mon Sep 17 00:00:00 2001 From: nick5616 Date: Fri, 3 Sep 2021 10:22:29 -0700 Subject: [PATCH 050/124] added WS2022 to build platforms --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index e9839daf9..c3e52751d 100755 --- a/prow.sh +++ b/prow.sh @@ -78,7 +78,7 @@ version_to_git () { # the list of windows versions was matched from: # - https://hub.docker.com/_/microsoft-windows-nanoserver # - https://hub.docker.com/_/microsoft-windows-servercore -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm -arm; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm -arm; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2; windows amd64 .exe nanoserver:ltsc2022 servercore:ltsc2022" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because From 0c675d4c0b5ac87f9f5966e9764d445118e59a8c Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Thu, 9 Sep 2021 16:24:34 +0000 Subject: [PATCH 051/124] Bump kind version to v0.11.1 --- prow.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/prow.sh b/prow.sh index 69c11a774..85b89007b 100755 --- a/prow.sh +++ b/prow.sh @@ -138,7 +138,7 @@ kind_version_default () { latest|master) echo main;; *) - echo v0.11.0;; + echo v0.11.1;; esac } @@ -149,14 +149,15 @@ configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind" # kind images to use. Must match the kind version. # The release notes of each kind release list the supported images. -configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.21.1@sha256:fae9a58f17f18f06aeac9772ca8b5ac680ebbed985e266f711d936e91d113bad -kindest/node:v1.20.7@sha256:e645428988191fc824529fd0bb5c94244c12401cf5f5ea3bd875eb0a787f0fe9 -kindest/node:v1.19.11@sha256:7664f21f9cb6ba2264437de0eb3fe99f201db7a3ac72329547ec4373ba5f5911 -kindest/node:v1.18.19@sha256:530378628c7c518503ade70b1df698b5de5585dcdba4f349328d986b8849b1ee -kindest/node:v1.17.17@sha256:c581fbf67f720f70aaabc74b44c2332cc753df262b6c0bca5d26338492470c17 -kindest/node:v1.16.15@sha256:430c03034cd856c1f1415d3e37faf35a3ea9c5aaa2812117b79e6903d1fc9651 -kindest/node:v1.15.12@sha256:8d575f056493c7778935dd855ded0e95c48cb2fab90825792e8fc9af61536bf9 -kindest/node:v1.14.10@sha256:6033e04bcfca7c5f2a9c4ce77551e1abf385bcd2709932ec2f6a9c8c0aff6d4f" "kind images" +configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047 +kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 +kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9 +kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729 +kindest/node:v1.18.19@sha256:7af1492e19b3192a79f606e43c35fb741e520d195f96399284515f077b3b622c +kindest/node:v1.17.17@sha256:66f1d0d91a88b8a001811e2f1054af60eef3b669a9a74f9b6db871f2f1eeed00 +kindest/node:v1.16.15@sha256:83067ed51bf2a3395b24687094e283a7c7c865ccc12a8b1d7aa673ba0c5e8861 +kindest/node:v1.15.12@sha256:b920920e1eda689d9936dfcf7332701e80be12566999152626b2c9d730397a95 +kindest/node:v1.14.10@sha256:f8a66ef82822ab4f7569e91a5bccaf27bceee135c1457c512e54de8c6f7219f8" "kind images" # By default, this script tests sidecars with the CSI hostpath driver, # using the install_csi_driver function. That function depends on From 5eeb60299803ea211774ba0a6d92fab999ff8301 Mon Sep 17 00:00:00 2001 From: sig-dagosti Date: Wed, 13 Oct 2021 20:11:59 -0400 Subject: [PATCH 052/124] images: use k8s-staging-test-infra/gcb-docker-gcloud --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index d88fca6f1..f36f265fc 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -27,7 +27,7 @@ steps: # The image must contain bash and curl. Ideally it should also contain # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583' + - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20210917-12df099d55' entrypoint: ./.cloudbuild.sh env: - GIT_TAG=${_GIT_TAG} From 0a2cf636c073a859dbc09a7f93baea7c7000e7bc Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 16 Nov 2021 14:19:40 +0100 Subject: [PATCH 053/124] prow.sh: bump Go to 1.17.3 This changes the Go version that is going to be used when building components. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 85b89007b..ac8d8b0ca 100755 --- a/prow.sh +++ b/prow.sh @@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.16" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.17.3" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below From a30efeac8538ba7a289b99672cfaf39894f3fc5a Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Tue, 21 Dec 2021 18:36:09 +0000 Subject: [PATCH 054/124] Add kind image for 1.23 --- prow.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 85b89007b..5f989db88 100755 --- a/prow.sh +++ b/prow.sh @@ -149,7 +149,8 @@ configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind" # kind images to use. Must match the kind version. # The release notes of each kind release list the supported images. -configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047 +configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.23.0@sha256:49824ab1727c04e56a21a5d8372a402fcd32ea51ac96a2706a12af38934f81ac +kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047 kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9 kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729 From 2b9897eb61e8ff5d72385a20890c4d8c4a88d0cd Mon Sep 17 00:00:00 2001 From: xing-yang Date: Wed, 5 Jan 2022 22:40:09 +0000 Subject: [PATCH 055/124] Increase build timeout --- cloudbuild.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index f36f265fc..4bcffad3a 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -16,9 +16,8 @@ # To promote release images, see https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage. # This must be specified in seconds. If omitted, defaults to 600s (10 mins). -# Building three images in external-snapshotter takes roughly half an hour, -# sometimes more. -timeout: 3600s +# Building three images in external-snapshotter takes more than an hour. +timeout: 7200s # This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, # or any new substitutions added in the future. options: From 05c18012fd984faf696dfd7466a400d5005eadad Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Fri, 7 Jan 2022 22:59:04 +0000 Subject: [PATCH 056/124] Add support to build arm/v7 images --- build.make | 28 +++++++++++++++------------- prow.sh | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/build.make b/build.make index 6caf64fd5..ff428890d 100644 --- a/build.make +++ b/build.make @@ -70,7 +70,7 @@ endif # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. -# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image] +# BUILD_PLATFORMS contains a set of tuples [os arch buildx_platform suffix base_image addon_image] # separated by semicolon. An empty variable or empty entry (= just a # semicolon) builds for the default platform of the current Go # toolchain. @@ -85,12 +85,12 @@ FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS) # defined by BUILD_PLATFORMS. $(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin - # os_arch_seen captures all of the $$os-$$arch seen for the current binary - # that we want to build, if we've seen an $$os-$$arch before it means that + # os_arch_seen captures all of the $$os-$$arch-$$buildx_platform seen for the current binary + # that we want to build, if we've seen an $$os-$$arch-$$buildx_platform before it means that # we don't need to build it again, this is done to avoid building # the windows binary multiple times (see the default value of $$BUILD_PLATFORMS) - export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ - os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \ + export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \ + os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch-$$buildx_platform*}; \ if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \ continue; \ fi; \ @@ -98,7 +98,7 @@ $(CMDS:%=build-%): build-%: check-go-version-go echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ exit 1; \ fi; \ - os_arch_seen+=";$$os-$$arch"; \ + os_arch_seen+=";$$os-$$arch-$$buildx_platform"; \ done $(CMDS:%=container-%): container-%: build-% @@ -154,16 +154,17 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% dockerfile_windows=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile.Windows ]; then echo ./$(CMDS_DIR)/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ if ! [ -f "$$dockerfile_windows" ]; then \ - build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \ fi; \ pushMultiArch () { \ tag=$$1; \ - echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \ escaped_base_image=$${base_image/:/-}; \ + escaped_buildx_platform=$${buildx_platform//\//-}; \ if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ docker buildx build --push \ - --tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \ - --platform=$$os/$$arch \ + --tag $(IMAGE_NAME):$$escaped_buildx_platform-$$os-$$escaped_base_image$$tag \ + --platform=$$os/$$buildx_platform \ --file $$(eval echo \$${dockerfile_$$os}) \ --build-arg binary=./bin/$*$$suffix \ --build-arg ARCH=$$arch \ @@ -172,13 +173,14 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% --label revision=$(REV) \ .; \ done; \ - images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \ escaped_base_image=$${base_image/:/-}; \ + escaped_buildx_platform=$${buildx_platform//\//-}; \ if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ - echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + echo $(IMAGE_NAME):$$escaped_buildx_platform-$$os-$$escaped_base_image$$tag; \ done); \ docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ - echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \ if [ $$os = "windows" ]; then \ escaped_base_image=$${base_image/:/-}; \ if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ diff --git a/prow.sh b/prow.sh index 81db64ec8..1b662519a 100755 --- a/prow.sh +++ b/prow.sh @@ -78,7 +78,7 @@ version_to_git () { # the list of windows versions was matched from: # - https://hub.docker.com/_/microsoft-windows-nanoserver # - https://hub.docker.com/_/microsoft-windows-servercore -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm -arm; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2; windows amd64 .exe nanoserver:ltsc2022 servercore:ltsc2022" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -ppc64le; linux s390x s390x -s390x; linux arm arm -arm; linux arm64 arm64 -arm64; linux arm arm/v7 -armv7; windows amd64 amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 amd64 .exe nanoserver:1909 servercore:1909; windows amd64 amd64 .exe nanoserver:2004 servercore:2004; windows amd64 amd64 .exe nanoserver:20H2 servercore:20H2; windows amd64 amd64 .exe nanoserver:ltsc2022 servercore:ltsc2022" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because From f49e141ce8b3499dfe14ea58c72925c7122e30ae Mon Sep 17 00:00:00 2001 From: Chris Henzie Date: Wed, 12 Jan 2022 11:12:32 -0800 Subject: [PATCH 057/124] Update csi-sanity test suite to v4.3.0 Allows for testing of the SINGLE_NODE_MULTI_WRITER capability. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 1b662519a..819d13dbd 100755 --- a/prow.sh +++ b/prow.sh @@ -234,7 +234,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version" +configvar CSI_PROW_SANITY_VERSION v4.3.0 "csi-test version" configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" From 81e26c3f8583444120a97c380145735446d01564 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 19 Jan 2022 20:55:05 +0100 Subject: [PATCH 058/124] SIDECAR_RELEASE_PROCESS.md: add troubleshooting for image publishing --- SIDECAR_RELEASE_PROCESS.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index e4b30e898..1f3ed3116 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -100,6 +100,29 @@ naming convention `-on-`. and [k/k in-tree](https://github.com/kubernetes/kubernetes/tree/HEAD/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) +### Troubleshooting + +#### Image build jobs + +The following jobs are triggered after tagging to produce the corresponding +image(s): +https://k8s-testgrid.appspot.com/sig-storage-image-build + +Clicking on a failed build job opens that job in https://prow.k8s.io. Next to +the job title is a rerun icon (circle with arrow). Clicking it opens a popup +with a "rerun" button that maintainers with enough permissions can use. If in +doubt, ask someone on #sig-release to rerun the job. + +Another way to rerun a job is to search for it in https://prow.k8s.io and click +the rerun icon in the resulting job list: +https://prow.k8s.io/?job=canary-csi-test-push-images + +#### Verify images + +Canary and staged images can be viewed at https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage + +Promoted images can be viewed at https://console.cloud.google.com/gcr/images/k8s-artifacts-prod/us/sig-storage + ## Adding support for a new Kubernetes release 1. Add the new release to `k8s_versions` in From d066f1ba6fe61c24e6f6307d7bccf32b73d2dc66 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Tue, 25 Jan 2022 15:53:50 +0530 Subject: [PATCH 059/124] Correct prow.sh typo and make codespell linter pass Signed-off-by: Humble Chirammal --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 819d13dbd..1f2d0d9c3 100755 --- a/prow.sh +++ b/prow.sh @@ -737,7 +737,7 @@ install_csi_driver () { fi } -# Installs all nessesary snapshotter CRDs +# Installs all necessary snapshotter CRDs install_snapshot_crds() { # Wait until volumesnapshot CRDs are in place. CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd" From 59ae38b7bfcab3ccac7bd4b339c89f57d3b98267 Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Wed, 26 Jan 2022 22:10:08 +0000 Subject: [PATCH 060/124] Remove EOL windows versions from BUILD_PLATFORMS --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 1f2d0d9c3..8053b205a 100755 --- a/prow.sh +++ b/prow.sh @@ -78,7 +78,7 @@ version_to_git () { # the list of windows versions was matched from: # - https://hub.docker.com/_/microsoft-windows-nanoserver # - https://hub.docker.com/_/microsoft-windows-servercore -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -ppc64le; linux s390x s390x -s390x; linux arm arm -arm; linux arm64 arm64 -arm64; linux arm arm/v7 -armv7; windows amd64 amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 amd64 .exe nanoserver:1909 servercore:1909; windows amd64 amd64 .exe nanoserver:2004 servercore:2004; windows amd64 amd64 .exe nanoserver:20H2 servercore:20H2; windows amd64 amd64 .exe nanoserver:ltsc2022 servercore:ltsc2022" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -ppc64le; linux s390x s390x -s390x; linux arm arm -arm; linux arm64 arm64 -arm64; linux arm arm/v7 -armv7; windows amd64 amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 amd64 .exe nanoserver:20H2 servercore:20H2; windows amd64 amd64 .exe nanoserver:ltsc2022 servercore:ltsc2022" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because From f1de2c66b373fa7f092828958d85482a8fa12cb7 Mon Sep 17 00:00:00 2001 From: Garima Date: Tue, 8 Feb 2022 23:45:51 -0800 Subject: [PATCH 061/124] Fix OWNERS file - squashed commits --- KUBERNETES_CSI_OWNERS_ALIASES | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES index 8ea922edb..2f3e2dfc9 100644 --- a/KUBERNETES_CSI_OWNERS_ALIASES +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -31,15 +31,7 @@ aliases: # This documents who previously contributed to Kubernetes-CSI # as approver. -emeritus_approver: +emeritus_approvers: - lpabon - sbezverk - vladimirvivien - -# This documents who previously contributed to Kubernetes-CSI -# as reviewer. -emeritus_reviewer: -- lpabon -- saad-ali -- sbezverk -- vladimirvivien From 274bc9ba42c4e592eb51a4ee069d4f2afea8ccbe Mon Sep 17 00:00:00 2001 From: Peter Schuurman Date: Wed, 9 Feb 2022 12:22:11 -0800 Subject: [PATCH 062/124] Update Sidecar Release Process documentation to reference latest syntax for release-notes tool --- SIDECAR_RELEASE_PROCESS.md | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 1f3ed3116..8977fbe6d 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -46,10 +46,13 @@ naming convention `-on-`. ## Release Process 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. -1. Download v2.8+ [K8s release notes - generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes) +1. Download the latest version of the + [K8s release notes generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes) +1. Create a + [Github personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) + with `repo:public_repo` access 1. Generate release notes for the release. Replace arguments with the relevant - information. + information. * Clean up old cached information (also needed if you are generating release notes for multiple repos) ```bash @@ -57,15 +60,24 @@ naming convention `-on-`. ``` * For new minor releases on master: ```bash - GITHUB_TOKEN= release-notes --discover=mergebase-to-latest - --github-org=kubernetes-csi --github-repo=external-provisioner - --required-author="" --output out.md + GITHUB_TOKEN= release-notes \ + --discover=mergebase-to-latest \ + --org=kubernetes-csi \ + --repo=external-provisioner \ + --required-author="" \ + --markdown-links \ + --output out.md ``` * For new patch releases on a release branch: ```bash - GITHUB_TOKEN= release-notes --discover=patch-to-latest --branch=release-1.1 - --github-org=kubernetes-csi --github-repo=external-provisioner - --required-author="" --output out.md + GITHUB_TOKEN= release-notes \ + --discover=patch-to-latest \ + --branch=release-1.1 \ + --org=kubernetes-csi \ + --repo=external-provisioner \ + --required-author="" \ + --markdown-links \ + --output out.md ``` 1. Compare the generated output to the new commits for the release to check if any notable change missed a release note. From db917f5c17a890753dc0ec4a2ac7315f41813245 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 1 Apr 2022 16:43:08 +0200 Subject: [PATCH 063/124] update to Go 1.18 This is the latest stable release and the one used by Kubernetes 1.24. --- prow.sh | 7 ++----- verify-spelling.sh | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/prow.sh b/prow.sh index 8053b205a..e6df4763b 100755 --- a/prow.sh +++ b/prow.sh @@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.17.3" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.18" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below @@ -441,10 +441,7 @@ install_ginkgo () { if [ "v$(ginkgo version 2>/dev/null | sed -e 's/.* //')" = "${CSI_PROW_GINKGO_VERSION}" ]; then return fi - git_checkout https://github.com/onsi/ginkgo "$GOPATH/src/github.com/onsi/ginkgo" "${CSI_PROW_GINKGO_VERSION}" --depth=1 && - # We have to get dependencies and hence can't call just "go build". - run_with_go "${CSI_PROW_GO_VERSION_GINKGO}" go get github.com/onsi/ginkgo/ginkgo || die "building ginkgo failed" && - mv "$GOPATH/bin/ginkgo" "${CSI_PROW_BIN}" + run_with_go "${CSI_PROW_GO_VERSION_GINKGO}" env GOBIN="${CSI_PROW_BIN}" go install "github.com/onsi/ginkgo/ginkgo@${CSI_PROW_GINKGO_VERSION}" || die "building ginkgo failed" } # Ensure that we have the desired version of dep. diff --git a/verify-spelling.sh b/verify-spelling.sh index 4aeb34d68..6e87b1dbf 100755 --- a/verify-spelling.sh +++ b/verify-spelling.sh @@ -41,7 +41,7 @@ if [[ -z "$(command -v misspell)" ]]; then # perform go get in a temp dir as we are not tracking this version in a go module # if we do the go get in the repo, it will create / update a go.mod and go.sum cd "${TMP_DIR}" - GO111MODULE=on GOBIN="${TMP_DIR}" go get "github.com/client9/misspell/cmd/misspell@${TOOL_VERSION}" + GO111MODULE=on GOBIN="${TMP_DIR}" go install "github.com/client9/misspell/cmd/misspell@${TOOL_VERSION}" export PATH="${TMP_DIR}:${PATH}" fi From 9a0260c55cc2d6ec6ff1f82fdd2372a188bc5940 Mon Sep 17 00:00:00 2001 From: Andy Zhang Date: Mon, 25 Apr 2022 21:07:19 +0800 Subject: [PATCH 064/124] fix boilerplate header --- go-get-kubernetes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-get-kubernetes.sh b/go-get-kubernetes.sh index 0a960c47e..2b46c187a 100755 --- a/go-get-kubernetes.sh +++ b/go-get-kubernetes.sh @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# + # This script can be used while converting a repo from "dep" to "go mod" # by calling it after "go mod init" or to update the Kubernetes packages # in a repo that has already been converted. Only packages that are From 84a4d5a1d3f1b5600599808f7d678c5c020a4caf Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Fri, 3 Jun 2022 10:26:09 -0400 Subject: [PATCH 065/124] Move from k8s.gcr.io to registry.k8s.io --- prow.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prow.sh b/prow.sh index e6df4763b..74cc017ab 100755 --- a/prow.sh +++ b/prow.sh @@ -811,7 +811,7 @@ install_snapshot_controller() { modified="$(cat "$i" | while IFS= read -r line; do nocomments="$(echo "$line" | sed -e 's/ *#.*$//')" if echo "$nocomments" | grep -q '^[[:space:]]*image:[[:space:]]*'; then - # Split 'image: k8s.gcr.io/sig-storage/snapshot-controller:v3.0.0' + # Split 'image: registry.k8s.io/sig-storage/snapshot-controller:v3.0.0' # into image (snapshot-controller:v3.0.0), # name (snapshot-controller), # tag (v3.0.0). @@ -912,11 +912,11 @@ patch_kubernetes () { local source="$1" target="$2" if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then - # We cannot replace k8s.gcr.io/sig-storage with gcr.io/k8s-staging-sig-storage because + # We cannot replace registry.k8s.io/sig-storage with gcr.io/k8s-staging-sig-storage because # e2e.test does not support it (see test/utils/image/manifest.go). Instead we # invoke the e2e.test binary with KUBE_TEST_REPO_LIST set to a file that # overrides that registry. - find "$source/test/e2e/testing-manifests/storage-csi/mock" -name '*.yaml' -print0 | xargs -0 sed -i -e 's;k8s.gcr.io/sig-storage/\(.*\):v.*;k8s.gcr.io/sig-storage/\1:canary;' + find "$source/test/e2e/testing-manifests/storage-csi/mock" -name '*.yaml' -print0 | xargs -0 sed -i -e 's;registry.k8s.io/sig-storage/\(.*\):v.*;registry.k8s.io/sig-storage/\1:canary;' cat >"$target/e2e-repo-list" < Date: Thu, 9 Jun 2022 18:27:58 +0200 Subject: [PATCH 066/124] fix testing with latest Kubernetes It failed for two reasons: - CSIDriver v1beta1 as used in csi-driver-host-path v1.3.0 is no longer supported. - GenericEphemeralVolume was removed in Kubernetes master. --- prow.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/prow.sh b/prow.sh index 74cc017ab..b4957a895 100755 --- a/prow.sh +++ b/prow.sh @@ -196,7 +196,7 @@ kindest/node:v1.14.10@sha256:f8a66ef82822ab4f7569e91a5bccaf27bceee135c1457c512e5 # If the deployment script is called with CSI_PROW_TEST_DRIVER= as # environment variable, then it must write a suitable test driver configuration # into that file in addition to installing the driver. -configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version" +configvar CSI_PROW_DRIVER_VERSION "v1.8.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" @@ -346,7 +346,7 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # kubernetes-csi components must be updated, either by disabling # the failing test for "latest" or by updating the test and not running # it anymore for older releases. -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'GenericEphemeralVolume=true,CSIStorageCapacity=true' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment @@ -1304,7 +1304,11 @@ main () { delete_cluster_inside_prow_job non-alpha fi - if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then + # If the cluster for alpha tests doesn't need any feature gates, then we + # could reuse the same cluster as for the other tests. But that would make + # the flow in this script harder and wouldn't help in practice because + # we have separate Prow jobs for alpha and non-alpha tests. + if tests_need_alpha_cluster; then # Need to (re)create the cluster. start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed" From e51ff2cc0f58b80052f585f57d080f4251d0108e Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Fri, 10 Jun 2022 07:45:23 +0530 Subject: [PATCH 067/124] introduce control variable for non alpha feature gate configuration Some of the CSI components may need to tweak the by default enabled feature gates for latest clusters for their operations. This configuration knob helps them for controlling it. Signed-off-by: Humble Chirammal --- prow.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index b4957a895..dfe6f5d6e 100755 --- a/prow.sh +++ b/prow.sh @@ -349,6 +349,9 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" +configvar CSI_PROW_E2E_GATES_LATEST '' "non alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_GATES "$(get_versioned_variable CSI_PROW_E2E_GATES "${csi_prow_kubernetes_version_suffix}")" "non alpha E2E feature gates" + # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment default_csi_snapshotter_version () { if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ] || [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then @@ -1254,7 +1257,8 @@ main () { fi if tests_need_non_alpha_cluster; then - start_cluster || die "starting the non-alpha cluster failed" + # Need to (re)create the cluster. + start_cluster "${CSI_PROW_E2E_GATES}" || die "starting the non-alpha cluster failed" # Install necessary snapshot CRDs and snapshot controller install_snapshot_crds From b86d8e942eb313c76286f2542a4ed916567c59fe Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Aug 2022 15:40:43 +0200 Subject: [PATCH 068/124] support Kubernetes 1.25 + Ginkgo v2 Kubernetes 1.25 is switching to Ginkgo v2 for the e2e.test suite. This has two implications: - we must built a matching ginkgo binary because communication between the ginkgo binary and library changed, affecting for example running of parallel tests: this is done by using the Kubernetes make rules to build the binary when using the Kubernetes e2e.test suite - the ginkgo v2 JUnit file is slightly different: filter-junit.go now first tries the old format, then the new one --- filter-junit.go | 20 ++++++++++++++++++-- prow.sh | 13 +++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/filter-junit.go b/filter-junit.go index cf1cb4abf..5454092b1 100644 --- a/filter-junit.go +++ b/filter-junit.go @@ -35,10 +35,18 @@ var ( ) /* - * TestSuite represents a JUnit file. Due to how encoding/xml works, we have + * TestResults represents a JUnit file. Due to how encoding/xml works, we have * represent all fields that we want to be passed through. It's therefore * not a complete solution, but good enough for Ginkgo + Spyglass. + * + * Before Kubernetes 1.25 and ginkgo v2, we directly had in the + * JUnit file. Now we get and inside it the . */ +type TestResults struct { + XMLName string `xml:"testsuites"` + TestSuite TestSuite `xml:"testsuite"` +} + type TestSuite struct { XMLName string `xml:"testsuite"` TestCases []TestCase `xml:"testcase"` @@ -93,7 +101,15 @@ func main() { } } if err := xml.Unmarshal(data, &junit); err != nil { - panic(err) + if err.Error() != "expected element type but have " { + panic(err) + } + // Fall back to Ginkgo v2 format. + var junitv2 TestResults + if err := xml.Unmarshal(data, &junitv2); err != nil { + panic(err) + } + junit = junitv2.TestSuite } } diff --git a/prow.sh b/prow.sh index dfe6f5d6e..48c7e657a 100755 --- a/prow.sh +++ b/prow.sh @@ -93,7 +93,10 @@ configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version fo configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below # ginkgo test runner version to use. If the pre-installed version is -# different, the desired version is built from source. +# different, the desired version is built from source. For Kubernetes, +# the version built via "make WHAT=vendor/github.com/onsi/ginkgo/ginkgo" is +# used, which is guaranteed to match what the Kubernetes e2e.test binary +# needs. configvar CSI_PROW_GINKGO_VERSION v1.7.0 "Ginkgo" # Ginkgo runs the E2E test in parallel. The default is based on the number @@ -440,6 +443,10 @@ install_kind () { # Ensure that we have the desired version of the ginkgo test runner. install_ginkgo () { + if [ -e "${CSI_PROW_BIN}/ginkgo" ]; then + return + fi + # CSI_PROW_GINKGO_VERSION contains the tag with v prefix, the command line output does not. if [ "v$(ginkgo version 2>/dev/null | sed -e 's/.* //')" = "${CSI_PROW_GINKGO_VERSION}" ]; then return @@ -943,7 +950,9 @@ install_e2e () { patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" && go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" && run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" + ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" && + run_with_go "$go_version" make WHAT=vendor/github.com/onsi/ginkgo/ginkgo "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && + ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/ginkgo" "${CSI_PROW_BIN}" else run_with_go "${CSI_PROW_GO_VERSION_E2E}" go test -c -o "${CSI_PROW_WORK}/e2e.test" "${CSI_PROW_E2E_IMPORT_PATH}/test/e2e" fi From 41cb70d3854c5961c153695a11e106854a4f326c Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 4 Aug 2022 09:19:53 +0200 Subject: [PATCH 069/124] prow.sh: sanity testing with csi-test v5.0.0 The main difference is that csi-sanity is now built with Ginkgo v2. It should work as before. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 48c7e657a..5cddb69fe 100755 --- a/prow.sh +++ b/prow.sh @@ -237,7 +237,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v4.3.0 "csi-test version" +configvar CSI_PROW_SANITY_VERSION v5.0.0 "csi-test version" configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" From 7577454a7afa4f4aeea75671e31106a1126fa6bb Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 5 Aug 2022 10:50:34 +0200 Subject: [PATCH 070/124] prow.sh: bump Kubernetes to v1.22.0 This update is necessary because the default driver version (csi-driver-host-path v1.8.0) no longer supports Kubernetes v1.17.0 (it needs the v1 CSIDriver API). 1.22 is the oldest supported version at this point. This matches what we are using as base in Prow at the moment. v1.22.0 is what the current kind has pre-built images for. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 5cddb69fe..ba64d7198 100755 --- a/prow.sh +++ b/prow.sh @@ -121,7 +121,7 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # use the same settings as for "latest" Kubernetes. This works # as long as there are no breaking changes in Kubernetes, like # deprecating or changing the implementation of an alpha feature. -configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes" +configvar CSI_PROW_KUBERNETES_VERSION 1.22.0 "Kubernetes" # CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and # with underscore (1_13 instead of 1.13.3) and in uppercase (LATEST From 70915a8eaec27fee6041e30a3a0a3f7bd0958f4a Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 5 Aug 2022 19:10:50 +0200 Subject: [PATCH 071/124] prow.sh: update snapshotter version The right snapshotter for Kubernetes 1.22 is v4.0.0. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index ba64d7198..f5ba89b19 100755 --- a/prow.sh +++ b/prow.sh @@ -360,7 +360,7 @@ default_csi_snapshotter_version () { if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ] || [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then echo "master" else - echo "v3.0.2" + echo "v4.0.0" fi } configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external-snapshotter version tag" From 4ddce251b9e41c23e239a82cd600de93df82ff80 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Thu, 18 Aug 2022 14:57:55 -0400 Subject: [PATCH 072/124] Add 1.24 Kind image --- prow.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index f5ba89b19..d2c4a17f6 100755 --- a/prow.sh +++ b/prow.sh @@ -152,7 +152,8 @@ configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind" # kind images to use. Must match the kind version. # The release notes of each kind release list the supported images. -configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.23.0@sha256:49824ab1727c04e56a21a5d8372a402fcd32ea51ac96a2706a12af38934f81ac +configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.24.0@sha256:0866296e693efe1fed79d5e6c7af8df71fc73ae45e3679af05342239cdc5bc8e +kindest/node:v1.23.0@sha256:49824ab1727c04e56a21a5d8372a402fcd32ea51ac96a2706a12af38934f81ac kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047 kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9 From 0faa3fc7bad0d398696788defa9fb4a8d72bfe87 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Fri, 19 Aug 2022 11:58:34 -0400 Subject: [PATCH 073/124] Update to Kind v0.14.0 images --- prow.sh | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/prow.sh b/prow.sh index d2c4a17f6..63d844485 100755 --- a/prow.sh +++ b/prow.sh @@ -141,7 +141,7 @@ kind_version_default () { latest|master) echo main;; *) - echo v0.11.1;; + echo v0.14.0;; esac } @@ -153,16 +153,12 @@ configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind" # kind images to use. Must match the kind version. # The release notes of each kind release list the supported images. configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.24.0@sha256:0866296e693efe1fed79d5e6c7af8df71fc73ae45e3679af05342239cdc5bc8e -kindest/node:v1.23.0@sha256:49824ab1727c04e56a21a5d8372a402fcd32ea51ac96a2706a12af38934f81ac -kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047 -kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 -kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9 -kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729 -kindest/node:v1.18.19@sha256:7af1492e19b3192a79f606e43c35fb741e520d195f96399284515f077b3b622c -kindest/node:v1.17.17@sha256:66f1d0d91a88b8a001811e2f1054af60eef3b669a9a74f9b6db871f2f1eeed00 -kindest/node:v1.16.15@sha256:83067ed51bf2a3395b24687094e283a7c7c865ccc12a8b1d7aa673ba0c5e8861 -kindest/node:v1.15.12@sha256:b920920e1eda689d9936dfcf7332701e80be12566999152626b2c9d730397a95 -kindest/node:v1.14.10@sha256:f8a66ef82822ab4f7569e91a5bccaf27bceee135c1457c512e54de8c6f7219f8" "kind images" +kindest/node:v1.23.6@sha256:b1fa224cc6c7ff32455e0b1fd9cbfd3d3bc87ecaa8fcb06961ed1afb3db0f9ae +kindest/node:v1.22.9@sha256:8135260b959dfe320206eb36b3aeda9cffcb262f4b44cda6b33f7bb73f453105 +kindest/node:v1.21.12@sha256:f316b33dd88f8196379f38feb80545ef3ed44d9197dca1bfd48bcb1583210207 +kindest/node:v1.20.15@sha256:6f2d011dffe182bad80b85f6c00e8ca9d86b5b8922cdf433d53575c4c5212248 +kindest/node:v1.19.16@sha256:d9c819e8668de8d5030708e484a9fdff44d95ec4675d136ef0a0a584e587f65c +kindest/node:v1.18.20@sha256:738cdc23ed4be6cc0b7ea277a2ebcc454c8373d7d8fb991a7fcdbd126188e6d7" "kind images" # By default, this script tests sidecars with the CSI hostpath driver, # using the install_csi_driver function. That function depends on From 7ccc959455d9111d376225448d4739c966cf2d17 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Tue, 6 Sep 2022 11:50:47 +0530 Subject: [PATCH 074/124] release tools update to 1.19 This commit update go version to 1.19 in prow configuration Signed-off-by: Humble Chirammal --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 63d844485..47c64e533 100755 --- a/prow.sh +++ b/prow.sh @@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.18" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.19" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below From 4edd1d8adc1717443b46fffc0f51731073df3018 Mon Sep 17 00:00:00 2001 From: Raunak Pradip Shah Date: Fri, 23 Sep 2022 12:30:59 +0530 Subject: [PATCH 075/124] Add RaunakShah to CSI reviewers group --- KUBERNETES_CSI_OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES index 2f3e2dfc9..344fbe4db 100644 --- a/KUBERNETES_CSI_OWNERS_ALIASES +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -27,6 +27,7 @@ aliases: - jingxu97 - jsafrane - pohly + - RaunakShah - xing-yang # This documents who previously contributed to Kubernetes-CSI From 7d410d8888bb27fbfcd2af04737506467e71ed77 Mon Sep 17 00:00:00 2001 From: Raunak Pradip Shah Date: Thu, 22 Sep 2022 10:22:40 +0530 Subject: [PATCH 076/124] Changes to csi prow to run e2e tests in sidecars --- prow.sh | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index 63d844485..55c43bc08 100755 --- a/prow.sh +++ b/prow.sh @@ -228,6 +228,9 @@ configvar CSI_PROW_E2E_VERSION "$(version_to_git "${CSI_PROW_KUBERNETES_VERSION} configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo" configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" +# Local path for e2e tests. Set to "none" to disable. +configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package" + # csi-sanity testing from the csi-test repo can be run against the installed # CSI driver. For this to work, deploying the driver must expose the Unix domain # csi.sock as a TCP service for use by the csi-sanity command, which runs outside @@ -282,13 +285,18 @@ tests_enabled () { sanity_enabled () { [ "${CSI_PROW_TESTS_SANITY}" = "sanity" ] && tests_enabled "sanity" } + +sidecar_tests_enabled () { + [ "${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" != "none" ] +} + tests_need_kind () { tests_enabled "parallel" "serial" "serial-alpha" "parallel-alpha" || - sanity_enabled + sanity_enabled || sidecar_tests_enabled } tests_need_non_alpha_cluster () { tests_enabled "parallel" "serial" || - sanity_enabled + sanity_enabled || sidecar_tests_enabled } tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" @@ -352,6 +360,11 @@ configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_ configvar CSI_PROW_E2E_GATES_LATEST '' "non alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_GATES "$(get_versioned_variable CSI_PROW_E2E_GATES "${csi_prow_kubernetes_version_suffix}")" "non alpha E2E feature gates" +# Focus for local tests run in the sidecar E2E repo. Only used if CSI_PROW_SIDECAR_E2E_IMPORT_PATH +# is not set to "none". If empty, all tests in the sidecar repo will be run. +configvar CSI_PROW_SIDECAR_E2E_FOCUS '' "tags for local E2E tests" +configvar CSI_PROW_SIDECAR_E2E_SKIP '' "local tests that need to be skipped" + # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment default_csi_snapshotter_version () { if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ] || [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then @@ -942,6 +955,9 @@ install_e2e () { return fi + if sidecar_tests_enabled; then + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" go test -c -o "${CSI_PROW_WORK}/e2e-local.test" "${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" + fi git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 && if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" && @@ -997,8 +1013,13 @@ run_e2e () ( } trap move_junit EXIT - cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" + if [ "${name}" == "local" ]; then + cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" && + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e-local.test" -- -report-dir "${ARTIFACTS}" -report-prefix local + else + cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" + fi ) # Run csi-sanity against installed CSI driver. @@ -1310,6 +1331,15 @@ main () { ret=1 fi fi + + if sidecar_tests_enabled; then + if ! run_e2e local \ + -focus="${CSI_PROW_SIDECAR_E2E_FOCUS}" \ + -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then + warn "E2E sidecar failed" + ret=1 + fi + fi fi delete_cluster_inside_prow_job non-alpha fi From 36e433e2aaaaf96b3257812cecd9bc55a1f71999 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Thu, 10 Nov 2022 09:55:57 +0100 Subject: [PATCH 077/124] Skip SELinux tests in CI by default Currently, no CI job has SELinux configured, so skip all tests that require it by default.. When a job supports SELinux, it can set CSI_PROW_E2E_SKIP='Disruptive' in its prow yaml. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 061021eb2..55c3231d0 100755 --- a/prow.sh +++ b/prow.sh @@ -381,7 +381,7 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" +configvar CSI_PROW_E2E_SKIP '\[Disruptive\]|\[Feature:SELinux\]' "tests that need to be skipped" # This creates directories that are required for testing. ensure_paths () { From 38e11468fa6bf056f5be9748bb708a9612c6f2b9 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Sat, 17 Dec 2022 17:42:13 +0100 Subject: [PATCH 078/124] prow.sh: publish individual JUnit files as separate artifacts "junit/steps/junit_*.xml" contains the summary JUnit file of each step (sanity, make_test, etc.). The original files as written by the e2e.test binary are under "junit/" where name is "parallel", "serial", etc. This makes it possible to look up information that might get lost during the transformation with filter-junit.go and debug that command. --- prow.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index 55c3231d0..6154bd597 100755 --- a/prow.sh +++ b/prow.sh @@ -1008,7 +1008,10 @@ run_e2e () ( # the full Kubernetes E2E testsuite while only running a few tests. move_junit () { if ls "${ARTIFACTS}"/junit_[0-9]*.xml 2>/dev/null >/dev/null; then - run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml + mkdir -p "${ARTIFACTS}/junit/${name}" && + mkdir -p "${ARTIFACTS}/junit/steps" && + run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit/steps/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && + mv "${ARTIFACTS}"/junit_[0-9]*.xml "${ARTIFACTS}/junit/${name}/" fi } trap move_junit EXIT @@ -1085,13 +1088,14 @@ kubectl exec "$pod" -c "${CSI_PROW_SANITY_CONTAINER}" -- /bin/sh -c "\${CHECK_PA EOF chmod u+x "${CSI_PROW_WORK}"/*dir_in_pod.sh + mkdir -p "${ARTIFACTS}/junit/steps" # This cannot run in parallel, because -csi.junitfile output # from different Ginkgo nodes would go to the same file. Also the # staging and target directories are the same. run_with_loggers "${CSI_PROW_WORK}/csi-sanity" \ -ginkgo.v \ - -csi.junitfile "${ARTIFACTS}/junit_sanity.xml" \ + -csi.junitfile "${ARTIFACTS}/junit/steps/junit_sanity.xml" \ -csi.endpoint "dns:///$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' csi-prow-control-plane):$(kubectl get "services/${CSI_PROW_SANITY_SERVICE}" -o "jsonpath={..nodePort}")" \ -csi.stagingdir "/tmp/staging" \ -csi.mountdir "/tmp/mount" \ @@ -1121,7 +1125,8 @@ make_test_to_junit () { # Plain make-test.xml was not delivered as text/xml by the web # server and ignored by spyglass. It seems that the name has to # match junit*.xml. - out="${ARTIFACTS}/junit_make_test.xml" + out="${ARTIFACTS}/junit/steps/junit_make_test.xml" + mkdir -p "$(dirname "$out")" testname= echo "" >>"$out" @@ -1385,8 +1390,8 @@ main () { fi # Merge all junit files into one. This gets rid of duplicated "skipped" tests. - if ls "${ARTIFACTS}"/junit_*.xml 2>/dev/null >&2; then - run_filter_junit -o "${CSI_PROW_WORK}/junit_final.xml" "${ARTIFACTS}"/junit_*.xml && rm "${ARTIFACTS}"/junit_*.xml && mv "${CSI_PROW_WORK}/junit_final.xml" "${ARTIFACTS}" + if ls "${ARTIFACTS}"/junit/steps/junit_*.xml 2>/dev/null >&2; then + run_filter_junit -o "${ARTIFACTS}/junit_final.xml" "${ARTIFACTS}"/junit/steps/junit_*.xml fi return "$ret" From d4277839f43a0ea69ca37501a768dffaeac9a2de Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Sat, 17 Dec 2022 17:45:34 +0100 Subject: [PATCH 079/124] filter-junit.go: preserve system error log Ginkgo v2 uses "system-err" to represent the log output of a test. This only matters for failed tests, but for those it is important to preserve that information. --- filter-junit.go | 1 + 1 file changed, 1 insertion(+) diff --git a/filter-junit.go b/filter-junit.go index 5454092b1..8eb1b8e44 100644 --- a/filter-junit.go +++ b/filter-junit.go @@ -56,6 +56,7 @@ type TestCase struct { Name string `xml:"name,attr"` Time string `xml:"time,attr"` SystemOut string `xml:"system-out,omitempty"` + SystemErr string `xml:"system-err,omitempty"` Failure string `xml:"failure,omitempty"` Skipped SkipReason `xml:"skipped,omitempty"` } From b9b6763bd7237b925c04cd32b2ff0af7602852b2 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Sat, 17 Dec 2022 17:46:35 +0100 Subject: [PATCH 080/124] filter-junit.go: fix loss of testcases when parsing Ginkgo v2 JUnit The "junit" variable is meant to accumulate all testcases from the input files. Overwriting it with the content of an input file when that input file used the new Ginkgo V2 JUnit caused the result of prior input files to get lost. --- filter-junit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter-junit.go b/filter-junit.go index 8eb1b8e44..c1b7b6192 100644 --- a/filter-junit.go +++ b/filter-junit.go @@ -110,7 +110,7 @@ func main() { if err := xml.Unmarshal(data, &junitv2); err != nil { panic(err) } - junit = junitv2.TestSuite + junit.TestCases = append(junit.TestCases, junitv2.TestSuite.TestCases...) } } From dd9867540f663a7c3efa739a231d276e9586687c Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 29 Dec 2022 18:24:04 +0000 Subject: [PATCH 081/124] Add step for checking builds --- SIDECAR_RELEASE_PROCESS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 8977fbe6d..d130383c6 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -92,6 +92,8 @@ naming convention `-on-`. 1. Check that all [canary CI jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) are passing, and that test coverage is adequate for the changes that are going into the release. +1. Check that the post-\-push-images builds are succeeding. + [Example](https://k8s-testgrid.appspot.com/sig-storage-image-build#post-external-snapshotter-push-images) 1. Make sure that no new PRs have merged in the meantime, and no PRs are in flight and soon to be merged. 1. Create a new release following a previous release as a template. Be sure to select the correct From a11216e47a0a636231cc89a87421d2b5a58612e7 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 29 Dec 2022 18:35:54 +0000 Subject: [PATCH 082/124] add new reviewers and remove inactive reviewers --- KUBERNETES_CSI_OWNERS_ALIASES | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES index 344fbe4db..04a07bfb2 100644 --- a/KUBERNETES_CSI_OWNERS_ALIASES +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -22,17 +22,19 @@ aliases: - ggriffiths - gnufied - humblec + - mauriciopoppe - j-griffith - - Jiawei0227 - jingxu97 - jsafrane - pohly - RaunakShah + - sunnylovestiramisu - xing-yang # This documents who previously contributed to Kubernetes-CSI # as approver. emeritus_approvers: +- Jiawei0227 - lpabon - sbezverk - vladimirvivien From 7563d19638df20495ac5225c6cd806a71aa3e5c4 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Thu, 12 Jan 2023 14:51:57 -0500 Subject: [PATCH 083/124] Update CSI_PROW_DRIVER_VERSION to v1.11.0 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 6154bd597..f457c836c 100755 --- a/prow.sh +++ b/prow.sh @@ -196,7 +196,7 @@ kindest/node:v1.18.20@sha256:738cdc23ed4be6cc0b7ea277a2ebcc454c8373d7d8fb991a7fc # If the deployment script is called with CSI_PROW_TEST_DRIVER= as # environment variable, then it must write a suitable test driver configuration # into that file in addition to installing the driver. -configvar CSI_PROW_DRIVER_VERSION "v1.8.0" "CSI driver version" +configvar CSI_PROW_DRIVER_VERSION "v1.11.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" From 901bcb5a90b8cb40c505cffbd1e38c4e2be491e4 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Sat, 18 Feb 2023 08:56:53 +0530 Subject: [PATCH 084/124] Update registry k8s.gcr.io -> registry.k8s.io --- SIDECAR_RELEASE_PROCESS.md | 4 ++-- cloudbuild.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index d130383c6..0e3d516a5 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -102,9 +102,9 @@ naming convention `-on-`. 1. If release was a new major/minor version, create a new `release-` branch at that commit. 1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build). -1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From +1. Promote images from k8s-staging-sig-storage to registry.k8s.io/sig-storage. From the [k8s image - repo](https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage), + repo](https://github.com/kubernetes/k8s.io/tree/HEAD/registry.k8s.io/images/k8s-staging-sig-storage), run `./generate.sh > images.yaml`, and send a PR with the updated images. Once merged, the image promoter will copy the images from staging to prod. 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 4bcffad3a..40e845470 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -13,7 +13,7 @@ # See https://github.com/kubernetes/test-infra/blob/HEAD/config/jobs/image-pushing/README.md # for more details on image pushing process in Kubernetes. # -# To promote release images, see https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage. +# To promote release images, see https://github.com/kubernetes/k8s.io/tree/HEAD/registry.k8s.io/images/k8s-staging-sig-storage. # This must be specified in seconds. If omitted, defaults to 600s (10 mins). # Building three images in external-snapshotter takes more than an hour. From b74a512093d52879979971e62ca16c92740e6d1d Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Tue, 7 Mar 2023 13:48:58 +0000 Subject: [PATCH 085/124] test: fix golint error --- filter-junit.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/filter-junit.go b/filter-junit.go index c1b7b6192..aab1b8b60 100644 --- a/filter-junit.go +++ b/filter-junit.go @@ -24,7 +24,6 @@ package main import ( "encoding/xml" "flag" - "io/ioutil" "os" "regexp" ) @@ -96,7 +95,7 @@ func main() { } } else { var err error - data, err = ioutil.ReadFile(input) + data, err = os.ReadFile(input) if err != nil { panic(err) } @@ -143,7 +142,7 @@ func main() { panic(err) } } else { - if err := ioutil.WriteFile(*output, data, 0644); err != nil { + if err := os.WriteFile(*output, data, 0644); err != nil { panic(err) } } From 1d3f94dd5a13091c5e97e014c5f5f93a40b3aa6d Mon Sep 17 00:00:00 2001 From: Sunny Song Date: Tue, 28 Mar 2023 23:23:20 +0000 Subject: [PATCH 086/124] Update go version to 1.20 to match k/k v1.27 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index f457c836c..2ee353968 100755 --- a/prow.sh +++ b/prow.sh @@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.19" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.20" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below From 155854b099e97ea38e37b516f6d29673b6babd5c Mon Sep 17 00:00:00 2001 From: Xin Yang Date: Wed, 29 Mar 2023 13:15:52 +0000 Subject: [PATCH 087/124] Fix dep version mismatch Signed-off-by: Xin Yang --- prow.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prow.sh b/prow.sh index 2ee353968..6d68fd9f4 100755 --- a/prow.sh +++ b/prow.sh @@ -245,7 +245,7 @@ configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI dr # The version of dep to use for 'make test-vendor'. Ignored if the project doesn't # use dep. Only binary releases of dep are supported (https://github.com/golang/dep/releases). -configvar CSI_PROW_DEP_VERSION v0.5.1 "golang dep version to be used for vendor checking" +configvar CSI_PROW_DEP_VERSION v0.5.4 "golang dep version to be used for vendor checking" # Each job can run one or more of the following tests, identified by # a single word: @@ -469,7 +469,7 @@ install_dep () { if dep version 2>/dev/null | grep -q "version:.*${CSI_PROW_DEP_VERSION}$"; then return fi - run curl --fail --location -o "${CSI_PROW_WORK}/bin/dep" "https://github.com/golang/dep/releases/download/v0.5.4/dep-linux-amd64" && + run curl --fail --location -o "${CSI_PROW_WORK}/bin/dep" "https://github.com/golang/dep/releases/download/${CSI_PROW_DEP_VERSION}/dep-linux-amd64" && chmod u+x "${CSI_PROW_WORK}/bin/dep" } From 0e7ae993df9a62ab0d1b97209bf7061b7bf899fe Mon Sep 17 00:00:00 2001 From: Sunny Song Date: Wed, 5 Apr 2023 18:27:24 +0000 Subject: [PATCH 088/124] Update k8s image repo url --- SIDECAR_RELEASE_PROCESS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index d130383c6..d1cc80b8d 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -104,7 +104,7 @@ naming convention `-on-`. 1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build). 1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From the [k8s image - repo](https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage), + repo](https://github.com/kubernetes/k8s.io/tree/HEAD/registry.k8s.io/images/k8s-staging-sig-storage), run `./generate.sh > images.yaml`, and send a PR with the updated images. Once merged, the image promoter will copy the images from staging to prod. 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar From 26fdfffdd966a9e3e1f337bdbe220aebe7cce2a3 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 25 Apr 2023 03:26:56 +0000 Subject: [PATCH 089/124] Update cloudbuild image --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 4bcffad3a..6b76a9ed1 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -26,7 +26,7 @@ steps: # The image must contain bash and curl. Ideally it should also contain # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20210917-12df099d55' + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20230424-910a2a439d' entrypoint: ./.cloudbuild.sh env: - GIT_TAG=${_GIT_TAG} From 8d519d237c5dc78f0970544aaa25721dac18501a Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 25 Apr 2023 19:02:12 +0000 Subject: [PATCH 090/124] Pin buildkit to v0.10.6 to workaround v0.11 bug with docker manifest --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index ff428890d..bceab34d2 100644 --- a/build.make +++ b/build.make @@ -148,7 +148,7 @@ DOCKER_BUILDX_CREATE_ARGS ?= $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ export DOCKER_CLI_EXPERIMENTAL=enabled; \ - docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ + docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest --driver-opt image=moby/buildkit:v0.10.6; \ trap "docker buildx rm multiarchimage-buildertest" EXIT; \ dockerfile_linux=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile ]; then echo ./$(CMDS_DIR)/$*/Dockerfile; else echo Dockerfile; fi); \ dockerfile_windows=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile.Windows ]; then echo ./$(CMDS_DIR)/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ From f6491af0e1ccb59140fdd2097f3b936e5999d4f1 Mon Sep 17 00:00:00 2001 From: coulof Date: Mon, 15 May 2023 05:52:50 -0400 Subject: [PATCH 091/124] Script to verify EOL sidecar version --- contrib/get_supported_version_csi-sidecar.py | 134 +++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 contrib/get_supported_version_csi-sidecar.py diff --git a/contrib/get_supported_version_csi-sidecar.py b/contrib/get_supported_version_csi-sidecar.py new file mode 100644 index 000000000..bf6668167 --- /dev/null +++ b/contrib/get_supported_version_csi-sidecar.py @@ -0,0 +1,134 @@ +import argparse +import datetime +import re +from collections import defaultdict +import subprocess +import shutil +from dateutil.relativedelta import relativedelta + +# Check that the `gh` command is in the path +def check_gh_command(): + if not shutil.which('gh'): + print("Error: The `gh` command is not available in the PATH.") + print("Please install the GitHub CLI (https://cli.github.com/) and try again.") + exit(1) + +# humanize duration outputs +def duration_ago(dt): + delta = relativedelta(datetime.datetime.now(), dt) + if delta.years > 0: + return f"{delta.years} year{'s' if delta.years > 1 else ''} ago" + elif delta.months > 0: + return f"{delta.months} month{'s' if delta.months > 1 else ''} ago" + elif delta.days > 0: + return f"{delta.days} day{'s' if delta.days > 1 else ''} ago" + elif delta.hours > 0: + return f"{delta.hours} hour{'s' if delta.hours > 1 else ''} ago" + elif delta.minutes > 0: + return f"{delta.minutes} minute{'s' if delta.minutes > 1 else ''} ago" + else: + return "just now" + +def parse_version(version): + # version pattern + pattern = r"v(\d+)\.(\d+)\.(\d+)" + match = re.match(pattern, version) + if match: + major, minor, patch = map(int, match.groups()) + return (major, minor, patch) + +# Calculate the end of life date for a minor release version +# according to : https://kubernetes-csi.github.io/docs/project-policies.html#support +def end_of_life_grouped_versions(versions): + supported_versions = [] + # Prepare dates for later calculation + now = datetime.datetime.now() + one_year = datetime.timedelta(days=365) + three_months = datetime.timedelta(days=90) + + # get the newer versions on top + sorted_versions_list = sorted(versions.items(), key=lambda x: x[0], reverse=True) + # structure example : + # [((3, 5), [('v3.5.0', datetime.datetime(2023, 4, 27, 22, 28, 6))]), + # ((3, 4), + # [('v3.4.1', datetime.datetime(2023, 4, 5, 17, 41, 15)), + # ('v3.4.0', datetime.datetime(2022, 12, 27, 23, 43, 41))])] + latest = sorted_versions_list.pop(0) + + # the latest version is always supported no matter the release date + supported_versions.append(latest[1][-1]) + + for v in sorted_versions_list: + first_release = v[1][-1] + last_release = v[1][0] + # if the release is less than a year old we support the latest patch version + if now - first_release[1] < one_year: + supported_versions.append(last_release) + # if the main release is older than a year and has a recent path, this is supported + elif now - last_release[1] < three_months: + supported_versions.append(last_release) + return supported_versions + +def get_release_docker_image(repo, version): + output = subprocess.check_output(['gh', 'release', '-R', repo, 'view', version], text=True) + match = re.search(r"`docker pull (.*)`", output) + docker_image = match.group(1) + return((version, docker_image)) + +def get_versions_from_releases(repo): + # Run the `gh release` command to get the release list + output = subprocess.check_output(['gh', 'release', '-R', repo, 'list'], text=True) + # Parse the output and group by major and minor version numbers + versions = defaultdict(lambda: []) + for line in output.strip().split('\n'): + parts = line.split('\t') + # pprint.pprint(parts) + version = parts[0] + parsed_version = parse_version(version) + if parsed_version is None: + continue + major, minor, patch = parsed_version + + published = datetime.datetime.strptime(parts[3], '%Y-%m-%dT%H:%M:%SZ') + versions[(major, minor)].append((version, published)) + return(versions) + + +def main(): + # Argument parser + parser = argparse.ArgumentParser(description='Get the currently supported versions for a GitHub repository.') + parser.add_argument( + '--repo', + '-R', required=True, + action='append', dest='repos', + help='''The name of the repository in the format owner/repo. You can specify multiple -R repo to query multiple repositories e.g.:\n + python -R kubernetes-csi/external-attacher -R kubernetes-csi/external-provisioner -R kubernetes-csi/external-resizer -R kubernetes-csi/external-snapshotter -R kubernetes-csi/livenessprobe -R kubernetes-csi/node-driver-registrar -R kubernetes-csi/external-health-monitor''' + ) + parser.add_argument('--display', '-d', action='store_true', help='(default) Display EOL versions with their dates', default=True) + parser.add_argument('--doc', '-D', action='store_true', help='Helper to https://kubernetes-csi.github.io/docs/ that prints Docker image for each EOL version') + + args = parser.parse_args() + + # Verify pre-reqs + check_gh_command() + + # Process all repos + for repo in args.repos: + versions = get_versions_from_releases(repo) + eol_versions = end_of_life_grouped_versions(versions) + + if args.display: + print(f"Supported versions with release date and age of `{repo}`:\n") + for version in eol_versions: + print(f"{version[0]}\t{version[1].strftime('%Y-%m-%d')}\t{duration_ago(version[1])}") + + # TODO : generate proper doc ouput for the tables of: https://kubernetes-csi.github.io/docs/sidecar-containers.html + if args.doc: + print("\nSupported Versions with docker images for each end of life version:\n") + for version in eol_versions: + _, image = get_release_docker_image(args.repo, version[0]) + print(f"{version[0]}\t{image}") + print() + +if __name__ == '__main__': + main() \ No newline at end of file From c39d73c335277773b788720781fca3930b5714ad Mon Sep 17 00:00:00 2001 From: coulof Date: Tue, 16 May 2023 05:11:37 -0400 Subject: [PATCH 092/124] Add comments --- contrib/get_supported_version_csi-sidecar.py | 70 +++++++++++++------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/contrib/get_supported_version_csi-sidecar.py b/contrib/get_supported_version_csi-sidecar.py index bf6668167..747ca9651 100644 --- a/contrib/get_supported_version_csi-sidecar.py +++ b/contrib/get_supported_version_csi-sidecar.py @@ -6,15 +6,20 @@ import shutil from dateutil.relativedelta import relativedelta -# Check that the `gh` command is in the path def check_gh_command(): + """ + Pretty much everything is processed from `gh` + Check that the `gh` command is in the path before anything else + """ if not shutil.which('gh'): print("Error: The `gh` command is not available in the PATH.") print("Please install the GitHub CLI (https://cli.github.com/) and try again.") exit(1) -# humanize duration outputs def duration_ago(dt): + """ + Humanize duration ouputs + """ delta = relativedelta(datetime.datetime.now(), dt) if delta.years > 0: return f"{delta.years} year{'s' if delta.years > 1 else ''} ago" @@ -30,16 +35,29 @@ def duration_ago(dt): return "just now" def parse_version(version): - # version pattern + """ + Parse version assuming it is in the form of v1.2.3 + """ pattern = r"v(\d+)\.(\d+)\.(\d+)" match = re.match(pattern, version) if match: major, minor, patch = map(int, match.groups()) return (major, minor, patch) -# Calculate the end of life date for a minor release version -# according to : https://kubernetes-csi.github.io/docs/project-policies.html#support def end_of_life_grouped_versions(versions): + """ + Calculate the end of life date for a minor release version according to : https://kubernetes-csi.github.io/docs/project-policies.html#support + + The input is an array of tuples of: + * grouped versions (e.g. 1.0, 1.1) + * array of that contains all versions and their release date (e.g. 1.0.0, 01-01-2013) + + versions structure example : + [((3, 5), [('v3.5.0', datetime.datetime(2023, 4, 27, 22, 28, 6))]), + ((3, 4), + [('v3.4.1', datetime.datetime(2023, 4, 5, 17, 41, 15)), + ('v3.4.0', datetime.datetime(2022, 12, 27, 23, 43, 41))])] + """ supported_versions = [] # Prepare dates for later calculation now = datetime.datetime.now() @@ -48,14 +66,9 @@ def end_of_life_grouped_versions(versions): # get the newer versions on top sorted_versions_list = sorted(versions.items(), key=lambda x: x[0], reverse=True) - # structure example : - # [((3, 5), [('v3.5.0', datetime.datetime(2023, 4, 27, 22, 28, 6))]), - # ((3, 4), - # [('v3.4.1', datetime.datetime(2023, 4, 5, 17, 41, 15)), - # ('v3.4.0', datetime.datetime(2022, 12, 27, 23, 43, 41))])] - latest = sorted_versions_list.pop(0) # the latest version is always supported no matter the release date + latest = sorted_versions_list.pop(0) supported_versions.append(latest[1][-1]) for v in sorted_versions_list: @@ -70,12 +83,21 @@ def end_of_life_grouped_versions(versions): return supported_versions def get_release_docker_image(repo, version): + """ + Extract docker image name from the relase page documentation + """ output = subprocess.check_output(['gh', 'release', '-R', repo, 'view', version], text=True) - match = re.search(r"`docker pull (.*)`", output) - docker_image = match.group(1) + #Extract matching image name excluding ` + match = re.search(r"docker pull ([\.\/\-\:\w\d]*)", output) + docker_image = match.group(1) if match else '' return((version, docker_image)) def get_versions_from_releases(repo): + """ + Using `gh` cli get the github releases page details then + create a list of grouped version on major.minor + and for each give all major.minor.patch with release dates + """ # Run the `gh release` command to get the release list output = subprocess.check_output(['gh', 'release', '-R', repo, 'list'], text=True) # Parse the output and group by major and minor version numbers @@ -95,15 +117,15 @@ def get_versions_from_releases(repo): def main(): - # Argument parser - parser = argparse.ArgumentParser(description='Get the currently supported versions for a GitHub repository.') - parser.add_argument( - '--repo', - '-R', required=True, - action='append', dest='repos', - help='''The name of the repository in the format owner/repo. You can specify multiple -R repo to query multiple repositories e.g.:\n - python -R kubernetes-csi/external-attacher -R kubernetes-csi/external-provisioner -R kubernetes-csi/external-resizer -R kubernetes-csi/external-snapshotter -R kubernetes-csi/livenessprobe -R kubernetes-csi/node-driver-registrar -R kubernetes-csi/external-health-monitor''' - ) + manual = """ + This script lists the supported versions Github releases according to https://kubernetes-csi.github.io/docs/project-policies.html#support + It has been designed to help to update the tables from : https://kubernetes-csi.github.io/docs/sidecar-containers.html\n\n + It can take multiple repos as argument, for all CSI sidecars details you can run: + ./get_supported_version_csi-sidecar.py -R kubernetes-csi/external-attacher -R kubernetes-csi/external-provisioner -R kubernetes-csi/external-resizer -R kubernetes-csi/external-snapshotter -R kubernetes-csi/livenessprobe -R kubernetes-csi/node-driver-registrar -R kubernetes-csi/external-health-monitor\n + With the output you can then update the documentation manually. + """ + parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description=manual) + parser.add_argument('--repo', '-R', required=True, action='append', dest='repos', help='The name of the repository in the format owner/repo.') parser.add_argument('--display', '-d', action='store_true', help='(default) Display EOL versions with their dates', default=True) parser.add_argument('--doc', '-D', action='store_true', help='Helper to https://kubernetes-csi.github.io/docs/ that prints Docker image for each EOL version') @@ -126,9 +148,9 @@ def main(): if args.doc: print("\nSupported Versions with docker images for each end of life version:\n") for version in eol_versions: - _, image = get_release_docker_image(args.repo, version[0]) + _, image = get_release_docker_image(repo, version[0]) print(f"{version[0]}\t{image}") print() if __name__ == '__main__': - main() \ No newline at end of file + main() From bd0a10b659e1ba060af90ac0ec384b26296105b1 Mon Sep 17 00:00:00 2001 From: coulof Date: Tue, 16 May 2023 05:27:01 -0400 Subject: [PATCH 093/124] typo --- contrib/get_supported_version_csi-sidecar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/get_supported_version_csi-sidecar.py b/contrib/get_supported_version_csi-sidecar.py index 747ca9651..2ab485454 100644 --- a/contrib/get_supported_version_csi-sidecar.py +++ b/contrib/get_supported_version_csi-sidecar.py @@ -144,7 +144,7 @@ def main(): for version in eol_versions: print(f"{version[0]}\t{version[1].strftime('%Y-%m-%d')}\t{duration_ago(version[1])}") - # TODO : generate proper doc ouput for the tables of: https://kubernetes-csi.github.io/docs/sidecar-containers.html + # TODO : generate proper doc output for the tables of: https://kubernetes-csi.github.io/docs/sidecar-containers.html if args.doc: print("\nSupported Versions with docker images for each end of life version:\n") for version in eol_versions: From b05553510a30c92854e12217eeebea6419283ee6 Mon Sep 17 00:00:00 2001 From: coulof Date: Tue, 16 May 2023 06:11:43 -0400 Subject: [PATCH 094/124] Header --- contrib/get_supported_version_csi-sidecar.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/contrib/get_supported_version_csi-sidecar.py b/contrib/get_supported_version_csi-sidecar.py index 2ab485454..b5380ab92 100644 --- a/contrib/get_supported_version_csi-sidecar.py +++ b/contrib/get_supported_version_csi-sidecar.py @@ -1,3 +1,17 @@ +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import argparse import datetime import re From 72984ec0a6f5505424855d6f3fb3c33cac55dd5e Mon Sep 17 00:00:00 2001 From: mowangdk Date: Sat, 10 Jun 2023 20:41:25 +0800 Subject: [PATCH 095/124] chore: adopt kubernetes recommand label --- prow.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/prow.sh b/prow.sh index 6d68fd9f4..538935683 100755 --- a/prow.sh +++ b/prow.sh @@ -872,10 +872,17 @@ install_snapshot_controller() { cnt=0 expected_running_pods=$(kubectl apply --dry-run=client -o "jsonpath={.spec.replicas}" -f "$SNAPSHOT_CONTROLLER_YAML") expected_namespace=$(kubectl apply --dry-run=client -o "jsonpath={.metadata.namespace}" -f "$SNAPSHOT_CONTROLLER_YAML") - while [ "$(kubectl get pods -n "$expected_namespace" -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do + expect_key='app\.kubernetes\.io/name' + expected_label=$(kubectl apply --dry-run=client -o "jsonpath={.spec.template.metadata.labels['$expect_key']}" -f "$SNAPSHOT_CONTROLLER_YAML") + if [ -z "${expected_label}" ]; then + expect_key='app' + expected_label=$(kubectl apply --dry-run=client -o "jsonpath={.spec.template.metadata.labels['$expect_key']}" -f "$SNAPSHOT_CONTROLLER_YAML") + fi + expect_key=${expect_key//\\/} + while [ "$(kubectl get pods -n "$expected_namespace" -l "$expect_key"="$expected_label" | grep 'Running' -c)" -lt "$expected_running_pods" ]; do if [ $cnt -gt 30 ]; then echo "snapshot-controller pod status:" - kubectl describe pods -n "$expected_namespace" -l app=snapshot-controller + kubectl describe pods -n "$expected_namespace" -l "$expect_key"="$expected_label" echo >&2 "ERROR: snapshot controller not ready after over 5 min" exit 1 fi From 2b8b80eadd9d9db8bc7bf0235d289d2ebf6ddbfd Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 25 Jul 2023 10:22:09 -0700 Subject: [PATCH 096/124] fixing some codespell errors Signed-off-by: Mark Rossetti --- contrib/get_supported_version_csi-sidecar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/get_supported_version_csi-sidecar.py b/contrib/get_supported_version_csi-sidecar.py index b5380ab92..30bbcf24b 100644 --- a/contrib/get_supported_version_csi-sidecar.py +++ b/contrib/get_supported_version_csi-sidecar.py @@ -32,7 +32,7 @@ def check_gh_command(): def duration_ago(dt): """ - Humanize duration ouputs + Humanize duration outputs """ delta = relativedelta(datetime.datetime.now(), dt) if delta.years > 0: @@ -98,7 +98,7 @@ def end_of_life_grouped_versions(versions): def get_release_docker_image(repo, version): """ - Extract docker image name from the relase page documentation + Extract docker image name from the release page documentation """ output = subprocess.check_output(['gh', 'release', '-R', repo, 'view', version], text=True) #Extract matching image name excluding ` From 1f92b7e7cf4136f93bec0e4f4aed8e75a9e3ea90 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Fri, 28 Jul 2023 00:35:28 +0000 Subject: [PATCH 097/124] Add ginkgo timeout to e2e tests to help catch any stuck tests --- prow.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/prow.sh b/prow.sh index 6d68fd9f4..1ae3ec97c 100755 --- a/prow.sh +++ b/prow.sh @@ -101,7 +101,10 @@ configvar CSI_PROW_GINKGO_VERSION v1.7.0 "Ginkgo" # Ginkgo runs the E2E test in parallel. The default is based on the number # of CPUs, but typically this can be set to something higher in the job. -configvar CSI_PROW_GINKO_PARALLEL "-p" "Ginko parallelism parameter(s)" +configvar CSI_PROW_GINKGO_PARALLEL "-p" "Ginkgo parallelism parameter(s)" + +# Timeout value for the overall ginkgo test suite. +configvar CSI_PROW_GINKGO_TIMEOUT "1h" "Ginkgo timeout" # Enables building the code in the repository. On by default, can be # disabled in jobs which only use pre-built components. @@ -1018,10 +1021,10 @@ run_e2e () ( if [ "${name}" == "local" ]; then cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" && - run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e-local.test" -- -report-dir "${ARTIFACTS}" -report-prefix local + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e-local.test" -- -report-dir "${ARTIFACTS}" -report-prefix local else cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" fi ) @@ -1310,7 +1313,7 @@ main () { if tests_enabled "parallel"; then # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 - if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ + if ! run_e2e parallel ${CSI_PROW_GINKGO_PARALLEL} \ -focus="$focus" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then warn "E2E parallel failed" @@ -1320,7 +1323,7 @@ main () { # Run tests that are feature tagged, but non-alpha # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 - if ! run_e2e parallel-features ${CSI_PROW_GINKO_PARALLEL} \ + if ! run_e2e parallel-features ${CSI_PROW_GINKGO_PARALLEL} \ -focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then warn "E2E parallel features failed" @@ -1368,7 +1371,7 @@ main () { if tests_enabled "parallel-alpha"; then # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 - if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \ + if ! run_e2e parallel-alpha ${CSI_PROW_GINKGO_PARALLEL} \ -focus="$focus.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then warn "E2E parallel alpha failed" From 8dd28211b107311f3c113f5dd2a308169e34bda6 Mon Sep 17 00:00:00 2001 From: coulof Date: Wed, 2 Aug 2023 12:32:27 -0400 Subject: [PATCH 098/124] Update cloudbuild image with go 1.20.5 --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 6b76a9ed1..4cd4ce933 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -26,7 +26,7 @@ steps: # The image must contain bash and curl. Ideally it should also contain # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20230424-910a2a439d' + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20230623-56e06d7c18' entrypoint: ./.cloudbuild.sh env: - GIT_TAG=${_GIT_TAG} From cee895e1fa3049f92eeef7941595b3c55e30deff Mon Sep 17 00:00:00 2001 From: Andy Zhang Date: Fri, 11 Aug 2023 22:33:00 +0800 Subject: [PATCH 099/124] remove windows 20H2 build since it's EOL long time ago remove windows 20H2 build since it's EOL more than one year ago --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 6fe58dd7f..7ec8a3f8f 100755 --- a/prow.sh +++ b/prow.sh @@ -78,7 +78,7 @@ version_to_git () { # the list of windows versions was matched from: # - https://hub.docker.com/_/microsoft-windows-nanoserver # - https://hub.docker.com/_/microsoft-windows-servercore -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -ppc64le; linux s390x s390x -s390x; linux arm arm -arm; linux arm64 arm64 -arm64; linux arm arm/v7 -armv7; windows amd64 amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 amd64 .exe nanoserver:20H2 servercore:20H2; windows amd64 amd64 .exe nanoserver:ltsc2022 servercore:ltsc2022" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -ppc64le; linux s390x s390x -s390x; linux arm arm -arm; linux arm64 arm64 -arm64; linux arm arm/v7 -armv7; windows amd64 amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 amd64 .exe nanoserver:ltsc2022 servercore:ltsc2022" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because From 1f7e6059920698c3f3bfbe9ac842ff85cc1fc448 Mon Sep 17 00:00:00 2001 From: siddhikhapare Date: Wed, 30 Aug 2023 00:10:39 +0530 Subject: [PATCH 100/124] fixed broken links of testgrid dashboard --- SIDECAR_RELEASE_PROCESS.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 0e3d516a5..41e8b6a6d 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -17,7 +17,7 @@ The release manager must: Whenever a new Kubernetes minor version is released, our kubernetes-csi CI jobs must be updated. -[Our CI jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) have the +[Our CI jobs](https://testgrid.k8s.io/sig-storage-csi-ci) have the naming convention `-on-`. 1. Jobs should be actively monitored to find and fix failures in sidecars and @@ -90,10 +90,10 @@ naming convention `-on-`. 1. Submit a PR for README changes, in particular, Compatibility, Feature status, and any other sections that may need updating. 1. Check that all [canary CI - jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) are passing, + jobs](https://testgrid.k8s.io/sig-storage-csi-ci) are passing, and that test coverage is adequate for the changes that are going into the release. 1. Check that the post-\-push-images builds are succeeding. - [Example](https://k8s-testgrid.appspot.com/sig-storage-image-build#post-external-snapshotter-push-images) + [Example](https://testgrid.k8s.io/sig-storage-image-build#post-external-snapshotter-push-images) 1. Make sure that no new PRs have merged in the meantime, and no PRs are in flight and soon to be merged. 1. Create a new release following a previous release as a template. Be sure to select the correct @@ -101,7 +101,7 @@ naming convention `-on-`. [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new) 1. If release was a new major/minor version, create a new `release-` branch at that commit. -1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build). +1. Check [image build status](https://testgrid.k8s.io/sig-storage-image-build). 1. Promote images from k8s-staging-sig-storage to registry.k8s.io/sig-storage. From the [k8s image repo](https://github.com/kubernetes/k8s.io/tree/HEAD/registry.k8s.io/images/k8s-staging-sig-storage), @@ -120,7 +120,7 @@ naming convention `-on-`. The following jobs are triggered after tagging to produce the corresponding image(s): -https://k8s-testgrid.appspot.com/sig-storage-image-build +https://testgrid.k8s.io/sig-storage-image-build Clicking on a failed build job opens that job in https://prow.k8s.io. Next to the job title is a rerun icon (circle with arrow). Clicking it opens a popup From b01fd5372f4bc557cd62172876cef86b04d57153 Mon Sep 17 00:00:00 2001 From: mowangdk Date: Fri, 13 Oct 2023 11:00:35 +0800 Subject: [PATCH 101/124] Bump csi-driver-host-path version up to v1.12.0 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 7ec8a3f8f..ff8f9e0bd 100755 --- a/prow.sh +++ b/prow.sh @@ -199,7 +199,7 @@ kindest/node:v1.18.20@sha256:738cdc23ed4be6cc0b7ea277a2ebcc454c8373d7d8fb991a7fc # If the deployment script is called with CSI_PROW_TEST_DRIVER= as # environment variable, then it must write a suitable test driver configuration # into that file in addition to installing the driver. -configvar CSI_PROW_DRIVER_VERSION "v1.11.0" "CSI driver version" +configvar CSI_PROW_DRIVER_VERSION "v1.12.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" From 2c625d41dda606064eeb1c602e307bc399f4b451 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 19 Oct 2023 04:59:29 +0000 Subject: [PATCH 102/124] Add script to generate patch release notes --- SIDECAR_RELEASE_PROCESS.md | 77 ++++++++++------------- generate_patch_release_notes.sh | 106 ++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 43 deletions(-) create mode 100755 generate_patch_release_notes.sh diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 41e8b6a6d..338a62292 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -46,54 +46,45 @@ naming convention `-on-`. ## Release Process 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. -1. Download the latest version of the - [K8s release notes generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes) -1. Create a - [Github personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) - with `repo:public_repo` access -1. Generate release notes for the release. Replace arguments with the relevant - information. - * Clean up old cached information (also needed if you are generating release - notes for multiple repos) - ```bash - rm -rf /tmp/k8s-repo - ``` - * For new minor releases on master: - ```bash - GITHUB_TOKEN= release-notes \ - --discover=mergebase-to-latest \ - --org=kubernetes-csi \ - --repo=external-provisioner \ - --required-author="" \ - --markdown-links \ - --output out.md - ``` - * For new patch releases on a release branch: - ```bash - GITHUB_TOKEN= release-notes \ - --discover=patch-to-latest \ - --branch=release-1.1 \ - --org=kubernetes-csi \ - --repo=external-provisioner \ - --required-author="" \ - --markdown-links \ - --output out.md - ``` -1. Compare the generated output to the new commits for the release to check if - any notable change missed a release note. -1. Reword release notes as needed. Make sure to check notes for breaking - changes and deprecations. -1. If release is a new major/minor version, create a new `CHANGELOG-..md` - file. Otherwise, add the release notes to the top of the existing CHANGELOG - file for that minor version. -1. Submit a PR for the CHANGELOG changes. -1. Submit a PR for README changes, in particular, Compatibility, Feature status, - and any other sections that may need updating. 1. Check that all [canary CI jobs](https://testgrid.k8s.io/sig-storage-csi-ci) are passing, and that test coverage is adequate for the changes that are going into the release. 1. Check that the post-\-push-images builds are succeeding. [Example](https://testgrid.k8s.io/sig-storage-image-build#post-external-snapshotter-push-images) +1. Generate release notes. + 1. Download the latest version of the [K8s release notes generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes) + 1. Create a + [Github personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) + with `repo:public_repo` access + 1. For patch release, use the script generate_patch_release_notes.sh. Read the instructions at the top of the + script. The script also creates PRs for each branch. + 1. For new minor releases, follow these steps and replace arguments with the relevant + information. + * Clean up old cached information (also needed if you are generating release + notes for multiple repos) + ```bash + rm -rf /tmp/k8s-repo + ``` + * For new minor releases on master: + ```bash + GITHUB_TOKEN= release-notes \ + --discover=mergebase-to-latest \ + --org=kubernetes-csi \ + --repo=external-provisioner \ + --required-author="" \ + --markdown-links \ + --output out.md + ``` + 1. Compare the generated output to the new commits for the release to check if + any notable change missed a release note. + 1. Reword release notes as needed, ideally in the original PRs so that the + release notes can be regnerated. Make sure to check notes for breaking + changes and deprecations. + 1. If release is a new major/minor version, create a new `CHANGELOG-..md` + file. + 1. Submit a PR for the CHANGELOG changes. +1. Submit a PR for README changes, in particular, Compatibility, Feature status, + and any other sections that may need updating. 1. Make sure that no new PRs have merged in the meantime, and no PRs are in flight and soon to be merged. 1. Create a new release following a previous release as a template. Be sure to select the correct diff --git a/generate_patch_release_notes.sh b/generate_patch_release_notes.sh new file mode 100755 index 000000000..606c24d22 --- /dev/null +++ b/generate_patch_release_notes.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Usage: generate_patch_release_notes.sh +# +# Generates and creates PRs for kubernetes-csi patch releases. +# +# Required environment variables +# CSI_RELEASE_TOKEN: Github token needed for generating release notes +# GITHUB_USER: Github username to create PRs with +# +# Instructions: +# 1. Login with "gh auth login" +# 2. Copy this script to the kubernetes-csi directory (one directory above the +# repos) +# 3. Update the repos and versions in the $releases array +# 4. Set environment variables +# 5. Run script from the kubernetes-csi directory +# +# Caveats: +# - This script doesn't handle regenerating and updating existing PRs yet. +# It might work if you comment out the PR creation line + +set -e +set -x + +releases=( +# "external-attacher 4.4.1" +# "external-provisioner 3.6.1" +# "external-snapshotter 6.2.3" +) + +function gen_patch_relnotes() { + rm out.md || true + rm -rf /tmp/k8s-repo || true + GITHUB_TOKEN="$CSI_RELEASE_TOKEN" \ + release-notes --discover=patch-to-latest --branch="$2" \ + --org=kubernetes-csi --repo="$1" \ + --required-author="" --markdown-links --output out.md +} + +for rel in "${releases[@]}"; do + read -r repo version <<< "$rel" + + # Parse minor version + minorPattern="(^[[:digit:]]+\.[[:digit:]]+)\." + [[ "$version" =~ $minorPattern ]] + minor="${BASH_REMATCH[1]}" + + echo "$repo" "$version" "$minor" + + pushd "$repo/CHANGELOG" + + git fetch upstream + + # Create branch + branch="changelog-release-$minor" + git checkout master + git branch -D "$branch" || true + git checkout --track "upstream/release-$minor" -b "$branch" + + # Generate release notes + gen_patch_relnotes "$repo" "release-$minor" + cat > tmp.md <> tmp.md + echo >> tmp.md + + file="CHANGELOG-$minor.md" + cat "$file" >> tmp.md + mv tmp.md "$file" + + git add -u + git commit -m "Add changelog for $version" + git push -f origin "$branch" + + # Create PR +prbody=$(cat < Date: Tue, 21 Nov 2023 12:44:44 +0000 Subject: [PATCH 103/124] Update go version to 1.21 to match k/k --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index ff8f9e0bd..48448750d 100755 --- a/prow.sh +++ b/prow.sh @@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.20" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.21" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below From 7b175a1e226544a3da75f52ef757cc364585a61c Mon Sep 17 00:00:00 2001 From: dannawang Date: Tue, 21 Nov 2023 22:35:47 +0000 Subject: [PATCH 104/124] Update csi-test version to v5.2.0 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 48448750d..f712153c7 100755 --- a/prow.sh +++ b/prow.sh @@ -240,7 +240,7 @@ configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v5.0.0 "csi-test version" +configvar CSI_PROW_SANITY_VERSION v5.2.0 "csi-test version" configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" From c4650889dd27ec583fd8ed3d4e1a06e43ef22a3a Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 22 Nov 2023 08:45:42 +0100 Subject: [PATCH 105/124] pull-test.sh: avoid "git subtree pull" error Without a prior `git reset --hard`, `git subtree pull` fails with: fatal: working tree has modifications. Cannot add. Debugging with -d doesn't show any explanation and neither `git diff` nor `git diff-index` show any modifications. The `git reset --hard` shouldn't be necessary. --- pull-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pull-test.sh b/pull-test.sh index b019c1776..5ea051eca 100755 --- a/pull-test.sh +++ b/pull-test.sh @@ -25,6 +25,7 @@ CSI_RELEASE_TOOLS_DIR="$(pwd)" # Update the other repo. cd "$PULL_TEST_REPO_DIR" +git reset --hard # Shouldn't be necessary, but somehow is to avoid "fatal: working tree has modifications. Cannot add." (https://stackoverflow.com/questions/3623351/git-subtree-pull-says-that-the-working-tree-has-modifications-but-git-status-sa) git subtree pull --squash --prefix=release-tools "$CSI_RELEASE_TOOLS_DIR" master git log -n2 From 5deaf667c98f3c300645b0f3d995e13528eced8b Mon Sep 17 00:00:00 2001 From: Sunny Song Date: Mon, 28 Nov 2022 21:06:42 -0800 Subject: [PATCH 106/124] Add wrapper script for sidecar release --- go-modules-update.sh | 126 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100755 go-modules-update.sh diff --git a/go-modules-update.sh b/go-modules-update.sh new file mode 100755 index 000000000..38ed8f910 --- /dev/null +++ b/go-modules-update.sh @@ -0,0 +1,126 @@ +#!/bin/sh -x + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Usage: go-modules-update.sh +# +# Batch update dependencies for sidecars. +# +# Required environment variables +# CSI_RELEASE_TOKEN: Github token needed for generating release notes +# GITHUB_USER: Github username to create PRs with +# +# Instructions: +# 1. Login with "gh auth login" +# 2. Copy this script to the kubernetes-csi directory (one directory above the +# repos) +# 3. Update the repos and branches +# 4. Set environment variables +# 5. Run script from the kubernetes-csi directory +# +# Caveats: +# - This script doesn't handle interface incompatibility of updates. + + +die () { + echo >&2 "$@" + exit 1 +} + +MAX_RETRY=10 + +# Get the options +while getopts ":u:v:" option; do + case $option in + u) # Set username + username=$OPTARG;; + v) # Set version + v=$OPTARG;; + \?) # Invalid option + echo "Error: Invalid option: $OPTARG" + exit;; + esac +done + +# Only need to do this once +gh auth login || die "gh auth login failed" + +while read -r repo branches; do + if [ "$repo" != "#" ]; then + ( + cd "$repo" || die "$repo: does not exit" + git fetch origin || die "$repo: git fetch" + for i in $branches; do + if [ "$(git rev-parse --verify "module-update-$i" 2>/dev/null)" ]; then + git checkout master && git branch -d "module-update-$i" + fi + git checkout -B "module-update-$i" "origin/$i" || die "$repo:$i checkout" + rm -rf .git/MERGE* + if ! git subtree pull --squash --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master; then + # Sometimes "--squash" leads to merge conflicts. Because we know that "release-tools" + # is an unmodified copy of csi-release-tools, we can automatically resolve that + # by replacing it completely. + if [ -e .git/MERGE_MSG ] && [ -e .git/FETCH_HEAD ] && grep -q "^# Conflict" .git/MERGE_MSG; then + rm -rf release-tools + mkdir release-tools + git archive FETCH_HEAD | tar -C release-tools -xf - || die "failed to re-create release-tools from FETCH_HEAD" + git add release-tools || die "add release-tools" + git commit --file=.git/MERGE_MSG || die "commit squashed release-tools" + else + die "git subtree pull --squash failed, cannot reover." + fi + fi + RETRY=0 + while ! ./release-tools/go-get-kubernetes.sh -p "$v" && RETRY < $MAX_RETRY + do + RETRY=$((RETRY+1)) + go mod tidy && go mod vendor && go mod tidy + done + go mod tidy && go mod vendor && go mod tidy || die "last go mod vendor && go mod tidy failed" + git add --all || die "git add -all failed" + git commit -m "Update dependency go modules for k8s v$v" || die "commit update modules" + git remote set-url origin "https://github.com/$username/$repo.git" || die "git remote set-url failed" + make test || die "$repo:$i make test" + git push origin "module-update-$i" --force || die "origin:module-update-$i push failed - probably there is already an unmerged branch and pending PR" + # Create PR +prbody=$(cat < Date: Tue, 12 Dec 2023 18:28:37 +0000 Subject: [PATCH 107/124] Use set -x instead of die --- SIDECAR_RELEASE_PROCESS.md | 3 +++ go-modules-update.sh | 45 ++++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 338a62292..4f5d033c2 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -46,6 +46,9 @@ naming convention `-on-`. ## Release Process 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. +1. Update dependencies for sidecars via + [go-modules-update.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/HEAD/release-tools/go-modules-update.sh), + and get PRs approved and merged. 1. Check that all [canary CI jobs](https://testgrid.k8s.io/sig-storage-csi-ci) are passing, and that test coverage is adequate for the changes that are going into the release. diff --git a/go-modules-update.sh b/go-modules-update.sh index 38ed8f910..bd605e970 100755 --- a/go-modules-update.sh +++ b/go-modules-update.sh @@ -1,4 +1,4 @@ -#!/bin/sh -x +#!/bin/sh # Copyright 2023 The Kubernetes Authors. # @@ -27,18 +27,21 @@ # 1. Login with "gh auth login" # 2. Copy this script to the kubernetes-csi directory (one directory above the # repos) -# 3. Update the repos and branches +# 3. Update the repos and master branch so locally it has the latest upstream +# change # 4. Set environment variables # 5. Run script from the kubernetes-csi directory # # Caveats: # - This script doesn't handle interface incompatibility of updates. +# You need to resolve interface incompatibility case by case. The +# most frequent case is to update the interface(new parameters, +# name change of the method, etc.)in the sidecar repo and make sure +# the build and test pass. -die () { - echo >&2 "$@" - exit 1 -} +set -e +set -x MAX_RETRY=10 @@ -56,18 +59,18 @@ while getopts ":u:v:" option; do done # Only need to do this once -gh auth login || die "gh auth login failed" +gh auth login while read -r repo branches; do if [ "$repo" != "#" ]; then ( - cd "$repo" || die "$repo: does not exit" - git fetch origin || die "$repo: git fetch" + cd "$repo" + git fetch origin for i in $branches; do if [ "$(git rev-parse --verify "module-update-$i" 2>/dev/null)" ]; then git checkout master && git branch -d "module-update-$i" fi - git checkout -B "module-update-$i" "origin/$i" || die "$repo:$i checkout" + git checkout -B "module-update-$i" "origin/$i" rm -rf .git/MERGE* if ! git subtree pull --squash --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master; then # Sometimes "--squash" leads to merge conflicts. Because we know that "release-tools" @@ -76,11 +79,11 @@ while read -r repo branches; do if [ -e .git/MERGE_MSG ] && [ -e .git/FETCH_HEAD ] && grep -q "^# Conflict" .git/MERGE_MSG; then rm -rf release-tools mkdir release-tools - git archive FETCH_HEAD | tar -C release-tools -xf - || die "failed to re-create release-tools from FETCH_HEAD" - git add release-tools || die "add release-tools" - git commit --file=.git/MERGE_MSG || die "commit squashed release-tools" + git archive FETCH_HEAD | tar -C release-tools -xf - + git add release-tools + git commit --file=.git/MERGE_MSG else - die "git subtree pull --squash failed, cannot reover." + exit 1 fi fi RETRY=0 @@ -89,12 +92,12 @@ while read -r repo branches; do RETRY=$((RETRY+1)) go mod tidy && go mod vendor && go mod tidy done - go mod tidy && go mod vendor && go mod tidy || die "last go mod vendor && go mod tidy failed" - git add --all || die "git add -all failed" - git commit -m "Update dependency go modules for k8s v$v" || die "commit update modules" - git remote set-url origin "https://github.com/$username/$repo.git" || die "git remote set-url failed" - make test || die "$repo:$i make test" - git push origin "module-update-$i" --force || die "origin:module-update-$i push failed - probably there is already an unmerged branch and pending PR" + go mod tidy && go mod vendor && go mod tidy + git add --all + git commit -m "Update dependency go modules for k8s v$v" + git remote set-url origin "https://github.com/$username/$repo.git" + make test + git push origin "module-update-$i" --force # Create PR prbody=$(cat < Date: Thu, 14 Dec 2023 10:05:05 +0800 Subject: [PATCH 108/124] nominate self (carlory) as kubernetes-csi reviewer --- KUBERNETES_CSI_OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES index 04a07bfb2..6e46cf290 100644 --- a/KUBERNETES_CSI_OWNERS_ALIASES +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -18,6 +18,7 @@ aliases: # when they are temporarily unable to review PRs. kubernetes-csi-reviewers: - andyzhangx + - carlory - chrishenzie - ggriffiths - gnufied From 5436c81e9e9b37ebe3bc97386b723a47934720f1 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Fri, 15 Dec 2023 12:59:37 -0500 Subject: [PATCH 109/124] Change go version to 1.21.5 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index f712153c7..a27c71f3d 100755 --- a/prow.sh +++ b/prow.sh @@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.21" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.21.5" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below From 463a0e9f57fa7edd20e0ecfedf5a335116221b7b Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Fri, 22 Dec 2023 00:12:14 +0000 Subject: [PATCH 110/124] Add script to update specific go modules --- go-modules-targeted-update.sh | 96 +++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100755 go-modules-targeted-update.sh diff --git a/go-modules-targeted-update.sh b/go-modules-targeted-update.sh new file mode 100755 index 000000000..52146f31d --- /dev/null +++ b/go-modules-targeted-update.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Usage: go-modules-targeted-update.sh +# +# Batch update specific dependencies for sidecars. +# +# Required environment variables +# CSI_RELEASE_TOKEN: Github token needed for generating release notes +# GITHUB_USER: Github username to create PRs with +# +# Instructions: +# 1. Login with "gh auth login" +# 2. Copy this script to the Github org directory (one directory above the +# repos) +# 3. Change $modules, $releases and $org if needed. +# 4. Set environment variables +# 5. Run script from the Github org directory +# +# Caveats: +# - This script doesn't handle interface incompatibility of updates. +# You need to resolve interface incompatibility case by case. The +# most frequent case is to update the interface(new parameters, +# name change of the method, etc.)in the sidecar repo and make sure +# the build and test pass. + + +set -e +set -x + +org="kubernetes-csi" + +modules=( +"github.com/kubernetes-csi/csi-lib-utils@v0.15.1" +) + +releases=( +#"external-attacher release-4.4" +#"external-provisioner release-3.6" +#"external-resizer release-1.9" +#"external-snapshotter release-6.3" +#"node-driver-registrar release-2.9" +) + +for rel in "${releases[@]}"; do + + read -r repo branch <<< "$rel" + if [ "$repo" != "#" ]; then + ( + cd "$repo" + git fetch upstream + + if [ "$(git rev-parse --verify "module-update-$branch" 2>/dev/null)" ]; then + git checkout master && git branch -D "module-update-$branch" + fi + git checkout -B "module-update-$branch" "upstream/$branch" + + for mod in "${modules[@]}"; do + go get "$mod" + done + go mod tidy + go mod vendor + + git add --all + git commit -m "Update go modules" + git push origin "module-update-$branch" --force + + # Create PR +prbody=$(cat < Date: Fri, 22 Dec 2023 01:05:07 +0000 Subject: [PATCH 111/124] Fix release notes script to use fixed tags --- ...ase_notes.sh => generate-patch-release-notes.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) rename generate_patch_release_notes.sh => generate-patch-release-notes.sh (87%) diff --git a/generate_patch_release_notes.sh b/generate-patch-release-notes.sh similarity index 87% rename from generate_patch_release_notes.sh rename to generate-patch-release-notes.sh index 606c24d22..2b9c13c38 100755 --- a/generate_patch_release_notes.sh +++ b/generate-patch-release-notes.sh @@ -48,7 +48,7 @@ function gen_patch_relnotes() { rm out.md || true rm -rf /tmp/k8s-repo || true GITHUB_TOKEN="$CSI_RELEASE_TOKEN" \ - release-notes --discover=patch-to-latest --branch="$2" \ + release-notes --start-rev="$3" --end-rev="$2" --branch="$2" \ --org=kubernetes-csi --repo="$1" \ --required-author="" --markdown-links --output out.md } @@ -57,11 +57,14 @@ for rel in "${releases[@]}"; do read -r repo version <<< "$rel" # Parse minor version - minorPattern="(^[[:digit:]]+\.[[:digit:]]+)\." - [[ "$version" =~ $minorPattern ]] + minorPatchPattern="(^[[:digit:]]+\.[[:digit:]]+)\.([[:digit:]]+)" + [[ "$version" =~ $minorPatchPattern ]] minor="${BASH_REMATCH[1]}" + patch="${BASH_REMATCH[2]}" - echo "$repo" "$version" "$minor" + echo "$repo $version $minor $patch" + prevPatch="$((patch-1))" + prevVer="v$minor.$prevPatch" pushd "$repo/CHANGELOG" @@ -74,7 +77,7 @@ for rel in "${releases[@]}"; do git checkout --track "upstream/release-$minor" -b "$branch" # Generate release notes - gen_patch_relnotes "$repo" "release-$minor" + gen_patch_relnotes "$repo" "release-$minor" "$prevVer" cat > tmp.md <> tmp.md echo >> tmp.md + rm out.md file="CHANGELOG-$minor.md" cat "$file" >> tmp.md From 9b4352e9f1105031b71c76f23c62e587c908c3f5 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 27 Dec 2023 00:11:23 +0000 Subject: [PATCH 112/124] Update release playbook --- SIDECAR_RELEASE_PROCESS.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 4f5d033c2..647d2342b 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -46,9 +46,12 @@ naming convention `-on-`. ## Release Process 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. -1. Update dependencies for sidecars via - [go-modules-update.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/HEAD/release-tools/go-modules-update.sh), - and get PRs approved and merged. +1. Update dependencies for sidecars + 1. For new minor versions, use + [go-modules-update.sh](https://github.com/kubernetes-csi/csi-release-tools/blob/HEAD/go-modules-update.sh), + 1. For CVE fixes on patch versions, use + [go-modules-targeted-update.sh](https://github.com/kubernetes-csi/csi-release-tools/blob/HEAD/go-modules-targeted-update.sh), + Read the instructions at the top of the script. 1. Check that all [canary CI jobs](https://testgrid.k8s.io/sig-storage-csi-ci) are passing, and that test coverage is adequate for the changes that are going into the release. From e681b170eb67141dad3a1430ab7f23641f15fb6f Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Wed, 21 Feb 2024 18:16:04 +0100 Subject: [PATCH 113/124] Use .go-version to get Kubernetes go version Kubernetes publishes .go-version since v1.23. Use it to get Kubernete's go version instead of parsing a shell script. --- prow.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index a27c71f3d..0af3fdcd8 100755 --- a/prow.sh +++ b/prow.sh @@ -564,7 +564,15 @@ go_version_for_kubernetes () ( local version="$2" local go_version - # We use the minimal Go version specified for each K8S release (= minimum_go_version in hack/lib/golang.sh). + # Try to get the version for .go-version + go_version="$( cat "$path/.go-version" )" + if [ "$go_version" ]; then + echo "$go_version" + return + fi + + # Fall back to hack/lib/golang.sh parsing. + # This is necessary in v1.26.0 and older Kubernetes releases that do not have .go-version. # More recent versions might also work, but we don't want to count on that. go_version="$(grep minimum_go_version= "$path/hack/lib/golang.sh" | sed -e 's/.*=go//')" if ! [ "$go_version" ]; then From b52e7ad35eda556c8038ad9432ea69a506a29c6b Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Tue, 23 Apr 2024 10:06:13 +0200 Subject: [PATCH 114/124] Update go to 1.22.2 Update go version to be in sync with Kubernetes 1.30, which uses 1.22.2. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 0af3fdcd8..0fc42a5df 100755 --- a/prow.sh +++ b/prow.sh @@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.21.5" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.22.2" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below From 043fd09919bb5a467971d578a43087a51f4d1bcb Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 10 May 2024 15:49:04 +0200 Subject: [PATCH 115/124] Add test-logcheck target The new test-logcheck + verify-logcheck.sh script checks for correct klog usage. It's optional and must be explicitly requested in the main Makefile by adding: ``` test: test-logcheck ``` --- build.make | 7 +++++++ verify-logcheck.sh | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100755 verify-logcheck.sh diff --git a/build.make b/build.make index bceab34d2..fe120c52b 100644 --- a/build.make +++ b/build.make @@ -322,3 +322,10 @@ test-spelling: test-boilerplate: @ echo; echo "### $@:" @ ./release-tools/verify-boilerplate.sh "$(pwd)" + +# Test klog usage. This test is optional and must be explicitly added to `test` target in the main Makefile: +# test: test-logcheck +.PHONY: test-logcheck +test-logcheck: + @ echo; echo "### $@:" + @ ./release-tools/verify-logcheck.sh diff --git a/verify-logcheck.sh b/verify-logcheck.sh new file mode 100755 index 000000000..ebd327754 --- /dev/null +++ b/verify-logcheck.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script uses the logcheck tool to analyze the source code +# for proper usage of klog contextual logging. + +set -o errexit +set -o nounset +set -o pipefail + +LOGCHECK_VERSION=${1:-0.8.2} + +# This will canonicalize the path +CSI_LIB_UTIL_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P) + +# Create a temporary directory for installing logcheck and +# set up a trap command to remove it when the script exits. +CSI_LIB_UTIL_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t csi-lib-utils.XXXXXX) +trap 'rm -rf "${CSI_LIB_UTIL_TEMP}"' EXIT + +echo "Installing logcheck to temp dir: sigs.k8s.io/logtools/logcheck@v${LOGCHECK_VERSION}" +GOBIN="${CSI_LIB_UTIL_TEMP}" go install "sigs.k8s.io/logtools/logcheck@v${LOGCHECK_VERSION}" +echo "Verifing logcheck: ${CSI_LIB_UTIL_TEMP}/logcheck -check-contextual ${CSI_LIB_UTIL_ROOT}/..." +"${CSI_LIB_UTIL_TEMP}/logcheck" -check-contextual -check-with-helpers "${CSI_LIB_UTIL_ROOT}/..." From 0a78505613d99e8bb4b642ce85c90548fd0812bc Mon Sep 17 00:00:00 2001 From: bells17 Date: Sat, 11 May 2024 00:53:50 +0900 Subject: [PATCH 116/124] Bump to Go 1.22.3 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 0fc42a5df..049fb79af 100755 --- a/prow.sh +++ b/prow.sh @@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.22.2" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.22.3" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below From c31745621111694c0f2a7ffc742c847767d158b1 Mon Sep 17 00:00:00 2001 From: bells17 Date: Sat, 11 May 2024 01:47:15 +0900 Subject: [PATCH 117/124] Fix typo --- verify-logcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verify-logcheck.sh b/verify-logcheck.sh index ebd327754..f3287e7b0 100755 --- a/verify-logcheck.sh +++ b/verify-logcheck.sh @@ -33,5 +33,5 @@ trap 'rm -rf "${CSI_LIB_UTIL_TEMP}"' EXIT echo "Installing logcheck to temp dir: sigs.k8s.io/logtools/logcheck@v${LOGCHECK_VERSION}" GOBIN="${CSI_LIB_UTIL_TEMP}" go install "sigs.k8s.io/logtools/logcheck@v${LOGCHECK_VERSION}" -echo "Verifing logcheck: ${CSI_LIB_UTIL_TEMP}/logcheck -check-contextual ${CSI_LIB_UTIL_ROOT}/..." +echo "Verifying logcheck: ${CSI_LIB_UTIL_TEMP}/logcheck -check-contextual ${CSI_LIB_UTIL_ROOT}/..." "${CSI_LIB_UTIL_TEMP}/logcheck" -check-contextual -check-with-helpers "${CSI_LIB_UTIL_ROOT}/..." From f5aebfc9f391bd976ce0c3ed28b7d2a91ff602c6 Mon Sep 17 00:00:00 2001 From: bells17 Date: Sat, 11 May 2024 03:27:04 +0900 Subject: [PATCH 118/124] Add GitHub Actions workflows --- .github/dependabot.yaml | 12 ++++++++++++ .github/workflows/codespell.yml | 15 +++++++++++++++ .github/workflows/trivy.yaml | 29 +++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 .github/dependabot.yaml create mode 100644 .github/workflows/codespell.yml create mode 100644 .github/workflows/trivy.yaml diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 000000000..814a34493 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,12 @@ +version: 2 +enable-beta-ecosystems: true +updates: +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + labels: + - "area/dependency" + - "release-note-none" + - "ok-to-test" + open-pull-requests-limit: 10 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 000000000..54d56916a --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,15 @@ +# GitHub Action to automate the identification of common misspellings in text files. +# https://github.com/codespell-project/actions-codespell +# https://github.com/codespell-project/codespell +name: codespell +on: [push, pull_request] +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: codespell-project/actions-codespell@master + with: + check_filenames: true + skip: ./.git,./.github/workflows/codespell.yml,.git,*.png,*.jpg,*.svg,*.sum,./vendor,go.sum,./release-tools/prow.sh,./client/vendor diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 000000000..472984780 --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,29 @@ +name: Run Trivy scanner for Go version vulnerabilities +on: + push: + branches: + - master + pull_request: +jobs: + trivy: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get Go version + id: go-version + run: | + GO_VERSION=$(cat prow.sh | grep "configvar CSI_PROW_GO_VERSION_BUILD" | awk '{print $3}' | sed 's/"//g') + echo "version=$GO_VERSION" >> $GITHUB_OUTPUT + + - name: Run Trivy scanner for Go version vulnerabilities + uses: aquasecurity/trivy-action@master + with: + image-ref: 'golang:${{ steps.go-version.outputs.version }}' + format: 'table' + exit-code: '1' + ignore-unfixed: true + vuln-type: 'library' + severity: 'CRITICAL,HIGH,MEDIUM,LOW,UNKNOWN' From d9bd160c2a756b1963c333a93953dfb47f7bae72 Mon Sep 17 00:00:00 2001 From: bells17 Date: Mon, 13 May 2024 18:59:32 +0900 Subject: [PATCH 119/124] Update skip list in codespell GitHub Action --- .github/workflows/codespell.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 54d56916a..e74edcef5 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -12,4 +12,4 @@ jobs: - uses: codespell-project/actions-codespell@master with: check_filenames: true - skip: ./.git,./.github/workflows/codespell.yml,.git,*.png,*.jpg,*.svg,*.sum,./vendor,go.sum,./release-tools/prow.sh,./client/vendor + skip: "*.png,*.jpg,*.svg,*.sum,./.git,./.github/workflows/codespell.yml,./prow.sh" From a5667bbbbf9297bf24afc36ded1bd2a9a8eeb56e Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Fri, 24 May 2024 11:39:02 +0530 Subject: [PATCH 120/124] fix typo in sidecar release process Signed-off-by: Humble Chirammal --- SIDECAR_RELEASE_PROCESS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 647d2342b..aab8d6e21 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -84,7 +84,7 @@ naming convention `-on-`. 1. Compare the generated output to the new commits for the release to check if any notable change missed a release note. 1. Reword release notes as needed, ideally in the original PRs so that the - release notes can be regnerated. Make sure to check notes for breaking + release notes can be regenerated. Make sure to check notes for breaking changes and deprecations. 1. If release is a new major/minor version, create a new `CHANGELOG-..md` file. From cfa92106cd9eac6b51e9257b71296a84cc03dbaf Mon Sep 17 00:00:00 2001 From: Daniel Fajmon Date: Tue, 4 Jun 2024 09:13:07 +0200 Subject: [PATCH 121/124] Instruction update missing release-notes tool as requirement --- generate-patch-release-notes.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/generate-patch-release-notes.sh b/generate-patch-release-notes.sh index 2b9c13c38..536a14902 100755 --- a/generate-patch-release-notes.sh +++ b/generate-patch-release-notes.sh @@ -23,13 +23,17 @@ # CSI_RELEASE_TOKEN: Github token needed for generating release notes # GITHUB_USER: Github username to create PRs with # +# Required tools: +# - gh +# - release-notes (https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md) +# # Instructions: -# 1. Login with "gh auth login" -# 2. Copy this script to the kubernetes-csi directory (one directory above the -# repos) -# 3. Update the repos and versions in the $releases array -# 4. Set environment variables -# 5. Run script from the kubernetes-csi directory +# 1. Install the required tools +# 2. Login with "gh auth login" +# 3. Copy this script to the kubernetes-csi directory (one directory above the repos) +# 4. Update the repos and versions in the $releases array +# 5. Set environment variables +# 6. Run script from the kubernetes-csi directory # # Caveats: # - This script doesn't handle regenerating and updating existing PRs yet. From 69bd71e8a0b0ae0731bbedfe4cdba829f14b8cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20M=C3=B6ller?= Date: Wed, 26 Jun 2024 11:14:24 +0200 Subject: [PATCH 122/124] chore: add CSI_PROW_SIDECAR_E2E_PATH adds CSI_PROW_SIDECAR_E2E_PATH, an alternative to CSI_PROW_SIDECAR_E2E_IMPORT_PATH that can be used if the go module referenced in the import path has a /v{x} suffix, making it differing from the directory of the module. --- prow.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/prow.sh b/prow.sh index 049fb79af..801b6f2f7 100755 --- a/prow.sh +++ b/prow.sh @@ -231,8 +231,11 @@ configvar CSI_PROW_E2E_VERSION "$(version_to_git "${CSI_PROW_KUBERNETES_VERSION} configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo" configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" -# Local path for e2e tests. Set to "none" to disable. -configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package" +# Local path & package path for e2e tests. Set to "none" to disable. +# When using versioned go modules, the import path is the module path whereas the path +# should not contain the version and be the directory where the module is checked out. +configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package (go import path)" +configvar CSI_PROW_SIDECAR_E2E_PATH "${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" "CSI Sidecar E2E path (directory)" # csi-sanity testing from the csi-test repo can be run against the installed # CSI driver. For this to work, deploying the driver must expose the Unix domain @@ -1035,7 +1038,7 @@ run_e2e () ( trap move_junit EXIT if [ "${name}" == "local" ]; then - cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" && + cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_PATH}" && run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e-local.test" -- -report-dir "${ARTIFACTS}" -report-prefix local else cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && From 028f8c6987ae8635556674fb99286f0ad78a9333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20M=C3=B6ller?= Date: Fri, 12 Jul 2024 08:21:11 +0200 Subject: [PATCH 123/124] chore: bump to Go 1.22.5 adds CSI_PROW_SIDECAR_E2E_PATH, an alternative to CSI_PROW_SIDECAR_E2E_IMPORT_PATH that can be used if the go module referenced in the import path has a /v{x} suffix, making it differing from the directory of the module. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 801b6f2f7..df8c61502 100755 --- a/prow.sh +++ b/prow.sh @@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.22.3" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.22.5" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below From e1ceee2877f15ceb2bbdd2027c3aca43c64e0c7f Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Mon, 19 Aug 2024 13:49:31 -0400 Subject: [PATCH 124/124] Always enable race detection while running tests --- build.make | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.make b/build.make index fe120c52b..5d3079308 100644 --- a/build.make +++ b/build.make @@ -62,9 +62,9 @@ IMAGE_NAME=$(REGISTRY_NAME)/$* ifdef V # Adding "-alsologtostderr" assumes that all test binaries contain glog. This is not guaranteed. -TESTARGS = -v -args -alsologtostderr -v 5 +TESTARGS = -race -v -args -alsologtostderr -v 5 else -TESTARGS = +TESTARGS = -race endif # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables