From eb1bf794e9ec3e250ff13b6d823e8ce626bd1d80 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 5 May 2021 14:11:03 +0200 Subject: [PATCH] Squashed 'release-tools/' changes from a1e11275..6616a6b5 https://github.com/kubernetes-csi/csi-release-tools/commit/6616a6b5 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/146 from pohly/kubernetes-1.21 https://github.com/kubernetes-csi/csi-release-tools/commit/510fb0f9 prow.sh: support Kubernetes 1.21 https://github.com/kubernetes-csi/csi-release-tools/commit/c63c61b3 prow.sh: add CSI_PROW_DEPLOYMENT_SUFFIX https://github.com/kubernetes-csi/csi-release-tools/commit/51ac11c3 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/144 from pohly/pull-jobs https://github.com/kubernetes-csi/csi-release-tools/commit/dd54c926 pull-test.sh: test importing csi-release-tools into other repo https://github.com/kubernetes-csi/csi-release-tools/commit/7d2643a5 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/143 from pohly/path-setup https://github.com/kubernetes-csi/csi-release-tools/commit/6880b0c8 prow.sh: avoid creating paths unless really running tests https://github.com/kubernetes-csi/csi-release-tools/commit/bc0504ad Merge https://github.com/kubernetes-csi/csi-release-tools/pull/140 from jsafrane/remove-unused-k8s-libs https://github.com/kubernetes-csi/csi-release-tools/commit/5b1de1ad go-get-kubernetes.sh: remove unused k8s libs https://github.com/kubernetes-csi/csi-release-tools/commit/49b42693 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/120 from pohly/add-kubernetes-release https://github.com/kubernetes-csi/csi-release-tools/commit/f7e7ee49 docs: steps for adding testing against new Kubernetes release git-subtree-dir: release-tools git-subtree-split: 6616a6b5294b6df39cfce37f4fce7cdce0a77583 --- SIDECAR_RELEASE_PROCESS.md | 44 ++++++++++++++++++++++++++++++++++ go-get-kubernetes.sh | 6 +++++ prow.sh | 49 ++++++++++++++++++++++++-------------- pull-test.sh | 32 +++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 18 deletions(-) create mode 100755 pull-test.sh diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 3aee7a81b..f5ec71b9c 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -104,3 +104,47 @@ naming convention `-on-`. CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) and [k/k in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) + +## Adding support for a new Kubernetes release + +1. Add the new release to `k8s_versions` in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L25 + to enable generating a job for it. Set `experimental_k8s_version` + in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L40 + to ensure that the new jobs aren't run for PRs unless explicitly + requested. Generate and submit the new jobs. +1. Create a test PR to try out the new job in some repo with `/test + pull-kubernetes-csi---on-kubernetes-` where x.y + matches the Kubernetes release. Alternatively, run .prow.sh in that + repo locally with `CSI_PROW_KUBERNETES_VERSION=x.y.z`. +1. Optional: update to a [new + release](https://github.com/kubernetes-sigs/kind/tags) of kind with + pre-built images for the new Kubernetes release. This is optional + if the current version of kind is able to build images for the new + Kubernetes release. However, jobs require less resources when they + don't need to build those images from the Kubernetes source code. + This change needs to be tried out in a PR against a component + first, then get submitted against csi-release-tools. +1. Optional: propagate the updated csi-release-tools to all components + with the script from + https://github.com/kubernetes-csi/csi-release-tools/issues/7#issuecomment-707025402 +1. Once it is likely to work in all components, unset + `experimental_k8s_version` and submit the updated jobs. +1. Once all sidecars for the new Kubernetes release are released, + either bump the version number of the images in the existing + [csi-driver-host-path + deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) + and/or create a new deployment, depending on what Kubernetes + release an updated sidecar is compatible with. If no new deployment + is needed, then add a symlink to document that there intentionally + isn't a separate deployment. This symlink is not needed for Prow + testing because that will use "kubernetes-latest" as fallback. + Update that link when creating a new deployment. +1. Create a new csi-driver-host-path release. +1. Bump `CSI_PROW_DRIVER_VERSION` in prow.sh to that new release and + (eventually) roll that change out to all repos by updating + `release-tools` in them. This is used when testing manually. The + Prow jobs override that value, so also update + `hostpath_driver_version` in + https://github.com/kubernetes/test-infra/blob/91b04e6af3a40a9bcff25aa030850a4721e2dd2b/config/jobs/kubernetes-csi/gen-jobs.sh#L46-L47 diff --git a/go-get-kubernetes.sh b/go-get-kubernetes.sh index 8c4e30244..cbbbb7c3c 100755 --- a/go-get-kubernetes.sh +++ b/go-get-kubernetes.sh @@ -55,6 +55,12 @@ mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubuserconten sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' ) || die "failed to determine Kubernetes staging modules" for mod in $mods; do + if ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then + echo "Kubernetes module $mod is not used, skipping" + # Remove the module from go.mod "replace" that was added by an older version of this script. + (set -x; env GO111MODULE=on go mod edit "-dropreplace=$mod") || die "'go mod edit' failed" + continue + fi # The presence of a potentially incomplete go.mod file affects this command, # so move elsewhere. modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") || diff --git a/prow.sh b/prow.sh index b09c29200..fe23cf516 100755 --- a/prow.sh +++ b/prow.sh @@ -136,6 +136,9 @@ kind_version_default () { case "${CSI_PROW_KUBERNETES_VERSION}" in latest|master) echo main;; + 1.21*|release-1.21) + # TODO: replace this special case once the next KinD release supports 1.21. + echo main;; *) echo v0.10.0;; esac @@ -159,11 +162,6 @@ kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f # Use kind node-image --type=bazel by default, but allow to disable that. configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" -# Work directory. It has to allow running executables, therefore /tmp -# is avoided. Cleaning up after the script is intentionally left to -# the caller. -configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" - # By default, this script tests sidecars with the CSI hostpath driver, # using the install_csi_driver function. That function depends on # a deployment script that it searches for in several places: @@ -190,8 +188,8 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # CSI_PROW_DEPLOYMENT variable can be set in the # .prow.sh of each component when there are breaking changes # that require using a non-default deployment. The default -# is a deployment named "kubernetes-x.yy" (if available), -# otherwise "kubernetes-latest". +# is a deployment named "kubernetes-x.yy${CSI_PROW_DEPLOYMENT_SUFFIX}" (if available), +# otherwise "kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}". # "none" disables the deployment of the hostpath driver. # # When no deploy script is found (nothing in `deploy` directory, @@ -203,6 +201,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" +configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" # The install_csi_driver function may work also for other CSI drivers, # as long as they follow the conventions of the CSI hostpath driver. @@ -361,10 +360,23 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external # to all the K8s versions we test against configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" -# This is the directory for additional result files. Usually set by Prow, but -# if not (for example, when invoking manually) it defaults to the work directory. -configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" -mkdir -p "${ARTIFACTS}" +# This creates directories that are required for testing. +ensure_paths () { + # Work directory. It has to allow running executables, therefore /tmp + # is avoided. Cleaning up after the script is intentionally left to + # the caller. + configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" + + # This is the directory for additional result files. Usually set by Prow, but + # if not (for example, when invoking manually) it defaults to the work directory. + configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" + mkdir -p "${ARTIFACTS}" + + # For additional tools. + CSI_PROW_BIN="${CSI_PROW_WORK}/bin" + mkdir -p "${CSI_PROW_BIN}" + PATH="${CSI_PROW_BIN}:$PATH" +} run () { echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2 @@ -384,11 +396,6 @@ die () { exit 1 } -# For additional tools. -CSI_PROW_BIN="${CSI_PROW_WORK}/bin" -mkdir -p "${CSI_PROW_BIN}" -PATH="${CSI_PROW_BIN}:$PATH" - # Ensure that PATH has the desired version of the Go tools, then run command given as argument. # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by # bumping the container image regularly. @@ -647,9 +654,9 @@ find_deployment () { # Ignore: See if you can use ${variable//search/replace} instead. # shellcheck disable=SC2001 - file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh" + file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" if ! [ -e "$file" ]; then - file="$dir/kubernetes-latest/deploy.sh" + file="$dir/kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -1098,6 +1105,9 @@ main () { local images ret ret=0 + # Set up work directory. + ensure_paths + images= if ${CSI_PROW_BUILD_JOB}; then # A successful build is required for testing. @@ -1259,6 +1269,9 @@ gcr_cloud_build () { # Required for "docker buildx build --push". gcloud auth configure-docker + # Might not be needed here, but call it just in case. + ensure_paths + if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then # Needed for "RUN" steps on non-linux/amd64 platforms. # See https://github.com/multiarch/qemu-user-static#getting-started diff --git a/pull-test.sh b/pull-test.sh new file mode 100755 index 000000000..b019c1776 --- /dev/null +++ b/pull-test.sh @@ -0,0 +1,32 @@ +#! /bin/sh + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is called by pull Prow jobs for the csi-release-tools +# repo to ensure that the changes in the PR work when imported into +# some other repo. + +set -ex + +# It must be called inside the updated csi-release-tools repo. +CSI_RELEASE_TOOLS_DIR="$(pwd)" + +# Update the other repo. +cd "$PULL_TEST_REPO_DIR" +git subtree pull --squash --prefix=release-tools "$CSI_RELEASE_TOOLS_DIR" master +git log -n2 + +# Now fall through to testing. +exec ./.prow.sh