-
Notifications
You must be signed in to change notification settings - Fork 1.3k
/
ci-e2e-lib.sh
291 lines (248 loc) · 12.5 KB
/
ci-e2e-lib.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
#!/bin/bash
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# capi:buildDockerImages builds all the required docker images, if not already present locally.
capi:buildDockerImages () {
# Configure provider images generation;
# please ensure the generated image name matches image names used in the E2E_CONF_FILE;
# also the same settings must be set in Makefile, docker-build-e2e target.
ARCH="$(go env GOARCH)"
export REGISTRY=gcr.io/k8s-staging-cluster-api
export TAG=dev
export ARCH
## Build all required docker image, if missing.
## Note: we check only for one image to exist, and assume that if one is missing all are missing.
if [[ "$(docker images -q "$REGISTRY/cluster-api-controller-$ARCH:$TAG" 2> /dev/null)" == "" ]]; then
echo "+ Building CAPI images and CAPD image"
make docker-build-e2e
else
echo "+ CAPI images already present in the system, skipping make"
fi
}
# k8s::prepareKindestImagesVariables defaults the environment variables KUBERNETES_VERSION_MANAGEMENT, KUBERNETES_VERSION,
# KUBERNETES_VERSION_UPGRADE_TO, KUBERNETES_VERSION_UPGRADE_FROM and KUBERNETES_VERSION_LATEST_CI
# depending on what is set in GINKGO_FOCUS.
# Note: We do this to ensure that the kindest/node image gets built if it does
# not already exist, e.g. for pre-releases, but only if necessary.
k8s::prepareKindestImagesVariables() {
# Always default KUBERNETES_VERSION_MANAGEMENT because we always create a management cluster out of it.
if [[ -z "${KUBERNETES_VERSION_MANAGEMENT:-}" ]]; then
KUBERNETES_VERSION_MANAGEMENT=$(grep KUBERNETES_VERSION_MANAGEMENT: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
echo "Defaulting KUBERNETES_VERSION_MANAGEMENT to ${KUBERNETES_VERSION_MANAGEMENT} to trigger image build (because env var is not set)"
fi
if [[ ${GINKGO_FOCUS:-} == *"K8s-Install-ci-latest"* ]]; then
# If the test focuses on [K8s-Install-ci-latest], only default KUBERNETES_VERSION_LATEST_CI
# to the value in the e2e config and only if it is not set.
# Note: We do this because we want to specify KUBERNETES_VERSION_LATEST_CI *only* in the e2e config.
if [[ -z "${KUBERNETES_VERSION_LATEST_CI:-}" ]]; then
KUBERNETES_VERSION_LATEST_CI=$(grep KUBERNETES_VERSION_LATEST_CI: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
echo "Defaulting KUBERNETES_VERSION_LATEST_CI to ${KUBERNETES_VERSION_LATEST_CI} to trigger image build (because env var is not set)"
fi
elif [[ ${GINKGO_FOCUS:-} != *"K8s-Upgrade"* ]]; then
# In any other case which is not [K8s-Upgrade], default KUBERNETES_VERSION if it is not set to make sure
# the corresponding kindest/node image exists.
if [[ -z "${KUBERNETES_VERSION:-}" ]]; then
KUBERNETES_VERSION=$(grep KUBERNETES_VERSION: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
echo "Defaulting KUBERNETES_VERSION to ${KUBERNETES_VERSION} to trigger image build (because env var is not set)"
fi
fi
# Tests not focusing on anything and skipping [Conformance] run a clusterctl upgrade test
# on the latest kubernetes version as management cluster.
if [[ ${GINKGO_FOCUS:-} == "" ]] && [[ ${GINKGO_SKIP} == *"Conformance"* ]]; then
# Note: We do this because we want to specify KUBERNETES_VERSION_LATEST_CI *only* in the e2e config.
if [[ -z "${KUBERNETES_VERSION_LATEST_CI:-}" ]]; then
KUBERNETES_VERSION_LATEST_CI=$(grep KUBERNETES_VERSION_LATEST_CI: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
echo "Defaulting KUBERNETES_VERSION_LATEST_CI to ${KUBERNETES_VERSION_LATEST_CI} to trigger image build (because env var is not set)"
fi
fi
# Tests not focusing on [PR-Blocking], [K8s-Install] or [K8s-Install-ci-latest],
# also run upgrade tests so default KUBERNETES_VERSION_UPGRADE_TO and KUBERNETES_VERSION_UPGRADE_FROM
# to the value in the e2e config if they are not set.
if [[ ${GINKGO_FOCUS:-} != *"PR-Blocking"* ]] && [[ ${GINKGO_FOCUS:-} != *"K8s-Install"* ]]; then
if [[ -z "${KUBERNETES_VERSION_UPGRADE_TO:-}" ]]; then
KUBERNETES_VERSION_UPGRADE_TO=$(grep KUBERNETES_VERSION_UPGRADE_TO: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
echo "Defaulting KUBERNETES_VERSION_UPGRADE_TO to ${KUBERNETES_VERSION_UPGRADE_TO} to trigger image build (because env var is not set)"
fi
if [[ -z "${KUBERNETES_VERSION_UPGRADE_FROM:-}" ]]; then
KUBERNETES_VERSION_UPGRADE_FROM=$(grep KUBERNETES_VERSION_UPGRADE_FROM: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
echo "Defaulting KUBERNETES_VERSION_UPGRADE_FROM to ${KUBERNETES_VERSION_UPGRADE_FROM} to trigger image build (because env var is not set)"
fi
fi
}
# k8s::prepareKindestImages checks all the e2e test variables representing a Kubernetes version,
# and makes sure a corresponding kindest/node image is available locally.
k8s::prepareKindestImages() {
if [ -n "${KUBERNETES_VERSION_MANAGEMENT:-}" ]; then
k8s::resolveVersion "KUBERNETES_VERSION_MANAGEMENT" "$KUBERNETES_VERSION_MANAGEMENT"
export KUBERNETES_VERSION_MANAGEMENT=$resolveVersion
kind::prepareKindestImage "$resolveVersion"
fi
if [ -n "${KUBERNETES_VERSION:-}" ]; then
k8s::resolveVersion "KUBERNETES_VERSION" "$KUBERNETES_VERSION"
export KUBERNETES_VERSION=$resolveVersion
kind::prepareKindestImage "$resolveVersion"
fi
if [ -n "${KUBERNETES_VERSION_UPGRADE_TO:-}" ]; then
k8s::resolveVersion "KUBERNETES_VERSION_UPGRADE_TO" "$KUBERNETES_VERSION_UPGRADE_TO"
export KUBERNETES_VERSION_UPGRADE_TO=$resolveVersion
kind::prepareKindestImage "$resolveVersion"
fi
if [ -n "${KUBERNETES_VERSION_UPGRADE_FROM:-}" ]; then
k8s::resolveVersion "KUBERNETES_VERSION_UPGRADE_FROM" "$KUBERNETES_VERSION_UPGRADE_FROM"
export KUBERNETES_VERSION_UPGRADE_FROM=$resolveVersion
kind::prepareKindestImage "$resolveVersion"
fi
if [ -n "${KUBERNETES_VERSION_LATEST_CI:-}" ]; then
k8s::resolveVersion "KUBERNETES_VERSION_LATEST_CI" "$KUBERNETES_VERSION_LATEST_CI"
export KUBERNETES_VERSION_LATEST_CI=$resolveVersion
kind::prepareKindestImage "$resolveVersion"
fi
if [ -n "${BUILD_NODE_IMAGE_TAG:-}" ]; then
k8s::resolveVersion "BUILD_NODE_IMAGE_TAG" "$BUILD_NODE_IMAGE_TAG"
export BUILD_NODE_IMAGE_TAG=$resolveVersion
kind::prepareKindestImage "$resolveVersion"
fi
}
# k8s::resolveVersion resolves kubernetes version labels (e.g. latest) to the corresponding version numbers.
# The result will be available in the resolveVersion variable which is accessible from the caller.
#
# NOTE: this can't be used for kindest/node images pulled from docker hub, given that there are not guarantees that
# such images are generated in sync with the Kubernetes release process.
k8s::resolveVersion() {
local variableName=$1
local version=$2
resolveVersion=$version
if [[ "$version" =~ ^v ]]; then
echo "+ $variableName=\"$version\" already a version, no need to resolve"
return
fi
if [[ "$version" =~ ^ci/ ]]; then
resolveVersion=$(curl -LsS "http://dl.k8s.io/ci/${version#ci/}.txt")
else
resolveVersion=$(curl -LsS "http://dl.k8s.io/release/${version}.txt")
fi
echo "+ $variableName=\"$version\" resolved to \"$resolveVersion\""
}
# kind::prepareKindestImage check if a kindest/image exist, and if yes, pre-pull it; otherwise it builds
# the kindest image locally
kind::prepareKindestImage() {
local version=$1
# ALWAYS_BUILD_KIND_IMAGES will default to false if unset.
ALWAYS_BUILD_KIND_IMAGES="${ALWAYS_BUILD_KIND_IMAGES:-"false"}"
# Try to pre-pull the image
kind::prepullImage "kindest/node:$version"
# if pre-pull failed, or ALWAYS_BUILD_KIND_IMAGES is true build the images locally.
if [[ "$retVal" != 0 ]] || [[ "$ALWAYS_BUILD_KIND_IMAGES" = "true" ]]; then
echo "+ building image for Kuberentes $version locally. This is either because the image wasn't available in docker hub or ALWAYS_BUILD_KIND_IMAGES is set to true"
kind::buildNodeImage "$version"
fi
}
# kind::buildNodeImage builds a kindest/node images starting from Kubernetes sources.
# the func expect an input parameter defining the image tag to be used.
kind::buildNodeImage() {
local version=$1
# move to the Kubernetes repository.
echo "KUBE_ROOT $GOPATH/src/k8s.io/kubernetes"
cd "$GOPATH/src/k8s.io/kubernetes" || exit
# checkouts the Kubernetes branch for the given version.
k8s::checkoutBranch "$version"
# sets the build version that will be applied by the Kubernetes build command called during kind build node-image.
k8s::setBuildVersion "$version"
# build the node image
version="${version//+/_}"
if [[ "${version}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "${version}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then
echo "+ Building kindest/node:$version using pre-built binaries"
kind build node-image --image "kindest/node:$version" "$version"
else
echo "+ Building kindest/node:$version from source"
kind build node-image --image "kindest/node:$version"
fi
# move back to Cluster API
cd "$REPO_ROOT" || exit
}
# k8s::checkoutBranch checkouts the Kubernetes branch for the given version.
k8s::checkoutBranch() {
local version=$1
echo "+ Checkout branch for Kubernetes $version"
# checkout the required tag/branch.
local buildMetadata
buildMetadata=$(echo "${version#v}" | awk '{split($0,a,"+"); print a[2]}')
if [[ "$buildMetadata" == "" ]]; then
# if there are no release metadata, it means we are looking for a Kubernetes version that
# should be already been tagged.
echo "+ checkout tag $version"
git fetch --all --tags
git checkout "tags/$version" -B "$version-branch"
else
# otherwise we are requiring a Kubernetes version that should be built from HEAD
# of one of the existing branches
echo "+ checking for existing branches"
git fetch --all
local major
local minor
major=$(echo "${version#v}" | awk '{split($0,a,"."); print a[1]}')
minor=$(echo "${version#v}" | awk '{split($0,a,"."); print a[2]}')
local releaseBranch
releaseBranch="$(git branch -r | grep "release-$major.$minor$" || true)"
if [[ "$releaseBranch" != "" ]]; then
# if there is already a release branch for the required Kubernetes branch, use it
echo "+ checkout $releaseBranch branch"
git checkout "$releaseBranch" -B "release-$major.$minor"
else
# otherwise, we should build from the main branch, which is the branch for the next release
# TODO(sbueringer) we can only change this to main after k/k has migrated to main
echo "+ checkout master branch"
git checkout master
fi
fi
}
# k8s::setBuildVersion sets the build version that will be applied by the Kubernetes build command.
# the func expect an input parameter defining the version to be used.
k8s::setBuildVersion() {
local version=$1
echo "+ Setting version for Kubernetes build to $version"
local major
local minor
major=$(echo "${version#v}" | awk '{split($0,a,"."); print a[1]}')
minor=$(echo "${version#v}" | awk '{split($0,a,"."); print a[2]}')
cat > build-version << EOL
export KUBE_GIT_MAJOR=$major
export KUBE_GIT_MINOR=$minor
export KUBE_GIT_VERSION=$version
export KUBE_GIT_TREE_STATE=clean
export KUBE_GIT_COMMIT=d34db33f
EOL
export KUBE_GIT_VERSION_FILE=$PWD/build-version
}
# kind:prepullAdditionalImages pre-pull all the additional (not Kindest/node) images that will be used in the e2e, thus making
# the actual test run less sensible to the network speed.
kind:prepullAdditionalImages () {
# Pulling cert manager images so we can pre-load in kind nodes
kind::prepullImage "quay.io/jetstack/cert-manager-cainjector:v1.16.1"
kind::prepullImage "quay.io/jetstack/cert-manager-webhook:v1.16.1"
kind::prepullImage "quay.io/jetstack/cert-manager-controller:v1.16.1"
}
# kind:prepullImage pre-pull a docker image if no already present locally.
# The result will be available in the retVal value which is accessible from the caller.
kind::prepullImage () {
local image=$1
image="${image//+/_}"
retVal=0
if [[ "$(docker images -q "$image" 2> /dev/null)" == "" ]]; then
echo "+ Pulling $image"
docker pull "$image" || retVal=$?
else
echo "+ image $image already present in the system, skipping pre-pull"
fi
}