forked from kubernetes/test-infra
-
Notifications
You must be signed in to change notification settings - Fork 0
/
kind-logs-e2e-k8s.sh
executable file
·325 lines (297 loc) · 10.8 KB
/
kind-logs-e2e-k8s.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
#!/bin/sh
# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# hack script for running a kind e2e
# must be run with a kubernetes checkout in $PWD (IE from the checkout)
# Usage: SKIP="ginkgo skip regex" FOCUS="ginkgo focus regex" kind-e2e.sh
set -o errexit -o nounset -o xtrace
# Settings:
# SKIP: ginkgo skip regex
# FOCUS: ginkgo focus regex
# GA_ONLY: true - limit to GA APIs/features as much as possible
# false - (default) APIs and features left at defaults
# FEATURE_GATES:
# JSON or YAML encoding of a string/bool map: {"FeatureGateA": true, "FeatureGateB": false}
# Enables or disables feature gates in the entire cluster.
# Cannot be used when GA_ONLY=true.
# RUNTIME_CONFIG:
# JSON or YAML encoding of a string/string (!) map: {"apia.example.com/v1alpha1": "true", "apib.example.com/v1beta1": "false"}
# Enables API groups in the apiserver via --runtime-config.
# Cannot be used when GA_ONLY=true.
# cleanup logic for cleanup on exit
CLEANED_UP=false
cleanup() {
if [ "$CLEANED_UP" = "true" ]; then
return
fi
# KIND_CREATE_ATTEMPTED is true once we: kind create
if [ "${KIND_CREATE_ATTEMPTED:-}" = true ]; then
kind "export" logs "${ARTIFACTS}" || true
kind delete cluster || true
fi
rm -f _output/bin/e2e.test || true
# remove our tempdir, this needs to be last, or it will prevent kind delete
if [ -n "${TMP_DIR:-}" ]; then
rm -rf "${TMP_DIR:?}"
fi
CLEANED_UP=true
}
# setup signal handlers
# shellcheck disable=SC2317 # this is not unreachable code
signal_handler() {
if [ -n "${GINKGO_PID:-}" ]; then
kill -TERM "$GINKGO_PID" || true
fi
cleanup
}
trap signal_handler INT TERM
# build kubernetes / node image, e2e binaries
build() {
# build the node image w/ kubernetes
kind build node-image -v 1
# Ginkgo v1 is used by Kubernetes 1.24 and earlier, fallback if v2 is not available.
GINKGO_SRC_DIR="vendor/github.com/onsi/ginkgo/v2/ginkgo"
if [ ! -d "$GINKGO_SRC_DIR" ]; then
GINKGO_SRC_DIR="vendor/github.com/onsi/ginkgo/ginkgo"
fi
# make sure we have e2e requirements
make all WHAT="cmd/kubectl test/e2e/e2e.test ${GINKGO_SRC_DIR}"
}
check_structured_log_support() {
case "${KUBE_VERSION}" in
v1.1[0-8].*)
echo "$1 is only supported on versions >= v1.19, got ${KUBE_VERSION}"
exit 1
;;
esac
}
# up a cluster with kind
create_cluster() {
# Grab the version of the cluster we're about to start
KUBE_VERSION="$(docker run --rm --entrypoint=cat "kindest/node:latest" /kind/version)"
# Default Log level for all components in test clusters
KIND_CLUSTER_LOG_LEVEL=${KIND_CLUSTER_LOG_LEVEL:-4}
# potentially enable --logging-format
CLUSTER_LOG_FORMAT=${CLUSTER_LOG_FORMAT:-}
scheduler_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\""
controllerManager_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\""
apiServer_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\""
if [ -n "$CLUSTER_LOG_FORMAT" ]; then
check_structured_log_support "CLUSTER_LOG_FORMAT"
scheduler_extra_args="${scheduler_extra_args}
\"logging-format\": \"${CLUSTER_LOG_FORMAT}\""
controllerManager_extra_args="${controllerManager_extra_args}
\"logging-format\": \"${CLUSTER_LOG_FORMAT}\""
apiServer_extra_args="${apiServer_extra_args}
\"logging-format\": \"${CLUSTER_LOG_FORMAT}\""
fi
kubelet_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\"
\"container-log-max-size\": 100Mi"
KUBELET_LOG_FORMAT=${KUBELET_LOG_FORMAT:-$CLUSTER_LOG_FORMAT}
if [ -n "$KUBELET_LOG_FORMAT" ]; then
check_structured_log_support "KUBECTL_LOG_FORMAT"
kubelet_extra_args="${kubelet_extra_args}
\"logging-format\": \"${KUBELET_LOG_FORMAT}\""
fi
# JSON or YAML map injected into featureGates config
feature_gates="${FEATURE_GATES:-{\}}"
# --runtime-config argument value passed to the API server, again as a map
runtime_config="${RUNTIME_CONFIG:-{\}}"
case "${GA_ONLY:-false}" in
false)
:
;;
true)
if [ "${feature_gates}" != "{}" ]; then
echo "GA_ONLY=true and FEATURE_GATES=${feature_gates} are mutually exclusive."
exit 1
fi
if [ "${runtime_config}" != "{}" ]; then
echo "GA_ONLY=true and RUNTIME_CONFIG=${runtime_config} are mutually exclusive."
exit 1
fi
echo "Limiting to GA APIs and features for ${KUBE_VERSION}"
feature_gates='{"AllAlpha":false,"AllBeta":false}'
runtime_config='{"api/alpha":"false", "api/beta":"false"}'
;;
*)
echo "\$GA_ONLY set to '${GA_ONLY}'; supported values are true and false (default)"
exit 1
;;
esac
# create the config file
cat <<EOF > "${ARTIFACTS}/kind-config.yaml"
# config for 1 control plane node and 2 workers (necessary for conformance)
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
ipFamily: ${IP_FAMILY:-ipv4}
kubeProxyMode: ${KUBE_PROXY_MODE:-iptables}
# don't pass through host search paths
# TODO: possibly a reasonable default in the future for kind ...
dnsSearch: []
nodes:
- role: control-plane
- role: worker
- role: worker
featureGates: ${feature_gates}
runtimeConfig: ${runtime_config}
kubeadmConfigPatches:
- |
kind: ClusterConfiguration
metadata:
name: config
apiServer:
extraArgs:
${apiServer_extra_args}
controllerManager:
extraArgs:
${controllerManager_extra_args}
scheduler:
extraArgs:
${scheduler_extra_args}
---
kind: InitConfiguration
nodeRegistration:
kubeletExtraArgs:
${kubelet_extra_args}
---
kind: JoinConfiguration
nodeRegistration:
kubeletExtraArgs:
${kubelet_extra_args}
EOF
# NOTE: must match the number of workers above
NUM_NODES=2
# actually create the cluster
# TODO(BenTheElder): settle on verbosity for this script
KIND_CREATE_ATTEMPTED=true
kind create cluster \
--image=kindest/node:latest \
--retain \
--wait=1m \
-v=3 \
"--config=${ARTIFACTS}/kind-config.yaml"
# debug cluster version
kubectl version
# Patch kube-proxy to set the verbosity level
kubectl patch -n kube-system daemonset/kube-proxy \
--type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--v='"${KIND_CLUSTER_LOG_LEVEL}"'" }]'
}
# run e2es with ginkgo-e2e.sh
run_tests() {
# IPv6 clusters need some CoreDNS changes in order to work in k8s CI:
# 1. k8s CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
# to work in an offline environment:
# https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
# 2. k8s CI adds following domains to resolv.conf search field:
# c.k8s-prow-builds.internal google.internal.
# CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
# otherwise pods stops trying to resolve the domain.
if [ "${IP_FAMILY:-ipv4}" = "ipv6" ]; then
# Get the current config
original_coredns=$(kubectl get -oyaml -n=kube-system configmap/coredns)
echo "Original CoreDNS config:"
echo "${original_coredns}"
# Patch it
fixed_coredns=$(
printf '%s' "${original_coredns}" | sed \
-e 's/^.*kubernetes cluster\.local/& internal/' \
-e '/^.*upstream$/d' \
-e '/^.*fallthrough.*$/d' \
-e '/^.*forward . \/etc\/resolv.conf$/d' \
-e '/^.*loop$/d' \
)
echo "Patched CoreDNS config:"
echo "${fixed_coredns}"
printf '%s' "${fixed_coredns}" | kubectl apply -f -
fi
# ginkgo regexes
SKIP="${SKIP:-}"
FOCUS="${FOCUS:-"\\[Conformance\\]"}"
# if we set PARALLEL=true, skip serial tests set --ginkgo-parallel
if [ "${PARALLEL:-false}" = "true" ]; then
export GINKGO_PARALLEL=y
if [ -z "${SKIP}" ]; then
SKIP="\\[Serial\\]"
else
SKIP="\\[Serial\\]|${SKIP}"
fi
fi
# setting this env prevents ginkgo e2e from trying to run provider setup
export KUBERNETES_CONFORMANCE_TEST='y'
# setting these is required to make RuntimeClass tests work ... :/
export KUBE_CONTAINER_RUNTIME=remote
export KUBE_CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
export KUBE_CONTAINER_RUNTIME_NAME=containerd
# ginkgo can take forever to exit, so we run it in the background and save the
# PID, bash will not run traps while waiting on a process, but it will while
# running a builtin like `wait`, saving the PID also allows us to forward the
# interrupt
./hack/ginkgo-e2e.sh \
'--provider=skeleton' "--num-nodes=${NUM_NODES}" \
"--ginkgo.focus=${FOCUS}" "--ginkgo.skip=${SKIP}" \
"--report-dir=${ARTIFACTS}" '--disable-log-dump=true' &
GINKGO_PID=$!
wait "$GINKGO_PID"
}
prune_kind_logs() {
# The logs for the pods are the same as for the containers. k/k/test/integration/logs/benchmark/get-logs.sh
# only looks for the container logs.
rm -rf "${ARTIFACTS}"/kind-*/pods
# One kubelet log is enough. get-logs.sh uses kind-worker.
rm -f "${ARTIFACTS}"/kind-control-plane/kubelet.log "${ARTIFACTS}"/kind-worker[0-9]*/kubelet.log
# The journal is large and only useful for debugging cluster startup. Let's
# assume that we don't need it.
rm -f "${ARTIFACTS}"/kind-*/journal.log
# Above we allow individual files to be as large as 100Mi before
# kubelet rotates them. Because "kind export logs" only copies
# the current log, each file is smaller than 100Mi. In practice,
# files are smaller. As a safeguard we fail the job here if the
# overall log data exceeds 300Mi.
total="$(du -b -s -c "${ARTIFACTS}"/kind-* | tail -1 | sed -e 's/\s*total//')"
limit=$((300 * 1024 * 1024 ))
if [ "$total" -gt "$limit" ]; then
echo "ERROR: Total amount of data in <ARTIFACTS>/kind-* is $total bytes, which is more than the limit of $limit. Try reducing verbosity or number of tests."
return 1
fi
}
main() {
# create temp dir and setup cleanup
TMP_DIR=$(mktemp -d)
# ensure artifacts (results) directory exists when not in CI
export ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
mkdir -p "${ARTIFACTS}"
# export the KUBECONFIG to a unique path for testing
KUBECONFIG="${HOME}/.kube/kind-test-config"
export KUBECONFIG
echo "exported KUBECONFIG=${KUBECONFIG}"
# debug kind version
kind version
# build kubernetes
build
# in CI attempt to release some memory after building
if [ -n "${KUBETEST_IN_DOCKER:-}" ]; then
sync || true
echo 1 > /proc/sys/vm/drop_caches || true
fi
# create the cluster and run tests
res=0
create_cluster || res=$?
run_tests || res=$?
cleanup || res=$?
prune_kind_logs || res=$?
exit $res
}
main