forked from envoyproxy/envoy
-
Notifications
You must be signed in to change notification settings - Fork 0
/
do_ci.sh
executable file
·1056 lines (968 loc) · 39.7 KB
/
do_ci.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bash
# Run a CI build/test target, e.g. docs, asan.
set -e
# TODO(phlax): Clarify and/or integrate SRCDIR and ENVOY_SRCDIR
export SRCDIR="${SRCDIR:-$PWD}"
export ENVOY_SRCDIR="${ENVOY_SRCDIR:-$PWD}"
CURRENT_SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
# shellcheck source=ci/build_setup.sh
. "${CURRENT_SCRIPT_DIR}"/build_setup.sh
echo "building using ${NUM_CPUS} CPUs"
echo "building for ${ENVOY_BUILD_ARCH}"
cd "${SRCDIR}"
# Its better to fetch too little rather than too much, as whatever is
# actually used is what will be cached.
# Fetching is mostly for robustness rather than optimization.
FETCH_TARGETS=(
@bazel_tools//tools/jdk:remote_jdk11
//bazel/rbe/toolchains/...
//tools/gsutil
//tools/zstd)
FETCH_BUILD_TARGETS=(
//contrib/exe/...
//distribution/...
//source/exe/...)
FETCH_GCC_TARGETS=(
//source/exe/...)
# TODO(phlax): add this as a general cache
# this fetches a bit too much for some of the targets
# but its not really possible to filter their needs so move
# to a shared precache
FETCH_TEST_TARGETS=(
@nodejs//...
//test/...)
FETCH_ALL_TEST_TARGETS=(
@com_github_google_quiche//:ci_tests
"${FETCH_TEST_TARGETS[@]}")
FETCH_API_TARGETS=(
@envoy_api//...
//tools/api_proto_plugin/...
//tools/protoprint/...
//tools/protoxform/...
//tools/type_whisperer/...
//tools/testdata/protoxform/...)
FETCH_DOCS_TARGETS+=(
//docs/...)
FETCH_FORMAT_TARGETS+=(
//tools/code_format/...)
FETCH_PROTO_TARGETS=(
@com_github_bufbuild_buf//:bin/buf
//tools/proto_format/...)
retry () {
local n wait iterations
wait="${1}"
iterations="${2}"
shift 2
n=0
until [ "$n" -ge "$iterations" ]; do
"${@}" \
&& break
n=$((n+1))
if [[ "$n" -lt "$iterations" ]]; then
sleep "$wait"
echo "Retrying ..."
else
echo "Fetch failed"
exit 1
fi
done
}
if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then
BUILD_ARCH_DIR="/linux/amd64"
elif [[ "${ENVOY_BUILD_ARCH}" == "aarch64" ]]; then
BUILD_ARCH_DIR="/linux/arm64"
else
# Fall back to use the ENVOY_BUILD_ARCH itself.
BUILD_ARCH_DIR="/linux/${ENVOY_BUILD_ARCH}"
fi
setup_clang_toolchain() {
CONFIG_PARTS=()
if [[ -n "${ENVOY_RBE}" ]]; then
CONFIG_PARTS+=("remote")
fi
CONFIG_PARTS+=("clang")
ENVOY_STDLIB="${ENVOY_STDLIB:-libc++}"
if [[ "${ENVOY_STDLIB}" == "libc++" ]]; then
CONFIG_PARTS+=("libc++")
fi
CONFIG="$(IFS=- ; echo "${CONFIG_PARTS[*]}")"
BAZEL_BUILD_OPTIONS+=("--config=${CONFIG}")
BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]}"
export BAZEL_BUILD_OPTION_LIST
echo "clang toolchain with ${ENVOY_STDLIB} configured: ${CONFIG}"
}
function collect_build_profile() {
local output_base
declare -g build_profile_count=${build_profile_count:-1}
output_base="$(bazel info "${BAZEL_BUILD_OPTIONS[@]}" output_base)"
mv -f \
"${output_base}/command.profile.gz" \
"${ENVOY_BUILD_PROFILE}/${build_profile_count}-$1.profile.gz" \
|| :
((build_profile_count++))
}
function bazel_with_collection() {
local failed_logs
declare -r BAZEL_OUTPUT="${ENVOY_SRCDIR}"/bazel.output.txt
bazel "$@" | tee "${BAZEL_OUTPUT}"
declare BAZEL_STATUS="${PIPESTATUS[0]}"
if [ "${BAZEL_STATUS}" != "0" ]
then
pushd bazel-testlogs
failed_logs=$(grep " /build.*test.log" "${BAZEL_OUTPUT}" | sed -e 's/ \/build.*\/testlogs\/\(.*\)/\1/')
while read -r f; do
cp --parents -f "$f" "${ENVOY_FAILED_TEST_LOGS}"
done <<< "$failed_logs"
popd
exit "${BAZEL_STATUS}"
fi
collect_build_profile "$1"
}
function cp_binary_for_image_build() {
local BINARY_TYPE="$1"
local COMPILE_TYPE="$2"
local EXE_NAME="$3"
# TODO(mattklein123): Replace this with caching and a different job which creates images.
local BASE_TARGET_DIR="${ENVOY_SRCDIR}${BUILD_ARCH_DIR}"
local TARGET_DIR=build_"${EXE_NAME}"_"${BINARY_TYPE}"
local FINAL_DELIVERY_DIR="${ENVOY_DELIVERY_DIR}"/"${EXE_NAME}"
echo "Copying binary for image build..."
mkdir -p "${BASE_TARGET_DIR}"/"${TARGET_DIR}"
cp -f "${FINAL_DELIVERY_DIR}"/envoy "${BASE_TARGET_DIR}"/"${TARGET_DIR}"
if [[ "${COMPILE_TYPE}" == "dbg" || "${COMPILE_TYPE}" == "opt" ]]; then
cp -f "${FINAL_DELIVERY_DIR}"/envoy.dwp "${BASE_TARGET_DIR}"/"${TARGET_DIR}"
fi
# Tools for the tools image. Strip to save size.
strip bazel-bin/test/tools/schema_validator/schema_validator_tool \
-o "${BASE_TARGET_DIR}"/"${TARGET_DIR}"/schema_validator_tool
strip bazel-bin/test/tools/router_check/router_check_tool \
-o "${BASE_TARGET_DIR}"/"${TARGET_DIR}"/router_check_tool
# Copy the su-exec utility binary into the image
cp -f bazel-bin/external/com_github_ncopa_suexec/su-exec "${BASE_TARGET_DIR}"/"${TARGET_DIR}"
# Stripped binaries for the debug image.
mkdir -p "${BASE_TARGET_DIR}"/"${TARGET_DIR}"_stripped
strip "${FINAL_DELIVERY_DIR}"/envoy -o "${BASE_TARGET_DIR}"/"${TARGET_DIR}"_stripped/envoy
# only if BUILD_REASON exists (running in AZP)
if [[ "${BUILD_REASON}" ]]; then
# Copy for azp which doesn't preserve permissions
tar czf "${ENVOY_BUILD_DIR}"/"${EXE_NAME}"_binary.tar.gz -C "${BASE_TARGET_DIR}" "${TARGET_DIR}" "${TARGET_DIR}"_stripped
# Remove binaries to save space
rm -rf "${BASE_TARGET_DIR:?}"/"${TARGET_DIR}" "${BASE_TARGET_DIR:?}"/"${TARGET_DIR}"_stripped "${FINAL_DELIVERY_DIR:?}"/envoy{,.dwp} \
bazel-bin/"${ENVOY_BIN}"{,.dwp}
fi
}
function bazel_binary_build() {
local BINARY_TYPE="$1"
if [[ "${BINARY_TYPE}" == "release" ]]; then
COMPILE_TYPE="opt"
elif [[ "${BINARY_TYPE}" == "debug" ]]; then
COMPILE_TYPE="dbg"
elif [[ "${BINARY_TYPE}" == "sizeopt" ]]; then
# The COMPILE_TYPE variable is redundant in this case and is only here for
# readability. It is already set in the .bazelrc config for sizeopt.
COMPILE_TYPE="opt"
CONFIG_ARGS=("--config=sizeopt")
elif [[ "${BINARY_TYPE}" == "fastbuild" ]]; then
COMPILE_TYPE="fastbuild"
fi
local BUILD_TARGET="$2"
local BUILD_DEBUG_INFORMATION="$3"
local EXE_NAME="$4"
local FINAL_DELIVERY_DIR="${ENVOY_DELIVERY_DIR}"/"${EXE_NAME}"
mkdir -p "${FINAL_DELIVERY_DIR}"
echo "Building (type=${BINARY_TYPE} target=${BUILD_TARGET} debug=${BUILD_DEBUG_INFORMATION} name=${EXE_NAME})..."
ENVOY_BIN=$(echo "${BUILD_TARGET}" | sed -e 's#^@\([^/]*\)/#external/\1#;s#^//##;s#:#/#')
echo "ENVOY_BIN=${ENVOY_BIN}"
# This is a workaround for https://github.com/bazelbuild/bazel/issues/11834
[[ -n "${ENVOY_RBE}" ]] && rm -rf bazel-bin/"${ENVOY_BIN}"*
bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" "${BUILD_TARGET}" "${CONFIG_ARGS[@]}"
collect_build_profile "${BINARY_TYPE}"_build
# Copy the built envoy binary somewhere that we can access outside of the
# container.
cp -f bazel-bin/"${ENVOY_BIN}" "${FINAL_DELIVERY_DIR}"/envoy
if [[ "${COMPILE_TYPE}" == "dbg" || "${COMPILE_TYPE}" == "opt" ]]; then
# Generate dwp file for debugging since we used split DWARF to reduce binary
# size
bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" "${BUILD_DEBUG_INFORMATION}" "${CONFIG_ARGS[@]}"
# Copy the debug information
cp -f bazel-bin/"${ENVOY_BIN}".dwp "${FINAL_DELIVERY_DIR}"/envoy.dwp
fi
# Validation tools for the tools image.
bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" \
//test/tools/schema_validator:schema_validator_tool "${CONFIG_ARGS[@]}"
bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" \
//test/tools/router_check:router_check_tool "${CONFIG_ARGS[@]}"
# Build su-exec utility
bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" @com_github_ncopa_suexec//:su-exec
cp_binary_for_image_build "${BINARY_TYPE}" "${COMPILE_TYPE}" "${EXE_NAME}"
}
function bazel_envoy_binary_build() {
bazel_binary_build "$1" "${ENVOY_BUILD_TARGET}" "${ENVOY_BUILD_DEBUG_INFORMATION}" envoy
}
function bazel_contrib_binary_build() {
bazel_binary_build "$1" "${ENVOY_CONTRIB_BUILD_TARGET}" "${ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION}" envoy-contrib
}
function bazel_envoy_api_build() {
# Use libstdc++ because the API booster links to prebuilt libclang*/libLLVM* installed in /opt/llvm/lib,
# which is built with libstdc++. Using libstdc++ for whole of the API CI job to avoid unnecessary rebuild.
ENVOY_STDLIB="libstdc++"
setup_clang_toolchain
export CLANG_TOOLCHAIN_SETUP=1
export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config
echo "Run protoxform test"
bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
--//tools/api_proto_plugin:default_type_db_target=//tools/testdata/protoxform:fix_protos \
--//tools/api_proto_plugin:extra_args=api_version:3.7 \
//tools/protoprint:protoprint_test
echo "Validating API structure..."
"${ENVOY_SRCDIR}"/tools/api/validate_structure.py
echo "Testing API..."
bazel_with_collection \
test "${BAZEL_BUILD_OPTIONS[@]}" \
--remote_download_minimal \
-c fastbuild \
@envoy_api//test/... \
@envoy_api//tools/... \
@envoy_api//tools:tap2pcap_test
echo "Building API..."
bazel build "${BAZEL_BUILD_OPTIONS[@]}" \
-c fastbuild @envoy_api//envoy/...
}
function bazel_envoy_api_go_build() {
if [[ -z "$CLANG_TOOLCHAIN_SETUP" ]]; then
setup_clang_toolchain
fi
GO_IMPORT_BASE="github.com/envoyproxy/go-control-plane"
GO_TARGETS=(@envoy_api//...)
read -r -a GO_PROTOS <<< "$(bazel query "${BAZEL_GLOBAL_OPTIONS[@]}" "kind('go_proto_library', ${GO_TARGETS[*]})" | tr '\n' ' ')"
echo "${GO_PROTOS[@]}" | grep -q envoy_api || echo "No go proto targets found"
bazel build "${BAZEL_BUILD_OPTIONS[@]}" \
--experimental_proto_descriptor_sets_include_source_info \
--remote_download_outputs=all \
"${GO_PROTOS[@]}"
rm -rf build_go
mkdir -p build_go
echo "Copying go protos -> build_go"
BAZEL_BIN="$(bazel info "${BAZEL_BUILD_OPTIONS[@]}" bazel-bin)"
for GO_PROTO in "${GO_PROTOS[@]}"; do
# strip @envoy_api//
RULE_DIR="$(echo "${GO_PROTO:12}" | cut -d: -f1)"
PROTO="$(echo "${GO_PROTO:12}" | cut -d: -f2)"
INPUT_DIR="${BAZEL_BIN}/external/envoy_api/${RULE_DIR}/${PROTO}_/${GO_IMPORT_BASE}/${RULE_DIR}"
OUTPUT_DIR="build_go/${RULE_DIR}"
mkdir -p "$OUTPUT_DIR"
if [[ ! -e "$INPUT_DIR" ]]; then
echo "Unable to find input ${INPUT_DIR}" >&2
exit 1
fi
# echo "Copying go files ${INPUT_DIR} -> ${OUTPUT_DIR}"
while read -r GO_FILE; do
cp -a "$GO_FILE" "$OUTPUT_DIR"
if [[ "$GO_FILE" = *.validate.go ]]; then
sed -i '1s;^;//go:build !disable_pgv\n;' "$OUTPUT_DIR/$(basename "$GO_FILE")"
fi
done <<< "$(find "$INPUT_DIR" -name "*.go")"
done
}
CI_TARGET=$1
shift
if [[ "$CI_TARGET" =~ bazel.* ]]; then
ORIG_CI_TARGET="$CI_TARGET"
CI_TARGET="$(echo "${CI_TARGET}" | cut -d. -f2-)"
echo "Using \`${ORIG_CI_TARGET}\` is deprecated, please use \`${CI_TARGET}\`"
fi
if [[ $# -ge 1 ]]; then
COVERAGE_TEST_TARGETS=("$@")
TEST_TARGETS=("$@")
else
# Coverage test will add QUICHE tests by itself.
COVERAGE_TEST_TARGETS=("//test/...")
if [[ "${CI_TARGET}" == "release" || "${CI_TARGET}" == "release.test_only" ]]; then
# We test contrib on release only.
COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "//contrib/...")
elif [[ "${CI_TARGET}" == "msan" ]]; then
COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "-//test/extensions/...")
fi
TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "@com_github_google_quiche//:ci_tests")
fi
case $CI_TARGET in
api)
bazel_envoy_api_build
if [[ -n "$ENVOY_API_ONLY" ]]; then
exit 0
fi
bazel_envoy_api_go_build
;;
api.go)
bazel_envoy_api_go_build
;;
api_compat)
echo "Checking API for breaking changes to protobuf backwards compatibility..."
BASE_BRANCH_REF=$("${ENVOY_SRCDIR}"/tools/git/last_github_commit.sh)
COMMIT_TITLE=$(git log -n 1 --pretty='format:%C(auto)%h (%s, %ad)' "${BASE_BRANCH_REF}")
echo -e "\tUsing base commit ${COMMIT_TITLE}"
# BAZEL_BUILD_OPTIONS needed for setting the repository_cache param.
bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
//tools/api_proto_breaking_change_detector:detector_ci \
"${BASE_BRANCH_REF}"
;;
asan)
setup_clang_toolchain
if [[ -n "$ENVOY_RBE" ]]; then
ASAN_CONFIG="--config=rbe-toolchain-asan"
else
ASAN_CONFIG="--config=clang-asan"
fi
BAZEL_BUILD_OPTIONS+=(
-c dbg
"${ASAN_CONFIG}"
"--build_tests_only"
"--remote_download_minimal")
echo "bazel ASAN/UBSAN debug build with tests"
echo "Building and testing envoy tests ${TEST_TARGETS[*]}"
bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" "${TEST_TARGETS[@]}"
# TODO(mattklein123): This part of the test is now flaky in CI and it's unclear why, possibly
# due to sandboxing issue. Debug and enable it again.
# if [ "${CI_SKIP_INTEGRATION_TEST_TRAFFIC_TAPPING}" != "1" ] ; then
# Also validate that integration test traffic tapping (useful when debugging etc.)
# works. This requires that we set TAP_PATH. We do this under bazel.asan to
# ensure a debug build in CI.
# echo "Validating integration test traffic tapping..."
# bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" \
# --run_under=@envoy//bazel/test:verify_tap_test.sh \
# //test/extensions/transport_sockets/tls/integration:ssl_integration_test
# fi
;;
format-api|check_and_fix_proto_format)
setup_clang_toolchain
echo "Check and fix proto format ..."
"${ENVOY_SRCDIR}/ci/check_and_fix_format.sh"
;;
check_proto_format)
setup_clang_toolchain
echo "Check proto format ..."
"${ENVOY_SRCDIR}/tools/proto_format/proto_format.sh" check
;;
clang_tidy)
# clang-tidy will warn on standard library issues with libc++
ENVOY_STDLIB="libstdc++"
setup_clang_toolchain
export CLANG_TIDY_FIX_DIFF="${ENVOY_TEST_TMPDIR}/lint-fixes/clang-tidy-fixed.diff"
export FIX_YAML="${ENVOY_TEST_TMPDIR}/lint-fixes/clang-tidy-fixes.yaml"
export CLANG_TIDY_APPLY_FIXES=1
mkdir -p "${ENVOY_TEST_TMPDIR}/lint-fixes"
CLANG_TIDY_TARGETS=(
//contrib/...
//source/...
//test/...
@envoy_api//...)
bazel build \
"${BAZEL_BUILD_OPTIONS[@]}" \
--config clang-tidy \
"${CLANG_TIDY_TARGETS[@]}"
;;
clean|expunge)
setup_clang_toolchain
if [[ "$CI_TARGET" == "expunge" ]]; then
CLEAN_ARGS+=(--expunge)
fi
bazel clean "${BAZEL_GLOBAL_OPTIONS[@]}" "${CLEAN_ARGS[@]}"
;;
compile_time_options)
# See `compile-time-options` in `.bazelrc`
setup_clang_toolchain
# This doesn't go into CI but is available for developer convenience.
echo "bazel with different compiletime options build with tests..."
TEST_TARGETS=("${TEST_TARGETS[@]/#\/\//@envoy\/\/}")
# Building all the dependencies from scratch to link them against libc++.
echo "Building and testing with wasm=wamr: ${TEST_TARGETS[*]}"
bazel_with_collection \
test "${BAZEL_BUILD_OPTIONS[@]}" \
--config=compile-time-options \
--define wasm=wamr \
-c fastbuild \
"${TEST_TARGETS[@]}" \
--test_tag_filters=-nofips \
--build_tests_only
echo "Building and testing with wasm=wasmtime: and admin_functionality and admin_html disabled ${TEST_TARGETS[*]}"
bazel_with_collection \
test "${BAZEL_BUILD_OPTIONS[@]}" \
--config=compile-time-options \
--define wasm=wasmtime \
--define admin_functionality=disabled \
-c fastbuild \
"${TEST_TARGETS[@]}" \
--test_tag_filters=-nofips \
--build_tests_only
# "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only
# these tests under "-c opt" to save time in CI.
bazel_with_collection \
test "${BAZEL_BUILD_OPTIONS[@]}" \
--config=compile-time-options \
--define wasm=wasmtime \
-c opt \
@envoy//test/common/common:assert_test \
@envoy//test/server:server_test
# "--define log_fast_debug_assert_in_release=enabled" must be tested with a release build, so run only these tests under "-c opt" to save time in CI. This option will test only ASSERT()s without SLOW_ASSERT()s, so additionally disable "--define log_debug_assert_in_release" which compiles in both.
bazel_with_collection \
test "${BAZEL_BUILD_OPTIONS[@]}" \
--config=compile-time-options \
--define wasm=wamtime \
-c opt \
@envoy//test/common/common:assert_test \
--define log_fast_debug_assert_in_release=enabled \
--define log_debug_assert_in_release=disabled
echo "Building binary with wasm=wasmtime... and logging disabled"
bazel build "${BAZEL_BUILD_OPTIONS[@]}" \
--config=compile-time-options \
--define wasm=wasmtime \
--define enable_logging=disabled \
-c fastbuild \
@envoy//source/exe:envoy-static \
--build_tag_filters=-nofips
collect_build_profile build
;;
coverage|fuzz_coverage)
setup_clang_toolchain
echo "${CI_TARGET} build with tests ${COVERAGE_TEST_TARGETS[*]}"
if [[ "$CI_TARGET" == "fuzz_coverage" ]]; then
export FUZZ_COVERAGE=true
fi
export BAZEL_GRPC_LOG="${ENVOY_BUILD_DIR}/grpc.log"
"${ENVOY_SRCDIR}/test/run_envoy_bazel_coverage.sh" \
"${COVERAGE_TEST_TARGETS[@]}"
collect_build_profile coverage
;;
coverage-upload|fuzz_coverage-upload)
setup_clang_toolchain
if [[ "$CI_TARGET" == "fuzz_coverage-upload" ]]; then
TARGET=fuzz_coverage
else
TARGET=coverage
fi
GCS_LOCATION=$(
bazel run //tools/gcs:upload \
"${GCS_ARTIFACT_BUCKET}" \
"${GCP_SERVICE_ACCOUNT_KEY_PATH}" \
"/source/generated/${TARGET}" \
"$TARGET" \
"${GCS_REDIRECT_PATH}")
if [[ "${COVERAGE_FAILED}" -eq 1 ]]; then
echo "##vso[task.logissue type=error]Coverage failed, check artifact at: ${GCS_LOCATION}"
fi
;;
debug)
setup_clang_toolchain
echo "Testing ${TEST_TARGETS[*]}"
# Make sure that there are no regressions to building Envoy with autolink disabled.
EXTRA_OPTIONS=(
"--define" "library_autolink=disabled")
bazel test "${BAZEL_BUILD_OPTIONS[@]}" \
"${EXTRA_OPTIONS[@]}" \
-c dbg \
"${TEST_TARGETS[@]}"
echo "bazel debug build with tests..."
bazel_envoy_binary_build debug
;;
debug.server_only)
setup_clang_toolchain
echo "bazel debug build..."
bazel_envoy_binary_build debug
;;
deps)
setup_clang_toolchain
echo "dependency validate_test..."
bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
//tools/dependency:validate_test
echo "verifying dependencies..."
# Validate dependency relationships between core/extensions and external deps.
time bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
//tools/dependency:validate
# Validate repository metadata.
echo "check repositories..."
"${ENVOY_SRCDIR}/tools/check_repositories.sh"
echo "check dependencies..."
# Using todays date as an action_env expires the NIST cache daily, which is the update frequency
TODAY_DATE=$(date -u -I"date")
export TODAY_DATE
bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:check \
--//tools/dependency:preload_cve_data \
--action_env=TODAY_DATE \
-- -v warn \
-c cves release_dates releases
# Run dependabot tests
echo "Check dependabot ..."
bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
//tools/dependency:dependatool
# Disable this pending resolution to https://github.com/envoyproxy/envoy/issues/30286
# Run pip requirements tests
# echo "Check pip requirements ..."
# bazel test "${BAZEL_BUILD_OPTIONS[@]}" \
# //tools/base:requirements_test
;;
dev)
setup_clang_toolchain
# This doesn't go into CI but is available for developer convenience.
echo "bazel fastbuild build with tests..."
echo "Building..."
bazel_envoy_binary_build fastbuild
echo "Testing ${TEST_TARGETS[*]}"
bazel test "${BAZEL_BUILD_OPTIONS[@]}" \
-c fastbuild "${TEST_TARGETS[@]}"
;;
dev.contrib)
setup_clang_toolchain
# This doesn't go into CI but is available for developer convenience.
echo "bazel fastbuild build with contrib extensions and tests..."
echo "Building..."
bazel_contrib_binary_build fastbuild
echo "Testing ${TEST_TARGETS[*]}"
bazel test "${BAZEL_BUILD_OPTIONS[@]}" \
-c fastbuild \
"${TEST_TARGETS[@]}"
;;
distribution)
echo "Building distro packages..."
setup_clang_toolchain
# Extract the Envoy binary from the tarball
mkdir -p distribution/custom
if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then
ENVOY_RELEASE_TARBALL="/build/release/x64/bin/release.tar.zst"
else
ENVOY_RELEASE_TARBALL="/build/release/arm64/bin/release.tar.zst"
fi
bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
//tools/zstd \
-- --stdout \
-d "$ENVOY_RELEASE_TARBALL" \
| tar xfO - envoy > distribution/custom/envoy
bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
//tools/zstd \
-- --stdout \
-d "$ENVOY_RELEASE_TARBALL" \
| tar xfO - envoy-contrib > distribution/custom/envoy-contrib
# Build the packages
bazel build "${BAZEL_BUILD_OPTIONS[@]}" \
--remote_download_toplevel \
-c opt \
--//distribution:envoy-binary=//distribution:custom/envoy \
--//distribution:envoy-contrib-binary=//distribution:custom/envoy-contrib \
//distribution:packages.tar.gz
if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then
cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.x64.tar.gz"
else
cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.arm64.tar.gz"
fi
;;
docker)
# This is limited to linux x86/arm64 and expects `release` or `release.server_only` to have
# been run first.
if ! docker ps &> /dev/null; then
echo "Unable to build with Docker. If you are running with ci/run_envoy_docker.sh" \
"you should set ENVOY_DOCKER_IN_DOCKER=1"
exit 1
fi
if [[ -z "$CI_SHA1" ]]; then
CI_SHA1="$(git rev-parse HEAD~1)"
export CI_SHA1
fi
ENVOY_ARCH_DIR="$(dirname "${ENVOY_BUILD_DIR}")"
ENVOY_TARBALL_DIR="${ENVOY_TARBALL_DIR:-${ENVOY_ARCH_DIR}}"
_PLATFORMS=()
PLATFORM_NAMES=(
x64:linux/amd64
arm64:linux/arm64)
# TODO(phlax): avoid copying bins
for platform_name in "${PLATFORM_NAMES[@]}"; do
path="$(echo "${platform_name}" | cut -d: -f1)"
platform="$(echo "${platform_name}" | cut -d: -f2)"
bin_folder="${ENVOY_TARBALL_DIR}/${path}/bin"
if [[ ! -e "${bin_folder}/release.tar.zst" ]]; then
continue
fi
_PLATFORMS+=("$platform")
if [[ -e "$platform" ]]; then
rm -rf "$platform"
fi
mkdir -p "${platform}"
cp -a "${bin_folder}"/* "$platform"
done
if [[ -z "${_PLATFORMS[*]}" ]]; then
echo "No tarballs found in ${ENVOY_TARBALL_DIR}, did you run \`release\` first?" >&2
exit 1
fi
PLATFORMS="$(IFS=, ; echo "${_PLATFORMS[*]}")"
export DOCKER_PLATFORM="$PLATFORMS"
if [[ -z "${DOCKERHUB_PASSWORD}" && "${#_PLATFORMS[@]}" -eq 1 && -z $ENVOY_DOCKER_SAVE_IMAGE ]]; then
# if you are not pushing the images and there is only one platform
# then load to Docker (ie local build)
export DOCKER_LOAD_IMAGES=1
fi
"${ENVOY_SRCDIR}/ci/docker_ci.sh"
;;
docker-upload)
setup_clang_toolchain
bazel run //tools/gcs:upload \
"${GCS_ARTIFACT_BUCKET}" \
"${GCP_SERVICE_ACCOUNT_KEY_PATH}" \
"${BUILD_DIR}/build_images" \
"docker" \
"${GCS_REDIRECT_PATH}"
;;
dockerhub-publish)
setup_clang_toolchain
bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
//tools/distribution:update_dockerhub_repository
;;
dockerhub-readme)
setup_clang_toolchain
bazel build "${BAZEL_BUILD_OPTIONS[@]}" \
--remote_download_toplevel \
//distribution/dockerhub:readme
cat bazel-bin/distribution/dockerhub/readme.md
;;
docs)
setup_clang_toolchain
echo "generating docs..."
# Build docs.
[[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs
rm -rf "${DOCS_OUTPUT_DIR}"
mkdir -p "${DOCS_OUTPUT_DIR}"
if [[ -n "${CI_TARGET_BRANCH}" ]] || [[ -n "${SPHINX_QUIET}" ]]; then
export SPHINX_RUNNER_ARGS="-v warn"
BAZEL_BUILD_OPTIONS+=("--action_env=SPHINX_RUNNER_ARGS")
fi
if [[ -n "${DOCS_BUILD_RST}" ]]; then
bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:rst
cp bazel-bin/docs/rst.tar.gz "$DOCS_OUTPUT_DIR"/envoy-docs-rst.tar.gz
fi
DOCS_OUTPUT_DIR="$(realpath "$DOCS_OUTPUT_DIR")"
bazel "${BAZEL_STARTUP_OPTIONS[@]}" run \
"${BAZEL_BUILD_OPTIONS[@]}" \
--//tools/tarball:target=//docs:html \
//tools/tarball:unpack \
"$DOCS_OUTPUT_DIR"
;;
docs-upload)
setup_clang_toolchain
bazel run //tools/gcs:upload \
"${GCS_ARTIFACT_BUCKET}" \
"${GCP_SERVICE_ACCOUNT_KEY_PATH}" \
/source/generated/docs \
docs \
"${GCS_REDIRECT_PATH}"
;;
fetch|fetch-*)
case $CI_TARGET in
fetch)
targets=("${FETCH_TARGETS[@]}")
;;
fetch-check_and_fix_proto_format)
targets=("${FETCH_PROTO_TARGETS[@]}")
;;
fetch-docs)
targets=("${FETCH_DOCS_TARGETS[@]}")
;;
fetch-format)
targets=("${FETCH_FORMAT_TARGETS[@]}")
;;
fetch-gcc)
targets=("${FETCH_GCC_TARGETS[@]}")
;;
fetch-release|fetch-release.test_only)
targets=(
"${FETCH_BUILD_TARGETS[@]}"
"${FETCH_ALL_TEST_TARGETS[@]}")
;;
fetch-release.server_only)
targets=(
"${FETCH_BUILD_TARGETS[@]}")
;;
fetch-*coverage)
targets=("${FETCH_TEST_TARGETS[@]}")
;;
fetch-*san|fetch-compile_time_options)
targets=("${FETCH_ALL_TEST_TARGETS[@]}")
;;
fetch-api)
targets=("${FETCH_API_TARGETS[@]}")
;;
*)
exit 0
;;
esac
setup_clang_toolchain
FETCH_ARGS=(
--noshow_progress
--noshow_loading_progress)
echo "Fetching ${targets[*]} ..."
retry 15 10 bazel \
fetch \
"${BAZEL_GLOBAL_OPTIONS[@]}" \
"${FETCH_ARGS[@]}" \
"${targets[@]}"
;;
fix_proto_format)
# proto_format.sh needs to build protobuf.
setup_clang_toolchain
"${ENVOY_SRCDIR}/tools/proto_format/proto_format.sh" fix
;;
format)
setup_clang_toolchain
"${ENVOY_SRCDIR}/ci/format_pre.sh"
;;
fuzz)
setup_clang_toolchain
FUZZ_TEST_TARGETS=("$(bazel query "${BAZEL_GLOBAL_OPTIONS[@]}" "attr('tags','fuzzer',${TEST_TARGETS[*]})")")
echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS[*]}"
echo "Building envoy fuzzers and executing 100 fuzz iterations..."
bazel_with_collection \
test "${BAZEL_BUILD_OPTIONS[@]}" \
--config=asan-fuzzer \
"${FUZZ_TEST_TARGETS[@]}" \
--test_arg="-runs=10"
;;
gcc)
if [[ -n "${ENVOY_STDLIB}" && "${ENVOY_STDLIB}" != "libstdc++" ]]; then
echo "gcc toolchain doesn't support ${ENVOY_STDLIB}."
exit 1
fi
if [[ -n "${ENVOY_RBE}" ]]; then
CONFIG_PREFIX="remote-"
fi
CONFIG="${CONFIG_PREFIX}gcc"
BAZEL_BUILD_OPTIONS+=("--config=${CONFIG}")
echo "gcc toolchain configured: ${CONFIG}"
echo "Testing ${TEST_TARGETS[*]}"
bazel_with_collection \
test "${BAZEL_BUILD_OPTIONS[@]}" \
-c fastbuild \
--remote_download_minimal \
-- "${TEST_TARGETS[@]}"
echo "bazel release build with gcc..."
bazel_envoy_binary_build fastbuild
;;
info)
setup_clang_toolchain
bazel info "${BAZEL_BUILD_OPTIONS[@]}"
;;
msan)
setup_clang_toolchain
# rbe-toolchain-msan must comes as first to win library link order.
BAZEL_BUILD_OPTIONS=(
"--config=rbe-toolchain-msan"
"${BAZEL_BUILD_OPTIONS[@]}"
"-c" "dbg"
"--build_tests_only"
"--remote_download_minimal")
echo "bazel MSAN debug build with tests"
echo "Building and testing envoy tests ${TEST_TARGETS[*]}"
bazel_with_collection \
test "${BAZEL_BUILD_OPTIONS[@]}" \
-- "${TEST_TARGETS[@]}"
;;
publish)
setup_clang_toolchain
BUILD_SHA="$(git rev-parse HEAD)"
ENVOY_COMMIT="${ENVOY_COMMIT:-${BUILD_SHA}}"
ENVOY_REPO="${ENVOY_REPO:-envoyproxy/envoy}"
VERSION_DEV="$(cut -d- -f2 < VERSION.txt)"
PUBLISH_ARGS=(
--publish-commitish="$ENVOY_COMMIT"
--publish-commit-message
--publish-assets=/build/release.signed/release.signed.tar.zst)
if [[ "$VERSION_DEV" == "dev" ]] || [[ -n "$ENVOY_PUBLISH_DRY_RUN" ]]; then
PUBLISH_ARGS+=(--dry-run)
fi
bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
@envoy_repo//:publish \
-- --repo="$ENVOY_REPO" \
"${PUBLISH_ARGS[@]}"
;;
release|release.server_only|release.test_only)
if [[ "$CI_TARGET" == "release" || "$CI_TARGET" == "release.test_only" ]]; then
# When testing memory consumption, we want to test against exact byte-counts
# where possible. As these differ between platforms and compile options, we
# define the 'release' builds as canonical and test them only in CI, so the
# toolchain is kept consistent. This ifdef is checked in
# test/common/stats/stat_test_utility.cc when computing
# Memory::TestUtil::MemoryTest::mode().
if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then
BAZEL_BUILD_OPTIONS+=("--test_env=ENVOY_MEMORY_TEST_EXACT=true")
fi
fi
setup_clang_toolchain
# As the binary build package enforces compiler options, adding here to ensure the tests and distribution build
# reuse settings and any already compiled artefacts, the bundle itself will always be compiled
# `--stripopt=--strip-all -c opt`
BAZEL_RELEASE_OPTIONS=(
--stripopt=--strip-all
-c opt)
if [[ "$CI_TARGET" == "release" || "$CI_TARGET" == "release.test_only" ]]; then
# Run release tests
echo "Testing with:"
echo " targets: ${TEST_TARGETS[*]}"
echo " build options: ${BAZEL_BUILD_OPTIONS[*]}"
echo " release options: ${BAZEL_RELEASE_OPTIONS[*]}"
bazel_with_collection \
test "${BAZEL_BUILD_OPTIONS[@]}" \
--remote_download_minimal \
"${BAZEL_RELEASE_OPTIONS[@]}" \
"${TEST_TARGETS[@]}"
fi
if [[ "$CI_TARGET" == "release.test_only" ]]; then
exit 0
fi
ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin"
if [[ -e "${ENVOY_BINARY_DIR}" ]]; then
echo "Existing output directory found (${ENVOY_BINARY_DIR}), removing ..."
rm -rf "${ENVOY_BINARY_DIR}"
fi
mkdir -p "$ENVOY_BINARY_DIR"
# Build
echo "Building with:"
echo " build options: ${BAZEL_BUILD_OPTIONS[*]}"
echo " release options: ${BAZEL_RELEASE_OPTIONS[*]}"
echo " binary dir: ${ENVOY_BINARY_DIR}"
# Build release binaries
bazel build "${BAZEL_BUILD_OPTIONS[@]}" \
"${BAZEL_RELEASE_OPTIONS[@]}" \
--remote_download_outputs=toplevel \
//distribution/binary:release
# Copy release binaries to binary export directory
cp -a \
"bazel-bin/distribution/binary/release.tar.zst" \
"${ENVOY_BINARY_DIR}/release.tar.zst"
# Grab the schema_validator_tool
# TODO(phlax): bundle this with the release when #26390 is resolved
bazel build "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_RELEASE_OPTIONS[@]}" \
--remote_download_toplevel \
//test/tools/schema_validator:schema_validator_tool.stripped
# Copy schema_validator_tool to binary export directory
cp -a \
bazel-bin/test/tools/schema_validator/schema_validator_tool.stripped \
"${ENVOY_BINARY_DIR}/schema_validator_tool"
bazel build "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_RELEASE_OPTIONS[@]}" \
--remote_download_toplevel \
//test/tools/router_check:router_check_tool.stripped
cp -a \
bazel-bin/test/tools/router_check/router_check_tool.stripped \
"${ENVOY_BINARY_DIR}/router_check_tool"
echo "Release files created in ${ENVOY_BINARY_DIR}"
;;
release.server_only.binary)
setup_clang_toolchain
echo "bazel release build..."
bazel_envoy_binary_build release
;;
release.signed)
echo "Signing binary packages..."
setup_clang_toolchain
bazel build "${BAZEL_BUILD_OPTIONS[@]}" //distribution:signed
cp -a bazel-bin/distribution/release.signed.tar.zst "${BUILD_DIR}/envoy/"
;;
sizeopt)
setup_clang_toolchain
echo "Testing ${TEST_TARGETS[*]}"
bazel_with_collection \
test "${BAZEL_BUILD_OPTIONS[@]}" \
--config=sizeopt \
"${TEST_TARGETS[@]}"
echo "bazel size optimized build with tests..."
bazel_envoy_binary_build sizeopt
;;
sizeopt.server_only)
setup_clang_toolchain
echo "bazel size optimized build..."
bazel_envoy_binary_build sizeopt
;;
tsan)
setup_clang_toolchain
echo "bazel TSAN debug build with tests"
echo "Building and testing envoy tests ${TEST_TARGETS[*]}"
bazel_with_collection \
test "${BAZEL_BUILD_OPTIONS[@]}" \
--config=rbe-toolchain-tsan \
-c dbg \
--build_tests_only \
--remote_download_minimal \
"${TEST_TARGETS[@]}"
;;
verify_distro)
# this can be required if any python deps require compilation
setup_clang_toolchain
if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then
PACKAGE_BUILD=/build/distribution/x64/packages.x64.tar.gz
else
PACKAGE_BUILD=/build/distribution/arm64/packages.arm64.tar.gz
fi
bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
//distribution:verify_packages \
"$PACKAGE_BUILD"
;;
verify_examples)
DEV_CONTAINER_ID=$(docker inspect --format='{{.Id}}' envoyproxy/envoy:dev)
bazel run --config=ci \
--action_env="DEV_CONTAINER_ID=${DEV_CONTAINER_ID}" \
--host_action_env="DEV_CONTAINER_ID=${DEV_CONTAINER_ID}" \
--sandbox_writable_path="${HOME}/.docker/" \
--sandbox_writable_path="$HOME" \
@envoy_examples//:verify_examples
;;
verify.trigger)
setup_clang_toolchain
WORKFLOW="envoy-publish.yml"
# * Note on vars *
# `ENVOY_REPO`: Should always be envoyproxy/envoy unless testing
# `ENVOY_BRANCH`: Target branch for PRs, source branch for others