forked from RIOT-OS/RIOT
-
Notifications
You must be signed in to change notification settings - Fork 0
/
.murdock
executable file
·827 lines (718 loc) · 23.3 KB
/
.murdock
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
#!/bin/sh
# uncomment and change this to limit builds, e.g.,
#export BOARDS="samr21-xpro native"
# and / or
#export APPS="examples/hello-world tests/unittests"
QUICKBUILD_BOARDS="
adafruit-itsybitsy-m4
atmega256rfr2-xpro
esp32-wroom-32
esp32s3-devkit
frdm-k64f
hifive1b
msb-430
msba2
native
nrf52840dk
qn9080dk
samr21-xpro
stk3200
stm32f429i-disc1"
# this configures boards that are available via pifleet
case "${CI_MURDOCK_PROJECT}" in
riot)
: ${TEST_BOARDS_AVAILABLE:="esp32-wroom-32 samr21-xpro"}
;;
riot-staging)
: ${TEST_BOARDS_AVAILABLE:=""}
;;
*)
: ${TEST_BOARDS_AVAILABLE:=""}
;;
esac
#: ${EMULATED_BOARDS_AVAILABLE:="microbit"}
# temporarily disabling llvm builds until https://github.com/RIOT-OS/RIOT/pull/15595
# is in
#: ${TEST_BOARDS_LLVM_COMPILE:="iotlab-m3 native nrf52dk mulle nucleo-f401re samr21-xpro slstk3402a"}
: ${TEST_BOARDS_LLVM_COMPILE:=""}
: ${TEST_KCONFIG_BOARDS_AVAILABLE:="
adafruit-itsybitsy-m4
arduino-due
arduino-leonardo
arduino-mkrzero
arduino-mega2560
arduino-nano
atxmega-a1-xplained
atxmega-a3bu-xplained
avr-rss2
avsextrem
bastwan
bluepill
b-l475e-iot01a
cc1352-launchpad
cc2650-launchpad
derfmega128
dwm1001
esp32-ci
esp32-heltec-lora32-v2
esp32-olimex-evb
esp32c3-ci
esp32c3-devkit
esp32s3-ci
esp32s3-devkit
esp32s2-ci
esp32s2-devkit
esp8266-ci
esp8266-esp-12x
hamilton
hifive1
mbed_lpc1768
mega-xplained
microbit
mulle
native
nrf52840dk
nrf9160dk
nucleo-f072rb
nucleo-f103rb
nucleo-f207zg
nucleo-f334r8
nucleo-f429zi
nucleo-f767zi
nucleo-g071rb
nucleo-g474re
nucleo-l011k4
nucleo-l073rz
nucleo-l152re
nucleo-l433rc
nucleo-l552ze-q
pba-d-01-kw2x
p-nucleo-wb55
qn9080dk
remote-revb
rpi-pico
same54-xpro
samr21-xpro
seeedstudio-gd32
slstk3400a
sltb001a
slwstk6220a
stm32f723e-disco
stm32mp157c-dk2
waspmote-pro
waveshare-nrf52840-eval-kit
weact-f401ce
z1
"}
: ${TEST_KCONFIG_ENFORCE_APP_GROUPS:="
examples/hello-world
tests/build_system/external_board_dirs
tests/drivers/b*
tests/drivers/f*
tests/drivers/g*
tests/drivers/h*
tests/drivers/i*
tests/drivers/j*
tests/drivers/l*
tests/drivers/o*
tests/drivers/p*
tests/drivers/q*
tests/drivers/r*
tests/drivers/s*
tests/drivers/t*
tests/drivers/u*
tests/drivers/v*
tests/net/ieee802154_hal
tests/periph/*
tests/pkg/elk
tests/pkg/uzlib
tests/sys/cb_mux*
tests/sys/congure_*
tests/sys/prng_*
tests/sys/trace
tests/sys/xtimer_*
tests/sys/ztimer_*
"}
# This list prevents boards from being tested by the TEST_KCONFIG_TEST_ALLOWLIST
# This should only be used for boards that require a lot of work, such as,
# requiring modelling of a clock tree.
# As a rule of thumb, only add boards here if there are not differences in
# modules or packages.
# Eventually this list will be removed...
#
: ${TEST_KCONFIG_BOARD_BLOCKLIST:="
"}
# This list will force all boards that are not in the TEST_KCONFIG_BOARD_BLOCKLIST
# to be tested on the apps. The purpose is to prevent new boards from skipping
# the kconfig implementations.
: ${TEST_KCONFIG_TEST_ALLOWLIST:="
examples/hello-world
tests/drivers/saul
tests/drivers/mtd_mapper
"}
: ${TEST_WITH_CONFIG_SUPPORTED:="examples/suit_update tests/drivers/at86rf2xx_aes"}
export RIOT_CI_BUILD=1
export CC_NOCOLOR=1
export STATIC_TESTS=0
export CFLAGS_DBG=""
export DLCACHE_DIR=${DLCACHE_DIR:-~/.dlcache}
export ENABLE_TEST_CACHE=${ENABLE_TEST_CACHE:-1}
export MURDOCK_REDIS_HOST=${MURDOCK_REDIS_HOST:-127.0.0.1}
NIGHTLY=${NIGHTLY:-0}
check_label() {
local label="${1}"
[ -z "${CI_PULL_LABELS}" ] && return 1
echo "${CI_PULL_LABELS}" | grep -q "${label}"
return $?
}
# this function returns 0 when this is build is building one of the two bors
# branches ("staging" or "trying", 1 otherwise.
is_bors_build() {
test "${CI_BUILD_BRANCH}" = "staging" || test "${CI_BUILD_BRANCH}" = "trying"
}
# Set CI_BASE_COMMIT in case of a bors build.
# This assumes that bors always adds a single merge commit (possibly itself
# including multiple merges for PRs) on the base branch.
# That git command outputs the hash of the second merge commit (the one before
# the topmost merge), which should be the base branch HEAD.
# (CI_BASE_COMMIT is used by `can_fast_ci_run.py` to only build changes.)
if test -z "${CI_BASE_COMMIT}" && is_bors_build ; then
previous_merge="$(git log --merges --max-count 1 --skip 1 --pretty=format:%H)"
export CI_BASE_COMMIT="${previous_merge}"
fi
# fullbuild logic
# non-full-builds are those where can_fast_ci_run might reduce the build
# automatically
if [ -z "${FULL_BUILD}" ]; then
if [ "${NIGHTLY}" = 1 ]; then
FULL_BUILD=1
elif check_label "CI: full build"; then
# full build if requested by label
FULL_BUILD=1
else
FULL_BUILD=0
fi
export FULL_BUILD
fi
# quickbuild logic
# a "quickbuild" is only building a representative subset of all build
# configurations.
if [ -z "${QUICK_BUILD}" ]; then
export QUICK_BUILD=0
if is_bors_build ; then
# always do full build for bors' branches
true
elif [ ${FULL_BUILD} -eq 1 ]; then
# full build if building nightly or full build requested by label
true
else
export QUICK_BUILD=1
fi
fi
# This is a work around for a bug in CCACHE which interacts very badly with
# some features of RIOT and of murdock. The result is that ccache is
# ineffective (i.e. objects are never reused, resulting in extreme cache miss
# rate) and murdock becomes slow.
#
# - CCACHE thinks that -gz by itself enables debugging, which is not true.
# see https://github.com/ccache/ccache/issues/464
# - When debug info is included, CCACHE hashes the file paths, as these
# influence the debug information (the name of compile units and/or their
# "comp_dir" attribute)
# - Riot does not set -fdebug-prefix-map. This is not that easy as it may not
# be supported in every toolchain (some are quite old).
# - Murdock builds PRs in different directories each time.
#
# It is only the combination of these three factors which causes this bug.
export OPTIONAL_CFLAGS_BLACKLIST="-gz"
DWQ_ENV="-E BOARDS -E APPS -E NIGHTLY -E RUN_TESTS -E ENABLE_TEST_CACHE
-E TEST_HASH -E CI_PULL_LABELS -ECI_BASE_BRANCH -ECI_BASE_COMMIT
-EPKG_USE_MIRROR -EAPPS_CHANGED -EBOARDS_CHANGED -ESTATIC_TESTS
-E CI_MURDOCK_PROJECT -EFULL_BUILD -EQUICK_BUILD"
if [ ${NIGHTLY} -eq 1 ]; then
export PKG_USE_MIRROR=0
fi
CFCR_ARGS="--upstreambranch ${CI_BASE_COMMIT}"
get_supported_kconfig_board_app() {
local board=$1
local appdir=$2
if is_in_list "${board}" "${TEST_KCONFIG_BOARD_BLOCKLIST}"; then
return 1
fi
# On nightlies run all possible kconfig tests on all boards
# Normally we don't want to do this as it adds quite a bit to build time
# and the subset of boards we are testing for are pretty good at catching
# any bugs...
# This will be over one day, I promise.
if [ ${NIGHTLY} -eq 1 ]; then
if is_in_list "${appdir}" "${TEST_KCONFIG_ENFORCE_APP_GROUPS}"; then
return 0
fi
if [ -f "${appdir}/app.config.test" ]; then
return 0
fi
fi
if is_in_list "${appdir}" "${TEST_KCONFIG_TEST_ALLOWLIST}"; then
return 0
fi
if is_in_list "${board}" "${TEST_KCONFIG_BOARDS_AVAILABLE}"; then
if is_in_list "${appdir}" "${TEST_KCONFIG_ENFORCE_APP_GROUPS}"; then
return 0
fi
if [ -f "${appdir}/app.config.test" ]; then
return 0
fi
fi
return 1
}
kconfig_module_packages_diff() {
local board=$1
local appdir=$2
make clean --no-print-directory -C ${appdir} BOARD=${board}
mmp=$(make --no-print-directory info-modules info-packages -C ${appdir} BOARD=${board})
make clean --no-print-directory -C ${appdir} BOARD=${board}
kmp=$(TEST_KCONFIG=1 make --no-print-directory info-modules info-packages -C ${appdir} BOARD=${board})
bash -c "diff <(echo \"${mmp}\") <(echo \"${kmp}\")"
}
# if RUN_TESTS is unset (e.g., not passed from the outside),
# set to 1 if NIGHTLY=1 or if the label "CI: run tests" is set,
# otherwise set 0.
if [ -z "$RUN_TESTS" ]; then
if [ "$NIGHTLY" = "1" ] || check_label "CI: run tests" ; then
RUN_TESTS=1
else
RUN_TESTS=0
fi
fi
[ "$ENABLE_TEST_CACHE" = "1" ] && {
check_label "CI: disable test cache" && export ENABLE_TEST_CACHE=0
}
error() {
echo "$@"
exit 1
}
# true if "$2" starts with "$1", false otherwise
startswith() {
case "${2}" in
${1}*) true ;;
*) false ;;
esac
}
# if MURDOCK_HOOK is set, this function will execute it and pass on all it's
# parameters. should the hook script exit with negative exit code, hook() makes
# this script exit with error, too.
# hook() will be called from different locations of this script.
# currently, the only caller is "run_test", which calls "hook run_test_pre".
# More hooks will be added as needed.
hook() {
if [ -n "${MURDOCK_HOOK}" ]; then
echo "- executing hook $1"
"${MURDOCK_HOOK}" "$@" || {
error "$0: hook \"${MURDOCK_HOOK} $@\" failed!"
}
echo "- hook $1 finished"
fi
}
# true if word "$1" is in list of words "$2", false otherwise
# uses grep -w, thus only alphanum and "_" count as word bounderies
# (word "def" matches "abc-def")
is_in_list() {
[ $# -ne 2 ] && return 1
local needle="$1"
local haystack="$2"
echo "$haystack" | grep -q -w "$needle"
}
# grep that doesn't return error on empty input
_grep() {
grep "$@"
true
}
_greplist() {
if [ $# -eq 0 ]; then
echo cat
else
echo -n "_grep -E ($1"
shift
for i in $*; do
echo -n "|$i"
done
echo ")"
fi
}
# get list of all app directories
get_apps() {
make -f makefiles/app_dirs.inc.mk info-applications \
| $(_greplist $APPS) | sort
}
# take app dir as parameter, print all boards that are supported
# Only print for boards in $BOARDS.
get_supported_boards() {
local appdir=$1
local only_changed=0
if [ -n "$APPS_CHANGED" ]; then
if is_in_list "$appdir" "${APPS_CHANGED}"; then
# this app has changed -> build for all boards
true
else
# this is not a changed app -> build for changed boards
only_changed=1
fi
else
if [ -n "$BOARDS_CHANGED" ]; then
echo d
# no changed apps, some changed boards -> build for changed boards
only_changed=1
else
# no changed apps list, no changed boards list -> build all boards
true
fi
fi
BOARDS_=${BOARDS}
if [ $only_changed -eq 1 ]; then
# if can_fast_ci_run figured out changes to specific boards,
# only consider those
export BOARDS="$BOARDS_CHANGED"
elif [ -z "${BOARDS}" -a ${QUICK_BUILD} -eq 1 ]; then
# quickbuild board filter is applied here
export BOARDS="${QUICKBUILD_BOARDS}"
fi
local boards="$(make --no-print-directory -C$appdir info-boards-supported 2>/dev/null || echo broken)"
export BOARDS="${BOARDS_}"
if [ "$boards" = broken ]; then
echo "makefile_broken"
return
fi
for board in $boards
do
echo $board
done | $(_greplist $BOARDS)
}
get_supported_toolchains() {
local appdir=$1
local board=$2
local toolchains="gnu"
if is_in_list "${board}" "${TEST_BOARDS_LLVM_COMPILE}"; then
toolchains="$(make -s --no-print-directory -C${appdir} BOARD=${board} \
info-toolchains-supported 2> /dev/null | grep -o -e "llvm" -e "gnu")"
fi
echo "${toolchains}"
}
# given an app dir as parameter, print "$appdir $board:$toolchain" for each
# supported board and toolchain. Only print for boards in $BOARDS.
# if extra args are given, they will be prepended to each output line.
get_app_board_toolchain_pairs() {
local appdir=$1
local boards="$(get_supported_boards $appdir)"
# collect extra arguments into prefix variable
shift
local prefix="$*"
if [ "$boards" = makefile_broken ]; then
echo "$0 error \"error: ${DWQ_WORKER}: get_supported_boards failed in $appdir\""
return
fi
for board in ${boards}
do
for toolchain in $(get_supported_toolchains $appdir $board)
do
echo $prefix $appdir $board:$toolchain
done
done | $(_greplist $BOARDS)
}
# use dwqc to create full "appdir board toolchain" compile job list
get_compile_jobs() {
check_label "CI: skip compile test" && return
update_changed_modules || return
get_apps | \
maybe_filter_changed_apps | \
dwqc ${DWQ_ENV} --queue default-first -s \
${DWQ_JOBID:+--subjob} \
"$0 get_app_board_toolchain_pairs \${1} $0 compile"
}
print_worker() {
[ -n "$DWQ_WORKER" ] && \
echo "-- running on worker ${DWQ_WORKER} thread ${DWQ_WORKER_THREAD}, build number $DWQ_WORKER_BUILDNUM."
}
test_hash_calc() {
local bindir=$1
# Why two times cut?
# "test-input-hash.sha1" contains a list of lines containing
# "<hash> <filename>" on each line.
# We need to filter out the filename, as it contains the full path name,
# which differs depending on the build machine.
#
# After piping through sha1sum, we get "<hash> -". " -" has to go so we save the
# hassle of escaping the resulting hash.
cat ${bindir}/test-input-hash.sha1 | cut -f1 -d' ' | sha1sum | cut -f1 -d' '
}
test_cache_get() {
test "${ENABLE_TEST_CACHE}" = "1" || return 1
test -n "$(redis-cli -h ${MURDOCK_REDIS_HOST} get $1)" > /dev/null
}
test_cache_put() {
redis-cli -h ${MURDOCK_REDIS_HOST} set "$1" ok
}
# compile one app for one board with one toolchain. delete intermediates.
compile() {
local appdir=$1
local board=$(echo $2 | cut -f 1 -d':')
local toolchain=$(echo $2 | cut -f 2 -d':')
[ "$board" = "makefile_broken" ] && {
echo "$0: There seems to be a problem in \"$appdir\" while getting supported boards!"
echo "$0: testing \"make -C$appdir info-boards-supported\"..."
make -C$appdir info-boards-supported && echo "$0: success. no idea what's wrong." || echo "$0: failed!"
exit 1
}
# set build directory. CI ensures only one build at a time in $(pwd).
export BINDIR="$(pwd)/build"
export PKGDIRBASE="${BINDIR}/pkg"
# Pre-build cleanup
rm -rf ${BINDIR}
print_worker
# sanity checks
[ $# -ne 2 ] && error "$0: compile: invalid parameters (expected \$appdir \$board:\$toolchain)"
[ ! -d "$appdir" ] && error "$0: compile: error: application directory \"$appdir\" doesn't exist"
# use our checkout as ccache temporary folder.
# On CI, this is a tmpfs, so using that instead of the `/cache/.ccache` volume
# reduces disk IO.
export CCACHE_TEMPDIR="$(pwd)/.ccache/tmp"
# We compile a first time with Kconfig based dependency
# resolution for regression purposes. $TEST_KCONFIG contains a
# list of board-application tuples that are currently modeled to
# run with Kconfig
should_check_kconfig_hash=0
if get_supported_kconfig_board_app "${board}" "${appdir}"; then
should_check_kconfig_hash=1
# As we have some issues with occasional unrelated hash mismatches
# we will stop the binary checks and rely only in a module/package
# check to ensure kconfig is matching make.
# Only nightlies will check the hash...
# Once we figure out the problem we can check the hashes again or
# better yet just finish!
if [ ${NIGHTLY} -eq 1 ]; then
BOARD=${board} make -C${appdir} clean
CCACHE_BASEDIR="$(pwd)" BOARD=${board} TOOLCHAIN=${toolchain} RIOT_CI_BUILD=1 TEST_KCONFIG=1 \
make -C${appdir} all test-input-hash -j${JOBS:-4}
RES=$?
if [ $RES -eq 0 ]; then
kconfig_hashes="$(cat ${BINDIR}/test-input-hash.sha1)"
else
kconfig_hashes="kconfig-build-failed"
echo "An error occurred while compiling using Kconfig";
fi
fi
fi
# compile without Kconfig
BOARD=${board} make -C${appdir} clean
CCACHE_BASEDIR="$(pwd)" BOARD=$board TOOLCHAIN=$toolchain RIOT_CI_BUILD=1 \
make -C${appdir} all test-input-hash -j${JOBS:-4}
RES=$?
if [ ${NIGHTLY} -eq 1 ]; then
no_kconfig_hashes="$(cat ${BINDIR}/test-input-hash.sha1)"
fi
# test hash is used to cache test results, not for comparing binaries
# generated with and without KConfig
test_hash=$(test_hash_calc "$BINDIR")
if [ ${should_check_kconfig_hash} != 0 ]; then
if [ ${NIGHTLY} -eq 1 ]; then
if [ "${kconfig_hashes}" != "${no_kconfig_hashes}" ]; then
echo "Hashes of binaries with and without Kconfig mismatch for ${appdir} with ${board}";
echo "Please check that all used modules are modelled in Kconfig and enabled";
echo "Input without KConfig:"
echo "${no_kconfig_hashes}"
echo "Input with KConfig:"
echo "${kconfig_hashes}"
kconfig_module_packages_diff ${board} ${appdir}
RES=1
fi
else
kconfig_module_packages_diff ${board} ${appdir}
RES=$(( $RES | $? ))
fi
fi
# run tests
if [ $RES -eq 0 ]; then
if is_in_list "$board" "$EMULATED_BOARDS_AVAILABLE"; then
EMULATED=1
else
EMULATED=0
fi
if [ $RUN_TESTS -eq 1 -o "$board" = "native" -o "$EMULATED" = "1" ]; then
if [ -f "${BINDIR}/.test" ]; then
if [ "$board" = "native" -o "${EMULATED}" = "1" ]; then
EMULATE=${EMULATED} BOARD=$board make -C${appdir} test
RES=$?
elif is_in_list "$board" "$TEST_BOARDS_AVAILABLE"; then
echo "-- test_hash=$test_hash"
if test_cache_get $test_hash; then
echo "-- skipping test due to positive cache hit"
else
BOARD=$board TOOLCHAIN=$toolchain TEST_HASH=$test_hash \
make -C${appdir} test-murdock
RES=$?
fi
fi
fi
fi
fi
# log some build stats
if [ -d ${BINDIR} ]
then
echo "{\"build/\": $(du -s ${BINDIR} | cut -f1)}"
# cleanup
rm -rf ${BINDIR}
fi
return $RES
}
test_job() {
local appdir=$1
local board=$(echo $2 | cut -f 1 -d':')
local toolchain=$(echo $2 | cut -f 2 -d':')
# interpret any extra arguments as file names.
# They will be sent along with the job to the test worker
# and stored in the application's binary folder.
shift 2
local files=""
for filename in "$@"; do
# check if the file is within $(BINDIR)
if startswith "${BINDIR}" "${filename}"; then
# get relative (to $(BINDIR) path
local relpath="$(realpath --relative-to ${BINDIR} ${filename})"
else
error "$0: error: extra test files not within \${BINDIR}!"
fi
# set remote file path.
# currently, the test workers build in the default build path.
local remote_bindir="${appdir}/bin/${board}"
files="${files} --file ${filename}:${remote_bindir}/${relpath}"
done
dwqc \
${DWQ_ENV} \
${DWQ_JOBID:+--subjob} \
--queue ${TEST_QUEUE:-$board} \
--maxfail 1 \
$files \
"./.murdock run_test $appdir $board:$toolchain"
}
run_test() {
local appdir=$1
local board=$(echo $2 | cut -f 1 -d':')
local toolchain=$(echo $2 | cut -f 2 -d':')
print_worker
echo "-- executing tests for $appdir on $board (compiled with $toolchain toolchain):"
hook run_test_pre
# do flashing and building of termdeps simultaneously
BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir flash-only termdeps -j2
# now run the actual test
if is_in_list "${appdir}" "${TEST_WITH_CONFIG_SUPPORTED}"; then
BOARD=${board} TOOLCHAIN=${toolchain} make -C${appdir} test-with-config
else
BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir test
fi
RES=$?
if [ $RES -eq 0 -a -n "$TEST_HASH" ]; then
echo -n "-- saving test result to cache: "
test_cache_put $TEST_HASH
fi
return $RES
}
basename_list () {
for path in $*; do
basename $path
done
}
# calls out to can_fast_ci_run.py.
#
# returns 1 if nothing should be built.
# or, returns 0, potentially meddling with BOARDS, APPS, BOARDS_CHANGED, APPS_CHANGED.
update_changed_modules() {
# early out if there's no base commit info
if [ -z "${CI_BASE_COMMIT}" ]; then
return 0
fi
# early out if a full build is requested
if [ $FULL_BUILD -eq 1 ]; then
return 0
fi
# if these are set, just build what's requested.
if [ -n "$BOARDS" -o -n "$APPS" ]; then
return 0
fi
# do full build if requested by label
check_label "CI: full build" && return 0
# call out to can_fast_ci_run.py
# that script will output e.g.,
# APPS_CHANGED="foo/bar foo/barfoo"
# BOARDS_CHANGED="foo/bar foo/barfoo"
# just eval that output, so we set those variables directly in this shell.
eval $(dist/tools/ci/can_fast_ci_run.py \
${CFCR_ARGS} \
--changed-boards --changed-apps \
2>/dev/null || echo CFCR_ERROR=1)
# if this errors, it means we cannot skip any builds.
if [ -n "$CFCR_ERROR" ]; then
return 0
fi
# if can_fast_ci_run exits 0 but doesn't output any changed apps or boards,
# it means we can skip all compile jobs.
if [ -z "$APPS_CHANGED" -a -z "$BOARDS_CHANGED" ]; then
return 1
fi
# can_fast_ci_run.py outputs "board/fooboard", but the rest of this script
# expects a board name without leading "board/".
if [ -n "$BOARDS_CHANGED" ]; then
export BOARDS_CHANGED="$(basename_list $BOARDS_CHANGED)"
fi
# if no board has changed, filter by just setting APPS.
# same for apps -> BOARDS.
if [ -z "$APPS_CHANGED" -a -n "$BOARDS_CHANGED" ]; then
export BOARDS="$(echo $BOARDS_CHANGED | $(_greplist ${BOARDS}))"
unset BOARDS_CHANGED
elif [ -n "$APPS_CHANGED" -a -z "$BOARDS_CHANGED" ]; then
export APPS="$(echo $APPS_CHANGED | $(_greplist ${APPS}))"
unset APPS_CHANGED
fi
export APPS_CHANGED
export BOARDS_CHANGED
}
maybe_filter_changed_apps() {
# if no boards have changes, only a subset of the apps need to be built.
# else, all apps need to be passed, as they'll need to be built for
# the changed boards.
if [ -n "$BOARDS_CHANGED" ]; then
cat
else
$(_greplist $APPS_CHANGED)
fi
}
# execute static tests
static_tests() {
print_worker
build_filter_status
[ "$STATIC_TESTS" = "1" ] && \
./dist/tools/ci/static_tests.sh
true
}
build_filter_status() {
echo "--- can_fast_ci_run:"
if [ $FULL_BUILD -eq 1 ]; then
echo "--- doing full build."
return
fi
dist/tools/ci/can_fast_ci_run.py ${CFCR_ARGS} --explain --json --changed-boards --changed-apps
if [ -n "$MURDOCK_TEST_CHANGE_FILTER" ]; then
echo MURDOCK_TEST_CHANGE_FILTER=$MURDOCK_TEST_CHANGE_FILTER
fi
echo ""
update_changed_modules
if [ -n "$CFCR_ERROR" ]; then
echo "-- can_fast_ci_run.py exited non-zero"
fi
}
get_non_compile_jobs() {
echo "$0 static_tests"
}
get_jobs() {
get_non_compile_jobs
get_compile_jobs
}
$*