Reject duplicated build steps in the outcome file

tests/scripts/all-core.sh

@@ -212,6 +212,9 @@ pre_initialize_variables () {
     # Specify character collation for regular expressions and sorting with C locale
     export LC_COLLATE=C
 
+    # Make the location of the root directory available to reporting scripts
+    export MBEDTLS_TEST_ROOT="$PWD"
+
     : ${MBEDTLS_TEST_OUTCOME_FILE=}
     : ${MBEDTLS_TEST_PLATFORM="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"}
     export MBEDTLS_TEST_OUTCOME_FILE
@@ -790,6 +793,19 @@ pre_prepare_outcome_file () {
     fi
 }
 
+## write_outcome_line SUITE CASE RESULT [CAUSE]
+## Write a line in the outcome file if enabled.
+## Report $MBEDTLS_TEST_PLATFORM, $MBEDTLS_TEST_CONFIGURATION and the
+## supplied arguments.
+write_outcome_line () {
+    if [ -z "$MBEDTLS_TEST_OUTCOME_FILE" ]; then
+        return
+    fi
+    printf '%s;%s;%s;%s;%s;%s\n' >>"$MBEDTLS_TEST_OUTCOME_FILE" \
+           "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \
+           "$1" "$2" "$3" "${4-}"
+}
+
 pre_print_configuration () {
     if [ $QUIET -eq 1 ]; then
         return
@@ -983,6 +999,12 @@ run_component () {
         fi
     fi
 
+    if [ $component_status -eq 0 ]; then
+        write_outcome_line "all.sh" "whole" "PASS"
+    else
+        write_outcome_line "all.sh" "whole" "FAIL" "$component_status"
+    fi
+
     # Restore the build tree to a clean state.
     cleanup
     unset current_component

tests/scripts/components-build-system.sh

@@ -193,15 +193,18 @@ component_build_cmake_custom_config_file () {
     cd "$OUT_OF_SOURCE_DIR"
 
     # Build once to get the generated files (which need an intact config file)
+    MBEDTLS_TEST_CONFIGURATION="$current_component/out_of_tree/default"
     cmake "$MBEDTLS_ROOT_DIR"
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/out_of_tree/MBEDTLS_CONFIG_FILE"
     msg "build: cmake with -DMBEDTLS_CONFIG_FILE"
     scripts/config.py -w full_config.h full
     echo '#error "cmake -DMBEDTLS_CONFIG_FILE is not working."' > "$MBEDTLS_ROOT_DIR/$CONFIG_H"
     cmake -DGEN_FILES=OFF -DMBEDTLS_CONFIG_FILE=full_config.h "$MBEDTLS_ROOT_DIR"
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/out_of_tree/MBEDTLS_USER_CONFIG_FILE"
     msg "build: cmake with -DMBEDTLS_CONFIG_FILE + -DMBEDTLS_USER_CONFIG_FILE"
     # In the user config, disable one feature (for simplicity, pick a feature
     # that nothing else depends on).
@@ -217,6 +220,7 @@ component_build_cmake_custom_config_file () {
     rm -rf "$OUT_OF_SOURCE_DIR"
 
     # Now repeat the test for an in-tree build:
+    MBEDTLS_TEST_CONFIGURATION="$current_component/in_tree/default"
 
     # Restore config for the in-tree test
     mv include/mbedtls_config_in_tree_copy.h "$CONFIG_H"
@@ -225,12 +229,14 @@ component_build_cmake_custom_config_file () {
     cmake .
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/in_tree/MBEDTLS_CONFIG_FILE"
     msg "build: cmake (in-tree) with -DMBEDTLS_CONFIG_FILE"
     scripts/config.py -w full_config.h full
     echo '#error "cmake -DMBEDTLS_CONFIG_FILE is not working."' > "$MBEDTLS_ROOT_DIR/$CONFIG_H"
     cmake -DGEN_FILES=OFF -DMBEDTLS_CONFIG_FILE=full_config.h .
     make
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/in_tree/MBEDTLS_USER_CONFIG_FILE"
     msg "build: cmake (in-tree) with -DMBEDTLS_CONFIG_FILE + -DMBEDTLS_USER_CONFIG_FILE"
     # In the user config, disable one feature (for simplicity, pick a feature
     # that nothing else depends on).

tests/scripts/components-compiler.sh

@@ -25,6 +25,7 @@ test_build_opt () {
     info=$1 cc=$2; shift 2
     $cc --version
     for opt in "$@"; do
+          MBEDTLS_TEST_CONFIGURATION="$current_component$opt"
           msg "build/test: $cc $opt, $info" # ~ 30s
           make CC="$cc" CFLAGS="$opt -std=c99 -pedantic -Wall -Wextra -Werror"
           # We're confident enough in compilers to not run _all_ the tests,
@@ -82,18 +83,21 @@ support_test_gcc_earliest_opt () {
 }
 
 component_build_mingw () {
+    MBEDTLS_TEST_CONFIGURATION="$current_component/w64-mingw32/default/static"
     msg "build: Windows cross build - mingw64, make (Link Library)" # ~ 30s
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra -maes -msse2 -mpclmul' WINDOWS_BUILD=1 lib programs
 
     # note Make tests only builds the tests, but doesn't run them
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -maes -msse2 -mpclmul' WINDOWS_BUILD=1 tests
     make WINDOWS_BUILD=1 clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/w64-mingw32/default/shared"
     msg "build: Windows cross build - mingw64, make (DLL)" # ~ 30s
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra -maes -msse2 -mpclmul' WINDOWS_BUILD=1 SHARED=1 lib programs
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra -maes -msse2 -mpclmul' WINDOWS_BUILD=1 SHARED=1 tests
     make WINDOWS_BUILD=1 clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/w64-mingw32/no_AESNI/static"
     msg "build: Windows cross build - mingw64, make (Library only, default config without MBEDTLS_AESNI_C)" # ~ 30s
     ./scripts/config.py unset MBEDTLS_AESNI_C #
     make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib
@@ -133,6 +137,7 @@ component_test_zeroize () {
 
     for optimization_flag in -O2 -O3 -Ofast -Os; do
         for compiler in clang gcc; do
+            MBEDTLS_TEST_CONFIGURATION="$current_component/$optimization_flag/$compiler"
             msg "test: $compiler $optimization_flag, mbedtls_platform_zeroize()"
             make programs CC="$compiler" DEBUG=1 CFLAGS="$optimization_flag"
             gdb -ex "$gdb_disable_aslr" -x tests/scripts/test_zeroize.gdb -nw -batch -nx 2>&1 | tee test_zeroize.log

tests/scripts/components-configuration-crypto.sh

@@ -1448,7 +1448,6 @@ component_test_tfm_config_p256m_driver_accel_ec () {
 
     common_tfm_config
 
-    # Build crypto library
     make CC=$ASAN_CC CFLAGS="$ASAN_CFLAGS -I../tests/include/spe" LDFLAGS="$ASAN_CFLAGS"
 
     # Make sure any built-in EC alg was not re-enabled by accident (additive config)
@@ -1483,7 +1482,7 @@ component_test_tfm_config_no_p256m () {
     echo "#undef MBEDTLS_PSA_P256M_DRIVER_ENABLED" >> "$CONFIG_H"
 
     msg "build: TF-M config without p256m"
-    make CFLAGS='-Werror -Wall -Wextra -I../tests/include/spe' tests
+    make CFLAGS='-Werror -Wall -Wextra -I../tests/include/spe'
 
     # Check that p256m was not built
     not grep p256_ecdsa_ library/libmbedcrypto.a
@@ -2283,6 +2282,7 @@ component_build_aes_variations () {
     # Test that all the combinations build cleanly.
 
     MBEDTLS_ROOT_DIR="$PWD"
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both_directions"
     msg "build: aes.o for all combinations of relevant config options"
 
     build_test_config_combos ${BUILTIN_SRC_PATH}/aes.o validate_aes_config_variations \
@@ -2291,6 +2291,8 @@ component_build_aes_variations () {
         "MBEDTLS_AESNI_C" "MBEDTLS_AESCE_C" "MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH"
 
     cd "$MBEDTLS_ROOT_DIR"
+
+    MBEDTLS_TEST_CONFIGURATION="$current_component/no_decrypt"
     msg "build: aes.o for all combinations of relevant config options + BLOCK_CIPHER_NO_DECRYPT"
 
     # MBEDTLS_BLOCK_CIPHER_NO_DECRYPT is incompatible with ECB in PSA, CBC/XTS/NIST_KW/DES,
@@ -2336,11 +2338,13 @@ END
         #define PSA_WANT_ALG_SHA3_512  1
 END
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/unrolled"
     msg "all loops unrolled"
     make clean
     make -C tests ../tf-psa-crypto/tests/test_suite_shax CFLAGS="-DMBEDTLS_SHA3_THETA_UNROLL=1 -DMBEDTLS_SHA3_PI_UNROLL=1 -DMBEDTLS_SHA3_CHI_UNROLL=1 -DMBEDTLS_SHA3_RHO_UNROLL=1"
     ./tf-psa-crypto/tests/test_suite_shax
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/rolled"
     msg "all loops rolled up"
     make clean
     make -C tests ../tf-psa-crypto/tests/test_suite_shax CFLAGS="-DMBEDTLS_SHA3_THETA_UNROLL=0 -DMBEDTLS_SHA3_PI_UNROLL=0 -DMBEDTLS_SHA3_CHI_UNROLL=0 -DMBEDTLS_SHA3_RHO_UNROLL=0"
@@ -2368,11 +2372,13 @@ component_build_aes_aesce_armcc () {
     scripts/config.py unset MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT
     scripts/config.py set MBEDTLS_HAVE_ASM
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/both"
     msg "AESCE, build with default configuration."
     scripts/config.py set MBEDTLS_AESCE_C
     scripts/config.py unset MBEDTLS_AES_USE_HARDWARE_ONLY
     helper_armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8-a+crypto"
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/hardware"
     msg "AESCE, build AESCE only"
     scripts/config.py set MBEDTLS_AESCE_C
     scripts/config.py set MBEDTLS_AES_USE_HARDWARE_ONLY
@@ -2536,20 +2542,24 @@ component_test_block_cipher_no_decrypt_aesni () {
     config_block_cipher_no_decrypt
 
     # test AESNI intrinsics
+    MBEDTLS_TEST_CONFIGURATION="$current_component/AESNI/intrinsics"
     helper_block_cipher_no_decrypt_build_test \
         -s "MBEDTLS_AESNI_C" \
         -c "-mpclmul -msse2 -maes"
 
     # test AESNI assembly
+    MBEDTLS_TEST_CONFIGURATION="$current_component/AESNI/assembly"
     helper_block_cipher_no_decrypt_build_test \
         -s "MBEDTLS_AESNI_C" \
         -c "-mno-pclmul -mno-sse2 -mno-aes"
 
     # test AES C implementation
+    MBEDTLS_TEST_CONFIGURATION="$current_component/software"
     helper_block_cipher_no_decrypt_build_test \
         -u "MBEDTLS_AESNI_C"
 
     # test AESNI intrinsics for i386 target
+    MBEDTLS_TEST_CONFIGURATION="$current_component/AESNI/intrinsics/m32"
     helper_block_cipher_no_decrypt_build_test \
         -s "MBEDTLS_AESNI_C" \
         -c "-m32 -mpclmul -msse2 -maes" \
@@ -2671,6 +2681,7 @@ component_test_psa_crypto_drivers () {
 }
 
 component_build_psa_config_file () {
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_PSA_CRYPTO_CONFIG_FILE"
     msg "build: make with MBEDTLS_PSA_CRYPTO_CONFIG_FILE" # ~40s
     scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG
     cp "$CRYPTO_CONFIG_H" psa_test_config.h
@@ -2680,6 +2691,7 @@ component_build_psa_config_file () {
     programs/test/query_compile_time_config MBEDTLS_CMAC_C
     make clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_PSA_CRYPTO_USER_CONFIG_FILE"
     msg "build: make with MBEDTLS_PSA_CRYPTO_CONFIG_FILE + MBEDTLS_PSA_CRYPTO_USER_CONFIG_FILE" # ~40s
     # In the user config, disable one feature and its dependencies, which will
     # reflect on the mbedtls configuration so we can query it with

tests/scripts/components-configuration-tls.sh

@@ -184,39 +184,6 @@ component_test_config_thread () {
     tests/ssl-opt.sh -f 'ECJPAKE.*nolog'
 }
 
-# We're not aware of any other (open source) implementation of EC J-PAKE in TLS
-# that we could use for interop testing. However, we now have sort of two
-# implementations ourselves: one using PSA, the other not. At least test that
-# these two interoperate with each other.
-component_test_tls1_2_ecjpake_compatibility () {
-    msg "build: TLS1.2 server+client w/ EC-JPAKE w/o USE_PSA"
-    scripts/config.py set MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED
-    # Explicitly make lib first to avoid a race condition:
-    # https://github.com/Mbed-TLS/mbedtls/issues/8229
-    make lib
-    make -C programs ssl/ssl_server2 ssl/ssl_client2
-    cp programs/ssl/ssl_server2 s2_no_use_psa
-    cp programs/ssl/ssl_client2 c2_no_use_psa
-
-    msg "build: TLS1.2 server+client w/ EC-JPAKE w/ USE_PSA"
-    scripts/config.py set MBEDTLS_USE_PSA_CRYPTO
-    make clean
-    make lib
-    make -C programs ssl/ssl_server2 ssl/ssl_client2
-    make -C programs test/udp_proxy test/query_compile_time_config
-
-    msg "test: server w/o USE_PSA - client w/ USE_PSA, text password"
-    P_SRV=../s2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: working, TLS"
-    msg "test: server w/o USE_PSA - client w/ USE_PSA, opaque password"
-    P_SRV=../s2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: opaque password client only, working, TLS"
-    msg "test: client w/o USE_PSA - server w/ USE_PSA, text password"
-    P_CLI=../c2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: working, TLS"
-    msg "test: client w/o USE_PSA - server w/ USE_PSA, opaque password"
-    P_CLI=../c2_no_use_psa tests/ssl-opt.sh -f "ECJPAKE: opaque password server only, working, TLS"
-
-    rm s2_no_use_psa c2_no_use_psa
-}
-
 component_test_tls1_2_ccm_psk () {
     msg "build: configs/config-ccm-psk-tls1_2.h"
     cp configs/config-ccm-psk-tls1_2.h "$CONFIG_H"

tests/scripts/components-configuration.sh

@@ -237,9 +237,11 @@ component_build_tfm () {
     # configs/config-tfm.h, tested via test-ref-configs.pl.
     cp configs/config-tfm.h "$CONFIG_H"
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/thumb2/clang"
     msg "build: TF-M config, clang, armv7-m thumb2"
     make lib CC="clang" CFLAGS="--target=arm-linux-gnueabihf -march=armv7-m -mthumb -Os -std=c99 -Werror -Wall -Wextra -Wwrite-strings -Wpointer-arith -Wimplicit-fallthrough -Wshadow -Wvla -Wformat=2 -Wno-format-nonliteral -Wshadow -Wasm-operand-widths -Wunused -I../tests/include/spe"
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/native/gcc"
     msg "build: TF-M config, gcc native build"
     make clean
     make lib CC="gcc" CFLAGS="-Os -std=c99 -Werror -Wall -Wextra -Wwrite-strings -Wpointer-arith -Wshadow -Wvla -Wformat=2 -Wno-format-nonliteral -Wshadow -Wformat-signedness -Wlogical-op -I../tests/include/spe"
@@ -287,6 +289,7 @@ component_test_no_platform () {
 }
 
 component_build_mbedtls_config_file () {
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_PSA_CRYPTO_CONFIG_FILE"
     msg "build: make with MBEDTLS_CONFIG_FILE" # ~40s
     scripts/config.py -w full_config.h full
     echo '#error "MBEDTLS_CONFIG_FILE is not working"' >"$CONFIG_H"
@@ -295,6 +298,7 @@ component_build_mbedtls_config_file () {
     programs/test/query_compile_time_config MBEDTLS_NIST_KW_C
     make clean
 
+    MBEDTLS_TEST_CONFIGURATION="$current_component/MBEDTLS_USER_CONFIG_FILE"
     msg "build: make with MBEDTLS_CONFIG_FILE + MBEDTLS_USER_CONFIG_FILE"
     # In the user config, disable one feature (for simplicity, pick a feature
     # that nothing else depends on).